From 45a894cf56243ba2f5e88950f8cbf8c2f8908315 Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Mon, 19 Oct 2026 17:50:04 +0200 Subject: [PATCH 1/2] fix(server): refuse what a chat backend cannot serve in the gateway's words sie-server's /v1/chat/completions, which its /v1/messages serves through, now refuses before the engine runs, with the gateway's status, envelope and wording: tools on a model that does not declare tool calling, and an image the serving path cannot read. Only the SGLang child on CUDA reads images; the MLX child reads text only and adapter-owned chat carries no media. Tool calling is checked first, as on the gateway. The Messages route turns these into "tools: ..." and the path of the first image block, the refusal Claude Code recovers from. The fake model declares the tool calling its scripted chat does. The MLX backend enforces the declared window, as SGLang's --context-length does: the prompt is counted as mlx_lm.server renders it, with the repo's tokenizer and chat template, and a prompt that does not fit with its max_tokens is refused with the worker's wording. The fake engine's window refusal carries its counts in the same wording, so both reach Messages clients as "prompt is too long: T tokens > C maximum". New shared vectors in conformance/anthropic_messages/capability_vectors.json pin both chat routes' refusals and the Messages answers. The gateway's and the server's tests replay each vector through both routes. --- .../capability_vectors.json | 140 ++++++++++ .../sie_gateway/docs/architecture-guide.md | 2 +- .../src/handlers/messages/tests.rs | 63 ++++- packages/sie_server/models/sie-fake.yaml | 1 + .../src/sie_server/adapters/fake/adapter.py | 12 +- .../src/sie_server/adapters/mlx/generation.py | 62 ++++- .../src/sie_server/api/openai_local.py | 26 +- .../tests/adapters/test_sglang_chat_tools.py | 7 +- .../api/test_anthropic_messages_route.py | 65 ++++- .../tests/api/test_fake_chat_route.py | 28 +- .../sie_server/tests/api/test_openai_local.py | 240 +++++++++++++++++- .../sie_server/tests/api/test_remote_chat.py | 50 ++-- 12 files changed, 662 insertions(+), 34 deletions(-) create mode 100644 conformance/anthropic_messages/capability_vectors.json diff --git a/conformance/anthropic_messages/capability_vectors.json b/conformance/anthropic_messages/capability_vectors.json new file mode 100644 index 000000000..7c060aa4a --- /dev/null +++ b/conformance/anthropic_messages/capability_vectors.json @@ -0,0 +1,140 @@ +{ + "description": "A Messages request to a generation model that declares the capabilities listed, the answer the chat route gives its translation before any engine runs, and the Anthropic answer the Messages route gives. Tool calling is checked before image input. Headers not listed under expect.headers are not asserted.", + "refusals": [ + { + "name": "tools on a model without tool calling", + "capabilities": {"tools": false, "image": false}, + "request": { + "model": "acme/text", + "max_tokens": 16, + "tools": [{"name": "Read", "input_schema": {"type": "object", "properties": {"file_path": {"type": "string"}}, "required": ["file_path"]}}], + "messages": [{"role": "user", "content": "Read notes.txt."}] + }, + "chat": {"status": 400, "body": {"error": {"message": "Model 'acme/text' does not support tool calling", "type": "invalid_request_error", "param": "tools", "code": "unsupported_field"}}}, + "expect": {"status": 400, "type": "invalid_request_error", "message": "tools: Model 'acme/text' does not support tool calling", "headers": {"x-sie-error-code": "unsupported_field"}} + }, + { + "name": "tools with tool_choice none on a model without tool calling", + "capabilities": {"tools": false, "image": false}, + "request": { + "model": "acme/text", + "max_tokens": 16, + "stream": true, + "tools": [{"name": "Read", "input_schema": {"type": "object", "properties": {"file_path": {"type": "string"}}}}], + "tool_choice": {"type": "none"}, + "messages": [{"role": "user", "content": "Say hello."}] + }, + "chat": {"status": 400, "body": {"error": {"message": "Model 'acme/text' does not support tool calling", "type": "invalid_request_error", "param": "tools", "code": "unsupported_field"}}}, + "expect": {"status": 400, "type": "invalid_request_error", "message": "tools: Model 'acme/text' does not support tool calling", "headers": {"x-sie-error-code": "unsupported_field"}} + }, + { + "name": "tool calling is checked before image input", + "capabilities": {"tools": false, "image": false}, + "request": { + "model": "acme/text", + "max_tokens": 16, + "tools": [{"name": "Read", "input_schema": {"type": "object", "properties": {"file_path": {"type": "string"}}}}], + "messages": [ + { + "role": "user", + "content": [ + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="}}, + {"type": "text", "text": "What is in this image?"} + ] + } + ] + }, + "chat": {"status": 400, "body": {"error": {"message": "Model 'acme/text' does not support tool calling", "type": "invalid_request_error", "param": "tools", "code": "unsupported_field"}}}, + "expect": {"status": 400, "type": "invalid_request_error", "message": "tools: Model 'acme/text' does not support tool calling", "headers": {"x-sie-error-code": "unsupported_field"}} + }, + { + "name": "tools on a vision model without tool calling", + "capabilities": {"tools": false, "image": true}, + "request": { + "model": "acme/vision", + "max_tokens": 16, + "tools": [{"name": "Read", "input_schema": {"type": "object", "properties": {"file_path": {"type": "string"}}}}], + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is in this image?"}, + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="}} + ] + } + ] + }, + "chat": {"status": 400, "body": {"error": {"message": "Model 'acme/vision' does not support tool calling", "type": "invalid_request_error", "param": "tools", "code": "unsupported_field"}}}, + "expect": {"status": 400, "type": "invalid_request_error", "message": "tools: Model 'acme/vision' does not support tool calling", "headers": {"x-sie-error-code": "unsupported_field"}} + }, + { + "name": "image in a user turn on a model without image input", + "capabilities": {"tools": true, "image": false}, + "request": { + "model": "acme/coder", + "max_tokens": 16, + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is in this image?"}, + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="}} + ] + } + ] + }, + "chat": {"status": 400, "body": {"error": {"message": "Model 'acme/coder' does not support image input", "type": "invalid_request_error", "param": "messages", "code": "unsupported_field"}}}, + "expect": {"status": 400, "type": "invalid_request_error", "message": "messages.0.content.1.image: Model 'acme/coder' does not support image input", "headers": {"x-sie-error-code": "unsupported_field"}} + }, + { + "name": "image in a tool result on a model without image input", + "capabilities": {"tools": true, "image": false}, + "request": { + "model": "acme/coder", + "max_tokens": 16, + "stream": true, + "tools": [{"name": "Read", "input_schema": {"type": "object", "properties": {"file_path": {"type": "string"}}, "required": ["file_path"]}}], + "messages": [ + {"role": "user", "content": "Read shot.png and describe it."}, + {"role": "assistant", "content": [{"type": "tool_use", "id": "toolu_01", "name": "Read", "input": {"file_path": "shot.png"}}]}, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_01", + "content": [ + {"type": "text", "text": "Read image (1x1)"}, + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="}} + ] + } + ] + } + ] + }, + "chat": {"status": 400, "body": {"error": {"message": "Model 'acme/coder' does not support image input", "type": "invalid_request_error", "param": "messages", "code": "unsupported_field"}}}, + "expect": {"status": 400, "type": "invalid_request_error", "message": "messages.2.content.0.tool_result.content.1.image: Model 'acme/coder' does not support image input", "headers": {"x-sie-error-code": "unsupported_field"}} + }, + { + "name": "the first of several images is named", + "capabilities": {"tools": false, "image": false}, + "request": { + "model": "acme/text", + "max_tokens": 16, + "messages": [ + {"role": "user", "content": [{"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="}}]}, + {"role": "assistant", "content": "A white pixel."}, + { + "role": "user", + "content": [ + {"type": "text", "text": "And this one?"}, + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="}} + ] + } + ] + }, + "chat": {"status": 400, "body": {"error": {"message": "Model 'acme/text' does not support image input", "type": "invalid_request_error", "param": "messages", "code": "unsupported_field"}}}, + "expect": {"status": 400, "type": "invalid_request_error", "message": "messages.0.content.0.image: Model 'acme/text' does not support image input", "headers": {"x-sie-error-code": "unsupported_field"}} + } + ] +} diff --git a/packages/sie_gateway/docs/architecture-guide.md b/packages/sie_gateway/docs/architecture-guide.md index 2a385e1c0..556cd5568 100644 --- a/packages/sie_gateway/docs/architecture-guide.md +++ b/packages/sie_gateway/docs/architecture-guide.md @@ -46,7 +46,7 @@ Key terms used throughout this document: - `POST /v1/score/{*model}` - `POST /v1/extract/{*model}` - `POST /v1/embeddings` — OpenAI-compatible JSON surface; the gateway accepts **string** or **list of strings** for `input`, supports `encoding_format=float` (default) or `base64`, translates to an internal **`POST /v1/encode/{model}`** with `items[].text` and `params.output_types=["dense"]`, then maps encode `items[].dense` vectors into OpenAI `data[].embedding` plus a rough `usage` estimate. Token-id / nested-array inputs are rejected with **`400`**. `dimensions` is accepted only when it equals the model's native dense width and otherwise rejected with **`400 unsupported_field`**, the same rule as the single-node server. The wrapper must buffer the whole encode reply before reshaping it, and that reply is an *amplification* of the request (N short texts in, N × `dim` floats out), so the buffer is bounded by its own **response**-specific cap, `MAX_COMPAT_RESPONSE_BODY` (16 MiB) — deliberately a separate constant from the request cap it used to be derived from, since the two quantities are unrelated and a change to ingress must not silently move this bound. The value holds today's effective bound rather than raising it: at the queue's item ceiling a float reply reaches ~90 MB at 1024 dims and ~224 MB at 2560 dims (`Qwen/Qwen3-Embedding-4B`, served here), the handler holds ~4 copies at once, and the deployed gateway is a 1 GiB container running 4 requests concurrently — so no bound both admits the largest real reply and fits. **Consequence, stated plainly: a batch above roughly 760 inputs at 1024 dims cannot be served on this surface.** The cure is an ingress item cap (terminal `400` before any GPU work), streaming assembly, or a larger container — each its own change. Over the cap the surface answers a **terminal `413`** naming the one remedy that works (fewer inputs — `encoding_format=base64` is inert here, because the field is never forwarded to `/v1/encode` and base64 is applied afterwards to the outer response), never a retryable `503`. Both that `413` and the unreadable-body `500` carry `GatewayOwnedFault`: the worker has already run and reported its units, so a metered composition must release the hold rather than bill for a response carrying zero embeddings. - - `POST /v1/messages` — Anthropic Messages compatibility surface (`handlers::messages`). The handler validates the body in Anthropic terms, translates it into a `/v1/chat/completions` body, awaits `proxy_chat` in process on the same task, and translates the answer back: a chat completion into a Message, chat SSE chunks into Messages events, and every error into the Anthropic envelope `{"type":"error","error":{"type","message"},"request_id"}` with the status the chat route chose. Every chat gate, profile limit, timeout, serving disclosure and metering point therefore applies unchanged. Unknown fields fail with `400 : Extra inputs are not permitted`; `metadata`, every `cache_control`, `thinking.display: "summarized"`, tool `eager_input_streaming` and the `anthropic-beta` header are accepted and have no effect, and `metadata` is never logged or forwarded. `thinking` `enabled` or `adaptive` sets the chat request's `include_reasoning`, and reasoning comes back as `thinking` blocks with an opaque signature. Usage reports `prompt_tokens - cached_tokens` as `input_tokens` and the cached part as `cache_read_input_tokens`, so the two add up to the chat route's prompt tokens. A stream sends `message_start` with zero usage when the chat stream opens, the authoritative usage on `message_delta`, a `ping` whenever nothing was sent for 10 seconds, and a mid-stream failure as an `error` event without `message_stop`. Dropping the response body drops the chat body, so a client disconnect cancels the generation as on chat. The Anthropic error envelope is also an outer middleware, `anthropic_error_envelope`, applied to `/v1/messages` and every path below it, so auth refusals, unknown sub-paths such as `/v1/messages/count_tokens` (not served) and the method check take the same shape; a composition that adds admission layers applies it outermost. `messages_dispatch_inputs` composes the translation with `chat_completions_dispatch_inputs` for estimates and admission checks. Shared vectors under `conformance/anthropic_messages/` pin the request translation, the buffered answer, the event sequence and the error mapping. + - `POST /v1/messages` — Anthropic Messages compatibility surface (`handlers::messages`). The handler validates the body in Anthropic terms, translates it into a `/v1/chat/completions` body, awaits `proxy_chat` in process on the same task, and translates the answer back: a chat completion into a Message, chat SSE chunks into Messages events, and every error into the Anthropic envelope `{"type":"error","error":{"type","message"},"request_id"}` with the status the chat route chose. Every chat gate, profile limit, timeout, serving disclosure and metering point therefore applies unchanged. Unknown fields fail with `400 : Extra inputs are not permitted`; `metadata`, every `cache_control`, `thinking.display: "summarized"`, tool `eager_input_streaming` and the `anthropic-beta` header are accepted and have no effect, and `metadata` is never logged or forwarded. `thinking` `enabled` or `adaptive` sets the chat request's `include_reasoning`, and reasoning comes back as `thinking` blocks with an opaque signature. Usage reports `prompt_tokens - cached_tokens` as `input_tokens` and the cached part as `cache_read_input_tokens`, so the two add up to the chat route's prompt tokens. A stream sends `message_start` with zero usage when the chat stream opens, the authoritative usage on `message_delta`, a `ping` whenever nothing was sent for 10 seconds, and a mid-stream failure as an `error` event without `message_stop`. Dropping the response body drops the chat body, so a client disconnect cancels the generation as on chat. The Anthropic error envelope is also an outer middleware, `anthropic_error_envelope`, applied to `/v1/messages` and every path below it, so auth refusals, unknown sub-paths such as `/v1/messages/count_tokens` (not served) and the method check take the same shape; a composition that adds admission layers applies it outermost. `messages_dispatch_inputs` composes the translation with `chat_completions_dispatch_inputs` for estimates and admission checks. Shared vectors under `conformance/anthropic_messages/` pin the request translation, the buffered answer, the event sequence, the error mapping, and the tool-calling and image-input refusals both chat routes give before any engine runs. 2. Gateway resolves the request to a model, a bundle, a machine profile, a logical API pool, and the backing queue pool using its in-memory registry. 3. Gateway ordinarily publishes work to JetStream on the backing queue subject `sie.work.{pool}.{machine_profile}.{bundle}.{model}` and carries the logical pool as `admission_pool` in the work item. A capped logical batch pool instead targets one assigned worker on `sie.work.{pool}.{machine_profile}.{bundle}.{model}.{worker_id}`. 4. A matching worker consumes the work item, verifies its logical pool assignment when `admission_pool` differs from its physical queue pool, and executes inference. diff --git a/packages/sie_gateway/src/handlers/messages/tests.rs b/packages/sie_gateway/src/handlers/messages/tests.rs index 565b635b9..665073b8f 100644 --- a/packages/sie_gateway/src/handlers/messages/tests.rs +++ b/packages/sie_gateway/src/handlers/messages/tests.rs @@ -16,7 +16,7 @@ use super::request::{messages_to_chat, MAX_TOOLS}; use super::response::chat_completion_to_message; use super::stream::{stream_body, FrameParser, StreamMachine}; use super::{messages_dispatch_inputs, proxy_messages}; -use crate::handlers::proxy::chat_completions_dispatch_inputs; +use crate::handlers::proxy::{chat_completions_dispatch_inputs, proxy_chat}; use crate::handlers::test_support::{TestGateway, HYBRID_GENERATE_MODEL, LOCAL_LANE}; const CODER_MODEL: &str = "\ @@ -799,6 +799,67 @@ async fn a_tool_request_on_a_tool_model_is_dispatched() { assert_eq!(gateway.dispatcher.dispatched().len(), 1); } +fn capability_model(model: &str, capabilities: &Value) -> String { + let tools = capabilities["tools"].as_bool().unwrap(); + let image = capabilities["image"].as_bool().unwrap(); + format!( + "sie_id: {model}\nhf_id: {model}\ninputs:\n text: true\n image: {image}\ntasks:\n generate:\n max_output_tokens: 64\n capabilities:\n tools: {tools}\nprofiles:\n default:\n adapter_path: sie_server.adapters.sglang:SGLangAdapter\n max_batch_tokens: 8192\n" + ) +} + +#[tokio::test] +async fn capability_vectors_are_refused_by_both_routes_before_dispatch() { + let vectors = vectors("capability_vectors.json"); + for case in vectors["refusals"].as_array().unwrap() { + let name = case["name"].as_str().unwrap(); + let model = case["request"]["model"].as_str().unwrap(); + let yaml = capability_model(model, &case["capabilities"]); + let gateway = chat_gateway(&[yaml.as_str()], &[model]).await; + + let translation = messages_to_chat(&case["request"], None).unwrap(); + let chat_request = Request::builder() + .method(Method::POST) + .uri("/v1/chat/completions") + .header("content-type", "application/json") + .body(Body::from(translation.chat.to_string())) + .unwrap(); + let chat = proxy_chat(State(Arc::clone(&gateway.state)), chat_request).await; + assert_eq!( + chat.status().as_u16() as u64, + case["chat"]["status"].as_u64().unwrap(), + "{name}" + ); + assert_eq!(json_body(chat).await, case["chat"]["body"], "{name}"); + + let response = proxy_messages( + State(Arc::clone(&gateway.state)), + messages_request(case["request"].clone()), + ) + .await; + let expect = &case["expect"]; + assert_eq!( + response.status().as_u16() as u64, + expect["status"].as_u64().unwrap(), + "{name}" + ); + for (header, value) in expect["headers"].as_object().unwrap() { + assert_eq!( + response + .headers() + .get(header.as_str()) + .map(|v| v.to_str().unwrap()), + value.as_str(), + "{name}: {header}" + ); + } + let body = json_body(response).await; + assert_eq!(body["type"], "error", "{name}"); + assert_eq!(body["error"]["type"], expect["type"], "{name}"); + assert_eq!(body["error"]["message"], expect["message"], "{name}"); + assert!(gateway.dispatcher.dispatched().is_empty(), "{name}"); + } +} + #[tokio::test] async fn a_translation_refusal_publishes_nothing() { let gateway = chat_gateway(&[HYBRID_GENERATE_MODEL], &["acme/chat"]).await; diff --git a/packages/sie_server/models/sie-fake.yaml b/packages/sie_server/models/sie-fake.yaml index 71b72d6ed..fd0d4a94e 100644 --- a/packages/sie_server/models/sie-fake.yaml +++ b/packages/sie_server/models/sie-fake.yaml @@ -41,6 +41,7 @@ tasks: max_output_tokens: 1024 capabilities: streaming: true + tools: true max_sequence_length: 512 profiles: default: diff --git a/packages/sie_server/src/sie_server/adapters/fake/adapter.py b/packages/sie_server/src/sie_server/adapters/fake/adapter.py index 4fcfb3e32..a0dbecb8a 100644 --- a/packages/sie_server/src/sie_server/adapters/fake/adapter.py +++ b/packages/sie_server/src/sie_server/adapters/fake/adapter.py @@ -698,7 +698,7 @@ async def generate( completion_tokens=count, ) - async def _chat_turn(self, body: dict[str, Any]) -> _ChatTurn: + async def _chat_turn(self, body: dict[str, Any], requested_model: str) -> _ChatTurn: if not self._loaded: raise RuntimeError(ERR_NOT_LOADED) await self._fault_runtime.on_dispatch_async() @@ -708,8 +708,10 @@ async def _chat_turn(self, body: dict[str, Any]) -> _ChatTurn: if self._max_seq_length is not None: room = self._max_seq_length - prompt_tokens if room < 1: - msg = f"the prompt is {prompt_tokens} tokens and leaves no room in the {self._max_seq_length}-token window" - raise GenerationInputTooLongError(msg) + raise GenerationInputTooLongError( + f"prompt_tokens ({prompt_tokens}) exceeds context_length ({self._max_seq_length}) " + f"for model '{requested_model}'" + ) budget = min(budget, room) messages = body["messages"] tools = [tool["function"] for tool in body.get("tools") or []] @@ -767,7 +769,7 @@ async def chat_completion( keep_reasoning: bool = False, ) -> dict[str, Any]: _ = max_response_bytes - turn = await self._chat_turn(body) + turn = await self._chat_turn(body, requested_model) return { "id": f"chatcmpl-{uuid.uuid4().hex}", "object": "chat.completion", @@ -793,7 +795,7 @@ async def chat_completion_stream( keep_reasoning: bool = False, ) -> AsyncIterator[dict[str, Any]]: _ = max_response_bytes - turn = await self._chat_turn(body) + turn = await self._chat_turn(body, requested_model) envelope = { "id": f"chatcmpl-{uuid.uuid4().hex}", "object": "chat.completion.chunk", diff --git a/packages/sie_server/src/sie_server/adapters/mlx/generation.py b/packages/sie_server/src/sie_server/adapters/mlx/generation.py index 7e452892a..7f984e773 100644 --- a/packages/sie_server/src/sie_server/adapters/mlx/generation.py +++ b/packages/sie_server/src/sie_server/adapters/mlx/generation.py @@ -28,6 +28,7 @@ import asyncio import contextlib +import copy import json import logging import os @@ -35,6 +36,7 @@ import tempfile import time from collections.abc import AsyncIterator, Mapping +from functools import lru_cache from pathlib import Path from typing import Any, cast @@ -44,12 +46,14 @@ FinishReason, GenerationAdapter, GenerationChunk, + GenerationInputTooLongError, GenerationUnsupportedFieldError, ) from sie_server.adapters._spec import AdapterSpec from sie_server.adapters._types import ERR_NOT_LOADED from sie_server.adapters.mlx import _server from sie_server.core.load_errors import EngineExitedError, EngineStartupError, ModelConfigurationError +from sie_server.core.tokenizer import load_tokenizer from sie_server.types.inputs import ImageInput, VideoInput logger = logging.getLogger(__name__) @@ -96,6 +100,35 @@ def _reject_grammar(grammar: object) -> None: ) +@lru_cache(maxsize=8) +def _chat_tokenizer(repo: str, trust_remote_code: bool) -> Any: + return load_tokenizer(repo, trust_remote_code=trust_remote_code) + + +def _chat_prompt_tokens(tokenizer: Any, body: Mapping[str, Any]) -> int: + """Count the prompt ``mlx_lm.server`` renders for a chat body: text parts joined, tool-call arguments parsed.""" + messages = copy.deepcopy(body["messages"]) + for message in messages: + content = message.get("content") + if isinstance(content, list): + message["content"] = "".join(part["text"] for part in content if part.get("type") == "text") + elif content is None: + message["content"] = "" + for call in message.get("tool_calls") or []: + function = call.get("function") or {} + arguments = function.get("arguments") + if isinstance(arguments, str) and arguments: + function["arguments"] = json.loads(arguments) + prompt = tokenizer.apply_chat_template( + messages, + tools=body.get("tools"), + tokenize=False, + add_generation_prompt=True, + **(body.get("chat_template_kwargs") or {}), + ) + return len(tokenizer.encode(prompt, add_special_tokens=False)) + + def normalize_mlx_seed(seed: int) -> int: """Convert a signed i64 seed to MLX's equivalent uint64 bit pattern.""" if isinstance(seed, bool) or not isinstance(seed, int) or not -(1 << 63) <= seed <= (1 << 63) - 1: @@ -127,7 +160,7 @@ def __init__( model_name_or_path: str, *, mlx_repo: str | None = None, - max_seq_length: int = 32768, + max_seq_length: int | None = 32768, default_sampling: dict[str, Any] | None = None, stop_tokens: list[str] | None = None, served_model_name: str | None = None, @@ -347,6 +380,33 @@ def preflight_generate(self, parameters: Mapping[str, Any], *, stream: bool) -> _ = stream _reject_grammar(parameters.get("grammar")) + async def check_chat_window(self, body: Mapping[str, Any], *, requested_model: str) -> None: + """Refuse a chat body whose prompt and ``max_tokens`` exceed ``max_seq_length``. + + ``mlx_lm.server`` serves any prompt length, so the declared window is + enforced here, on the prompt the child renders with this repo's + tokenizer and chat template. A prompt that cannot be counted is left + to the child. + + Raises: + GenerationInputTooLongError: The prompt and output do not fit. + """ + if self._max_seq_length is None or self._mlx_repo is None: + return + try: + tokenizer = await asyncio.to_thread(_chat_tokenizer, self._mlx_repo, self._trust_remote_code) + prompt_tokens = await asyncio.to_thread(_chat_prompt_tokens, tokenizer, body) + except Exception as exc: # noqa: BLE001 - the child refuses what cannot be rendered + logger.warning("the chat prompt for %s was not counted (%s)", self._mlx_repo, type(exc).__name__) + return + max_new_tokens = body["max_tokens"] + total = prompt_tokens + max_new_tokens + if total > self._max_seq_length: + raise GenerationInputTooLongError( + f"prompt_tokens ({prompt_tokens}) + max_new_tokens ({max_new_tokens}) = {total} " + f"exceeds context_length ({self._max_seq_length}) for model '{requested_model}'" + ) + def _build_sampling_body( self, prompt: str, diff --git a/packages/sie_server/src/sie_server/api/openai_local.py b/packages/sie_server/src/sie_server/api/openai_local.py index b7b2f05f5..686daa065 100644 --- a/packages/sie_server/src/sie_server/api/openai_local.py +++ b/packages/sie_server/src/sie_server/api/openai_local.py @@ -49,6 +49,7 @@ REASONING_CONTROL_TOKENS, GenerationAdapter, GenerationError, + GenerationInputTooLongError, GenerationUnsupportedFieldError, GenerationUpstreamError, ReasoningFormat, @@ -538,6 +539,15 @@ def _validate_chat_messages(messages: list[Any]) -> None: ) +def _has_image_parts(messages: list[Any]) -> bool: + return any( + part["type"] in ("image_url", "input_image") + for message in messages + if isinstance(message.get("content"), list) + for part in message["content"] + ) + + def _validate_cuda_chat_body(body: dict[str, Any]) -> None: unknown_fields = body.keys() - _CUDA_CHAT_FIELDS if unknown_fields: @@ -1674,6 +1684,17 @@ async def _chat_completions( param="stream", code="unsupported_field", ) + if body.get("tools") is not None and not gen_task.capabilities.tools: + raise _bad_request( + f"Model '{model}' does not support tool calling", param="tools", code="unsupported_field" + ) + adapter_chat = _adapter_answers_chat(config) + if _has_image_parts(messages) and not ( + config.inputs.image and str(device).startswith("cuda") and not adapter_chat + ): + raise _bad_request( + f"Model '{model}' does not support image input", param="messages", code="unsupported_field" + ) if videos: if not (config.inputs.video and str(device).startswith("cuda")): raise _bad_request( @@ -1683,7 +1704,6 @@ async def _chat_completions( ) # The child decodes video on its event loop; bound that work here. await asyncio.to_thread(_probe_chat_videos, videos) - adapter_chat = _adapter_answers_chat(config) if str(device).startswith("cuda") or adapter_chat: _validate_cuda_chat_body(body) else: @@ -1738,6 +1758,10 @@ async def _chat_completions( max_tokens=max_tokens, seed=seed, ) + try: + await adapter.check_chat_window(proxied, requested_model=model) + except GenerationInputTooLongError as exc: + raise _generation_http_exception(exc, registry) from exc tool_call_limit = _mlx_tool_call_limit(body, proxied) slot = adapter.generation_slot() elif isinstance(adapter, SGLangGenerationAdapter) and str(device).startswith("cuda"): diff --git a/packages/sie_server/tests/adapters/test_sglang_chat_tools.py b/packages/sie_server/tests/adapters/test_sglang_chat_tools.py index 7e3fbd0ea..1ee4db799 100644 --- a/packages/sie_server/tests/adapters/test_sglang_chat_tools.py +++ b/packages/sie_server/tests/adapters/test_sglang_chat_tools.py @@ -317,7 +317,12 @@ def _child(child_request: httpx.Request) -> httpx.Response: profile = SimpleNamespace(runtime={"default_sampling": {}}, loadtime={}) config = SimpleNamespace( tasks=SimpleNamespace( - generate=SimpleNamespace(max_output_tokens=64, chat_template_kwargs=dict(_TEMPLATE_KWARGS)), score=None + generate=SimpleNamespace( + max_output_tokens=64, + chat_template_kwargs=dict(_TEMPLATE_KWARGS), + capabilities=SimpleNamespace(tools=True), + ), + score=None, ), resolve_profile=MagicMock(return_value=profile), inputs=SimpleNamespace(video=False), diff --git a/packages/sie_server/tests/api/test_anthropic_messages_route.py b/packages/sie_server/tests/api/test_anthropic_messages_route.py index f64b29c79..2aafcee29 100644 --- a/packages/sie_server/tests/api/test_anthropic_messages_route.py +++ b/packages/sie_server/tests/api/test_anthropic_messages_route.py @@ -23,7 +23,7 @@ from fastapi.testclient import TestClient from sie_server.adapters.remote._http import RemoteUpstreamError from sie_server.api import anthropic_messages -from sie_server.api.anthropic_translation import StreamMachine +from sie_server.api.anthropic_translation import StreamMachine, messages_to_chat from sie_server.api.lora import LeasedStreamingResponse from sie_server.app.app_factory import AppFactory from sie_server.app.app_state_config import AppStateConfig @@ -31,6 +31,7 @@ from sie_server.core.loader import load_adapter, load_model_configs MODELS_DIR = Path(__file__).resolve().parents[2] / "models" +VECTORS = Path(__file__).resolve().parents[4] / "conformance" / "anthropic_messages" MODEL = "sie-fake:chat" QUESTION = "What is the weather in Paris?" WEATHER = { @@ -296,7 +297,12 @@ def test_a_translation_refusal_reaches_no_engine(client: TestClient, registry: M "invalid_request_error", "max_tokens: 5000 > 4096, which is the maximum allowed number of output tokens for this model", ), - (_body(" ".join(["w"] * 600), model="sie-fake"), 400, "invalid_request_error", "prompt is too long"), + ( + _body(" ".join(["w"] * 600), model="sie-fake"), + 400, + "invalid_request_error", + "prompt is too long: 601 tokens > 512 maximum", + ), ( _body(messages=[{"role": "user", "content": "hi"}, {"role": "system", "content": "short answers"}]), 400, @@ -313,7 +319,60 @@ def test_refusals_come_back_in_the_anthropic_envelope( for stream in (False, True): refused = _error(_post(client, {**body, "stream": stream}), status, error_type) if message is not None: - assert refused.startswith(message), refused + assert refused == message + + +def _capability_registry(model: str, capabilities: dict[str, bool]) -> MagicMock: + fake = _configs()["sie-fake"] + generate = fake.tasks.generate + assert generate is not None + config = fake.model_copy( + update={ + "sie_id": model, + "inputs": fake.inputs.model_copy(update={"image": capabilities["image"]}), + "tasks": fake.tasks.model_copy( + update={ + "generate": generate.model_copy( + update={ + "capabilities": generate.capabilities.model_copy(update={"tools": capabilities["tools"]}) + } + ) + } + ), + } + ) + registry = MagicMock() + registry.device = "cpu" + registry.has_model.side_effect = {model: config}.__contains__ + registry.get_config.side_effect = {model: config}.__getitem__ + registry.is_failed.return_value = False + registry.is_unloading.return_value = False + registry.is_loading.return_value = False + registry.is_loaded.return_value = True + return registry + + +@pytest.mark.parametrize( + "case", + json.loads((VECTORS / "capability_vectors.json").read_text(encoding="utf-8"))["refusals"], + ids=lambda case: case["name"], +) +def test_a_capability_vector_is_refused_by_both_routes_before_the_engine( + case: dict[str, Any], monkeypatch: pytest.MonkeyPatch +) -> None: + registry = _capability_registry(case["request"]["model"], case["capabilities"]) + client = _client(registry, monkeypatch) + + chat = client.post("/v1/chat/completions", json=messages_to_chat(case["request"], None).chat) + response = _post(client, case["request"]) + + assert chat.status_code == case["chat"]["status"] + assert chat.json() == case["chat"]["body"] + expect = case["expect"] + assert _error(response, expect["status"], expect["type"]) == expect["message"] + for header, value in expect["headers"].items(): + assert response.headers[header] == value + registry.get.assert_not_called() def test_a_loading_model_is_overloaded_with_its_retry_after(monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/packages/sie_server/tests/api/test_fake_chat_route.py b/packages/sie_server/tests/api/test_fake_chat_route.py index fe2db8994..218c47fdf 100644 --- a/packages/sie_server/tests/api/test_fake_chat_route.py +++ b/packages/sie_server/tests/api/test_fake_chat_route.py @@ -196,7 +196,33 @@ def test_the_window_bounds_the_prompt_and_the_answer() -> None: assert near.json()["usage"] == {"prompt_tokens": window - 4, "completion_tokens": 4, "total_tokens": window} assert near.json()["choices"][0]["finish_reason"] == "length" assert full.status_code == 400 - assert full.json()["error"]["code"] == "INPUT_TOO_LONG" + assert full.json()["error"] == { + "message": f"prompt_tokens ({window + 1}) exceeds context_length ({window}) for model 'sie-fake'", + "type": "invalid_request_error", + "param": "prompt", + "code": "INPUT_TOO_LONG", + } + + +def test_an_image_is_refused_as_unsupported_even_when_the_model_declares_image_input() -> None: + client = _client() + config = _config("sie-fake:chat") + registry = client.app.state.registry + registry.device = "cuda:0" + registry.get_config.return_value = config.model_copy( + update={"inputs": config.inputs.model_copy(update={"image": True})} + ) + pixel = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==" + response = _chat(client, [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": pixel}}]}]) + + assert response.status_code == 400, response.text + assert response.json()["error"] == { + "message": "Model 'sie-fake:chat' does not support image input", + "type": "invalid_request_error", + "param": "messages", + "code": "unsupported_field", + } + registry.get.assert_not_called() def test_a_profile_without_thinking_never_returns_reasoning() -> None: diff --git a/packages/sie_server/tests/api/test_openai_local.py b/packages/sie_server/tests/api/test_openai_local.py index bc9b5ed1d..4d325eee5 100644 --- a/packages/sie_server/tests/api/test_openai_local.py +++ b/packages/sie_server/tests/api/test_openai_local.py @@ -26,9 +26,10 @@ from fastapi import FastAPI from fastapi.testclient import TestClient from openai.types.chat import ChatCompletion, ChatCompletionChunk +from sie_server.adapters.mlx import generation as mlx_generation from sie_server.adapters.mlx.generation import MLXGenerationAdapter from sie_server.adapters.sglang.generation import SGLangGenerationAdapter -from sie_server.api import openai_local +from sie_server.api import anthropic_messages, openai_local from sie_server.api.openai_local import _validate_mlx_seed, router from sie_server.config.model import ModelConfig from sie_server.core import video_frames @@ -37,11 +38,33 @@ from sie_server.core.timing import RequestTiming from sie_server.core.worker import WorkerResult from sie_server.types.inputs import MAX_ITEM_TEXT_BYTES +from tokenizers import Tokenizer, models, pre_tokenizers +from transformers import PreTrainedTokenizerFast _GEMMA_OPEN = "<" + "|channel" + ">" + "thought\n" _GEMMA_CLOSE = "<" + "channel|" + ">" +_UNKNOWN = "[UNK]" +_WORD_TEMPLATE = ( + "{% if tools %}{{ 'tools' }}{% for tool in tools %}{{ ' ' ~ tool.function.name }}{% endfor %}{{ ' ' }}{% endif %}" + "{% for message in messages %}{{ message.role ~ ' ' ~ message.content }}" + "{% for call in message.tool_calls or [] %}{{ ' ' ~ call.function.name }}" + "{% for name, value in call.function.arguments.items() %}{{ ' ' ~ name ~ ' ' ~ value }}{% endfor %}" + "{% endfor %}{{ ' ' }}{% endfor %}" + "{% if enable_thinking %}{{ 'think ' }}{% endif %}{% if add_generation_prompt %}{{ 'assistant' }}{% endif %}" +) + + +def _word_tokenizer() -> PreTrainedTokenizerFast: + """One token per word of a prompt naming each tool, each message's role, text and call arguments, and thinking.""" + backend = Tokenizer(models.WordLevel({_UNKNOWN: 0}, unk_token=_UNKNOWN)) + backend.pre_tokenizer = pre_tokenizers.WhitespaceSplit() + tokenizer = PreTrainedTokenizerFast(tokenizer_object=backend, unk_token=_UNKNOWN) + tokenizer.chat_template = _WORD_TEMPLATE + return tokenizer + + def _client() -> TestClient: app = FastAPI() app.include_router(router) @@ -73,6 +96,8 @@ def _cuda_chat_client( max_output_tokens: int = 64, server_url: str = "http://127.0.0.1:30400", video: bool = False, + image: bool = False, + tools: bool = True, ) -> tuple[TestClient, MagicMock]: adapter = SGLangGenerationAdapter( model_name_or_path="upstream/repo", @@ -83,6 +108,7 @@ def _cuda_chat_client( generate_task = SimpleNamespace( max_output_tokens=max_output_tokens, chat_template_kwargs=chat_template_kwargs or {}, + capabilities=SimpleNamespace(tools=tools), ) profile = SimpleNamespace( runtime={ @@ -94,7 +120,7 @@ def _cuda_chat_client( config = SimpleNamespace( tasks=SimpleNamespace(generate=generate_task, score=None), resolve_profile=MagicMock(return_value=profile), - inputs=SimpleNamespace(video=video), + inputs=SimpleNamespace(video=video, image=image), profiles={}, routing=None, ) @@ -124,15 +150,21 @@ def _mlx_chat_client( chat_template_kwargs: dict[str, Any], default_sampling: dict[str, Any] | None = None, max_output_tokens: int = 64, + max_seq_length: int | None = 32768, + image: bool = False, + tools: bool = True, ) -> tuple[TestClient, MagicMock]: adapter = MLXGenerationAdapter( model_name_or_path="Qwen/Qwen3.5-4B", mlx_repo="mlx-community/Qwen3.5-4B-4bit", + max_seq_length=max_seq_length, ) adapter._server_url = "http://127.0.0.1:30401" + monkeypatch.setattr(mlx_generation, "_chat_tokenizer", lambda _repo, _trust_remote_code: _word_tokenizer()) generate_task = SimpleNamespace( max_output_tokens=max_output_tokens, chat_template_kwargs=chat_template_kwargs, + capabilities=SimpleNamespace(tools=tools), ) profile = SimpleNamespace( runtime={ @@ -144,6 +176,7 @@ def _mlx_chat_client( config = SimpleNamespace( tasks=SimpleNamespace(generate=generate_task, score=None), resolve_profile=MagicMock(return_value=profile), + inputs=SimpleNamespace(video=False, image=image), profiles={}, routing=None, ) @@ -2867,3 +2900,206 @@ def test_cuda_chat_blocking_ends_at_the_generation_budget(monkeypatch: pytest.Mo assert response.status_code == 504 assert "generation_timeout" in response.text + + +_PIXEL = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==" +_IMAGE_TURN = [ + {"role": "user", "content": [{"type": "text", "text": "Describe it."}, {"type": "image_url", "image_url": _PIXEL}]} +] + + +@pytest.mark.parametrize("backend", ["cuda", "mlx"]) +def test_chat_refuses_tools_on_a_model_without_tool_calling_before_the_child( + monkeypatch: pytest.MonkeyPatch, backend: str +) -> None: + def _handler(_request: httpx.Request) -> httpx.Response: + pytest.fail("a refused request must not reach the child") + + if backend == "cuda": + client, registry = _cuda_chat_client(monkeypatch, _handler, tools=False) + else: + client, registry = _mlx_chat_client(monkeypatch, _handler, chat_template_kwargs={}, tools=False) + response = client.post( + "/v1/chat/completions", + json={"model": "Qwen/Qwen3.5-4B", "messages": [{"role": "user", "content": "hi"}], "tools": [_WEATHER_TOOL]}, + ) + + assert response.status_code == 400, response.text + assert response.json()["error"] == { + "message": "Model 'Qwen/Qwen3.5-4B' does not support tool calling", + "type": "invalid_request_error", + "param": "tools", + "code": "unsupported_field", + } + registry.get.assert_not_called() + + +@pytest.mark.parametrize( + ("backend", "image"), + [("cuda", False), ("mlx", False), ("mlx", True)], + ids=["cuda text-only model", "mlx text-only model", "mlx child without image input"], +) +def test_chat_refuses_an_image_the_backend_cannot_read_before_the_child( + monkeypatch: pytest.MonkeyPatch, backend: str, image: bool +) -> None: + def _handler(_request: httpx.Request) -> httpx.Response: + pytest.fail("a refused request must not reach the child") + + if backend == "cuda": + client, registry = _cuda_chat_client(monkeypatch, _handler, image=image) + else: + client, registry = _mlx_chat_client(monkeypatch, _handler, chat_template_kwargs={}, image=image) + response = client.post("/v1/chat/completions", json={"model": "Qwen/Qwen3.5-4B", "messages": _IMAGE_TURN}) + + assert response.status_code == 400, response.text + assert response.json()["error"] == { + "message": "Model 'Qwen/Qwen3.5-4B' does not support image input", + "type": "invalid_request_error", + "param": "messages", + "code": "unsupported_field", + } + registry.get.assert_not_called() + + +def test_an_input_image_part_is_an_image(monkeypatch: pytest.MonkeyPatch) -> None: + client, registry = _cuda_chat_client( + monkeypatch, lambda _request: pytest.fail("a refused request must not reach the child") + ) + response = client.post( + "/v1/chat/completions", + json={ + "model": "Qwen/Qwen3.5-4B", + "messages": [{"role": "user", "content": [{"type": "input_image", "image_url": _PIXEL}]}], + }, + ) + + assert response.status_code == 400, response.text + assert response.json()["error"]["message"] == "Model 'Qwen/Qwen3.5-4B' does not support image input" + registry.get.assert_not_called() + + +def test_cuda_chat_sends_an_image_to_a_vision_model(monkeypatch: pytest.MonkeyPatch) -> None: + seen: list[dict[str, Any]] = [] + + def _handler(request: httpx.Request) -> httpx.Response: + seen.append(json.loads(request.content)) + return httpx.Response(200, json=_chat_completion("a pixel")) + + client, _ = _cuda_chat_client(monkeypatch, _handler, image=True) + response = client.post("/v1/chat/completions", json={"model": "Qwen/Qwen3.5-4B", "messages": _IMAGE_TURN}) + + assert response.status_code == 200, response.text + assert seen[0]["messages"][0]["content"][1] == {"type": "image_url", "image_url": _PIXEL} + + +_WINDOW = 64 + + +def _tool_turns(words: int) -> list[dict[str, Any]]: + return [ + {"role": "user", "content": " ".join(["w"] * words)}, + {"role": "assistant", "content": None, "tool_calls": [_PARIS_CALL]}, + {"role": "tool", "tool_call_id": "call_paris", "content": "18"}, + ] + + +@pytest.mark.parametrize( + ("fields", "template_kwargs", "prompt_tokens"), + [ + ({"messages": [{"role": "user", "content": "w " * 30}]}, {}, 32), + ( + { + "messages": [ + {"role": "user", "content": [{"type": "text", "text": "w w"}, {"type": "text", "text": " w"}]} + ] + }, + {}, + 5, + ), + ({"messages": [{"role": "user", "content": "w " * 30}], "tools": [_WEATHER_TOOL]}, {}, 34), + ({"messages": [{"role": "user", "content": "w " * 30}]}, {"enable_thinking": True}, 33), + ({"messages": _tool_turns(10), "tools": [_WEATHER_TOOL]}, {}, 20), + ], + ids=["text", "text parts", "tools", "template kwargs", "tool call and result"], +) +def test_mlx_chat_enforces_the_declared_window_on_the_rendered_prompt( + monkeypatch: pytest.MonkeyPatch, fields: dict[str, Any], template_kwargs: dict[str, Any], prompt_tokens: int +) -> None: + seen: list[dict[str, Any]] = [] + + def _handler(request: httpx.Request) -> httpx.Response: + seen.append(json.loads(request.content)) + return httpx.Response(200, json=_chat_completion("ok")) + + client, _ = _mlx_chat_client(monkeypatch, _handler, chat_template_kwargs=template_kwargs, max_seq_length=_WINDOW) + fits = client.post( + "/v1/chat/completions", json={"model": "Qwen/Qwen3.5-4B", "max_tokens": _WINDOW - prompt_tokens, **fields} + ) + over = client.post( + "/v1/chat/completions", json={"model": "Qwen/Qwen3.5-4B", "max_tokens": _WINDOW + 1 - prompt_tokens, **fields} + ) + + assert fits.status_code == 200, fits.text + assert len(seen) == 1 + assert over.status_code == 400, over.text + assert over.json()["error"] == { + "message": ( + f"prompt_tokens ({prompt_tokens}) + max_new_tokens ({_WINDOW + 1 - prompt_tokens}) = {_WINDOW + 1} " + f"exceeds context_length ({_WINDOW}) for model 'Qwen/Qwen3.5-4B'" + ), + "type": "invalid_request_error", + "param": "prompt", + "code": "INPUT_TOO_LONG", + } + + +_UNPARSED_CALL = {**_PARIS_CALL, "function": {"name": "get_weather", "arguments": "{"}} + + +@pytest.mark.parametrize( + ("max_seq_length", "messages"), + [ + (None, [{"role": "user", "content": "w " * 100}]), + (_WINDOW, [*_tool_turns(100)[:1], {"role": "assistant", "content": None, "tool_calls": [_UNPARSED_CALL]}]), + ], + ids=["no declared window", "arguments the template cannot read"], +) +def test_mlx_chat_leaves_a_prompt_it_cannot_count_to_the_child( + monkeypatch: pytest.MonkeyPatch, max_seq_length: int | None, messages: list[dict[str, Any]] +) -> None: + seen: list[dict[str, Any]] = [] + + def _handler(request: httpx.Request) -> httpx.Response: + seen.append(json.loads(request.content)) + return httpx.Response(200, json=_chat_completion("ok")) + + client, _ = _mlx_chat_client(monkeypatch, _handler, chat_template_kwargs={}, max_seq_length=max_seq_length) + response = client.post( + "/v1/chat/completions", + json={"model": "Qwen/Qwen3.5-4B", "max_tokens": 8, "messages": messages, "tools": [_WEATHER_TOOL]}, + ) + + assert response.status_code == 200, response.text + assert len(seen) == 1 + + +def test_mlx_messages_refuse_a_prompt_over_the_window_with_the_compaction_wording( + monkeypatch: pytest.MonkeyPatch, +) -> None: + client, _ = _mlx_chat_client( + monkeypatch, + lambda _request: pytest.fail("a refused request must not reach the child"), + chat_template_kwargs={}, + max_seq_length=_WINDOW, + ) + client.app.include_router(anthropic_messages.router) + response = client.post( + "/v1/messages", + json={"model": "Qwen/Qwen3.5-4B", "max_tokens": 32, "messages": [{"role": "user", "content": "w " * 40}]}, + ) + + assert response.status_code == 400, response.text + assert response.json()["error"] == { + "type": "invalid_request_error", + "message": f"prompt is too long: {42 + 32} tokens > {_WINDOW} maximum", + } diff --git a/packages/sie_server/tests/api/test_remote_chat.py b/packages/sie_server/tests/api/test_remote_chat.py index b58bd0804..7c93f0b30 100644 --- a/packages/sie_server/tests/api/test_remote_chat.py +++ b/packages/sie_server/tests/api/test_remote_chat.py @@ -95,7 +95,7 @@ def remote_chat( { "sie_id": MODEL, "remote_backed": True, - "tasks": {"generate": {"context_length": 4096, "max_output_tokens": 64}}, + "tasks": {"generate": {"context_length": 4096, "max_output_tokens": 64, "capabilities": {"tools": True}}}, "profiles": { "default": { "adapter_path": f"{adapter_class.__module__}:{adapter_class.__name__}", @@ -844,13 +844,14 @@ def test_a_native_remote_failure_before_output_logs_one_warning_with_its_reason( ] -def _cold_generation_bridge(remote_chat: tuple) -> MagicMock: +def _cold_generation_bridge(remote_chat: tuple, *, image: bool = False) -> MagicMock: client, _, remote_config, _ = remote_chat remote_config.profiles["default"].adapter_options.runtime.clear() data = remote_config.model_dump(mode="json") data.update( remote_backed=False, hf_id="weights/model", routing={"policy": "fallback", "fallback_profile": "remote"} ) + data["inputs"]["image"] = image data["profiles"] = { "remote": data["profiles"]["default"], "default": { @@ -1049,28 +1050,41 @@ def test_malformed_chat_stream_options_never_start_a_bridge(remote_chat: tuple, registry.start_load_async.assert_not_awaited() +_IMAGE_CHAT = { + "model": MODEL, + "messages": [ + {"role": "user", "content": [{"type": "image_url", "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}}]} + ], +} + + def test_chat_bridge_does_not_dispatch_unsupported_media(remote_chat: tuple) -> None: client, _, _, requests = remote_chat - registry = _cold_generation_bridge(remote_chat) + registry = _cold_generation_bridge(remote_chat, image=True) + registry.device = "cuda" answer_with(remote_chat, json_response(200, completion())) - response = client.post( - "/v1/chat/completions", - json={ - "model": MODEL, - "messages": [ - { - "role": "user", - "content": [ - {"type": "image_url", "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}}, - ], - } - ], - }, - ) + response = client.post("/v1/chat/completions", json=_IMAGE_CHAT) assert response.status_code == 503, response.text assert response.json()["error"]["code"] == "MODEL_LOADING" assert not requests - registry.start_load_async.assert_awaited_once_with(MODEL, "cpu") + registry.start_load_async.assert_awaited_once_with(MODEL, "cuda") + + +def test_an_image_on_a_model_without_image_input_starts_no_load_or_bridge(remote_chat: tuple) -> None: + client, _, _, requests = remote_chat + registry = _cold_generation_bridge(remote_chat) + registry.device = "cuda" + answer_with(remote_chat, json_response(200, completion())) + response = client.post("/v1/chat/completions", json=_IMAGE_CHAT) + assert response.status_code == 400, response.text + assert response.json()["error"] == { + "message": f"Model '{MODEL}' does not support image input", + "type": "invalid_request_error", + "param": "messages", + "code": "unsupported_field", + } + assert not requests + registry.start_load_async.assert_not_awaited() @pytest.mark.parametrize( From 72e9bbe3240701c04e5a61e44271540f1d1f3ada Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Mon, 19 Oct 2026 17:50:04 +0200 Subject: [PATCH 2/2] docs(examples): record the window and image refusals on the laptop MLX run The Messages conformance check against Qwen/Qwen3.5-4B:long-context on Apple Silicon now records the 64K-plus prompt as refused with Claude Code's compaction wording, and the limitations name the backends whose window refusal still carries no counts. --- examples/coding-agent-claude-code/README.md | 45 +++++++++++++-------- examples/coding-agent-opencode/README.md | 5 ++- 2 files changed, 32 insertions(+), 18 deletions(-) diff --git a/examples/coding-agent-claude-code/README.md b/examples/coding-agent-claude-code/README.md index ef0fab381..3a4c3a2b7 100644 --- a/examples/coding-agent-claude-code/README.md +++ b/examples/coding-agent-claude-code/README.md @@ -229,42 +229,53 @@ Busy refusals are retried as in the chat check: a `429` or `503`, or an repository started with `mise run serve -- -b sglang --host 127.0.0.1`, Claude Code 2.1.300, 2026-10-19. -The conformance check: +The conformance check, with the server checking the declared window on the +MLX backend: ``` streaming_tool_use pass 1 tool_use: get_weather({"city": "Paris"}); 1 input_json_delta; stop_reason=tool_use parallel_tool_use pass 2 tool_use in one turn: get_weather({"city": "Paris"}), get_weather({"city": "Tokyo"}) tool_result_round_trip pass final answer used the tool result: 'The weather in Paris is currently overcast with a temperature of 18°C.' - prompt_above_64k_tokens pass 74847 prompt tokens accepted and answered; needle found + prompt_above_64k_tokens recorded declared window 32768 is under 65536; refused with Claude Code's compaction wording: prompt is too long: 78943 tokens > 32768 maximum stop_reason_end_turn pass stop_reason=end_turn after 'ready' stop_reason_tool_use pass stop_reason=tool_use on the tool turn stop_reason_max_tokens pass stop_reason=max_tokens at max_tokens=8 (output_tokens=8) usage_on_message_delta pass message_delta carries usage {'input_tokens': 4, 'cache_creation_input_tokens': 0, 'cache_read_input_tokens': 16, 'output_tokens': 2} thinking_blocks recorded no thinking block came back; the model did not reason here or reasons privately tool_choice_any recorded refused: 400 invalid_request_error: tool_choice: 'tool_choice' must be 'auto' or 'none' on this device, which cannot force a tool call - buffered_message pass stop_reason=end_turn, usage {'input_tokens': 20, 'cache_creation_input_tokens': 0, 'cache_read_input_tokens': 0, 'output_tokens': 2} + buffered_message pass stop_reason=end_turn, usage {'input_tokens': 4, 'cache_creation_input_tokens': 0, 'cache_read_input_tokens': 16, 'output_tokens': 2} inert_fields_accepted pass metadata, cache_control and anthropic-beta accepted effort_refusal pass 400 invalid_request_error: output_config.effort: Extra inputs are not permitted system_turn_refusal pass 400 invalid_request_error: messages.1.role: role "system" is not supported in messages; use the top-level system parameter unknown_model pass 404 not_found_error: Model 'sie-conformance/no-such-model' not found - event_grammar pass 9 streams: message_start, one block at a time, message_delta then message_stop - ping_events recorded 9 ping events across the streams + event_grammar pass 8 streams: message_start, one block at a time, message_delta then message_stop + ping_events recorded 0 ping events across the streams count_tokens recorded not served: 404; Claude Code then estimates locally -14 pass, 0 fail, 4 recorded +13 pass, 0 fail, 5 recorded ``` - The profile does not ask the model to reason, so no thinking came back. - The MLX backend cannot force a tool call, so `tool_choice: any` is refused. Claude Code does not send it. -- The pings came during the 74,847-token prompt, which the server accepted - although the profile declares a 32,768-token window. Set - `CLAUDE_CODE_MAX_CONTEXT_TOKENS` to the declared window, as the recipe does, - so Claude Code compacts before that. +- The long prompt is 74,847 tokens, as the MLX backend counts it, and the + request asks for the 4,096-token output cap. The 78,943 tokens do not fit + the profile's 32,768-token window, so the server refuses the request before + the backend runs it, in the wording that starts Claude Code's compaction. + Set `CLAUDE_CODE_MAX_CONTEXT_TOKENS` to the declared window, as the recipe + does, so Claude Code compacts before that. +- No stream went 10 seconds without an event, so no `ping` was sent. - Prompt tokens served from the prefix cache come back as `cache_read_input_tokens`. - -`./run_gate.sh laptop` passed three times out of three on the same server. +- An image in a tool result is refused with its path, for example + `messages.2.content.0.tool_result.content.1.image: Model + 'Qwen/Qwen3.5-4B:long-context' does not support image input`, because the + MLX backend reads text only. + +`./run_gate.sh laptop` passed three times out of three on the same machine, +recorded before the server checked the window on the MLX backend. The first +run reports 11,546 prompt tokens, under the 28,672 the window leaves beside +the 4,096-token output cap, so each of its requests fits the window. The first run, Claude Code's JSON event lines left out: ``` @@ -369,8 +380,8 @@ uv run pytest - **Not a coding benchmark.** The fixture is two failing tests in a twenty-line module. - **The single-node server answers in its chat route's words.** `sie-server` - translates the refusals its chat route makes. Its chat route does not - report the token counts of a prompt that does not fit on every backend, so - such a refusal can read `prompt is too long` without them. On a backend that - refuses media, an image is refused for the whole request rather than the - image block. + translates the refusals its chat route makes. On the MLX backend and the + weightless fake it checks the declared window itself and reports the token + counts. On the SGLang backend the engine checks the window, and its refusal + comes back as `upstream generation request failed`. A remote upstream's + refusal reads `prompt is too long` without counts. diff --git a/examples/coding-agent-opencode/README.md b/examples/coding-agent-opencode/README.md index c5d234cbe..ee87354f4 100644 --- a/examples/coding-agent-opencode/README.md +++ b/examples/coding-agent-opencode/README.md @@ -392,7 +392,10 @@ repository at commit `530dcde8` started with `mise run serve -- -b sglang`, - The server accepted and answered a 74,847-token prompt although this profile declares a 32,768-token window. The declared window is what a cluster advertises in `/v1/models` and what OpenCode's `limit.context` - should match. + should match. The server now checks that window on the MLX backend too: on + 2026-10-19 the same case was refused with `400 INPUT_TOO_LONG: + prompt_tokens (74847) + max_new_tokens (4096) = 78943 exceeds context_length + (32768) for model 'Qwen/Qwen3.5-4B:long-context'`. The same check against the bare `Qwen/Qwen3.5-4B` gave the same table, with `declared window 8192` in the long-prompt line.