diff --git a/packages/sie_server/src/sie_server/adapters/sglang/_compat/sie_chat_tools.py b/packages/sie_server/src/sie_server/adapters/sglang/_compat/sie_chat_tools.py new file mode 100644 index 000000000..aadb5fee1 --- /dev/null +++ b/packages/sie_server/src/sie_server/adapters/sglang/_compat/sie_chat_tools.py @@ -0,0 +1,139 @@ +"""Render a chat request's tools into the prompt as the client sent them. + +SGLang's ``/v1/chat/completions`` parses ``tools`` into its own models and +hands the chat template ``tool.model_dump()`` instead of the request's JSON. +The dump lists each function's keys in model order (``description`` first), +adds ``"strict": false``, and on 0.5.20 adds a tool-level +``"defer_loading": null``. A template that writes tools with ``tojson`` or +iterates their keys puts those bytes in the prompt; one that reads +``description`` renders a missing description as ``None``. The queued worker +renders the tools as the request carries them, and so does a direct +``apply_chat_template`` call with the model's own template. + +This hook gives the template the request's ``tools`` exactly as sent, all of +them, whenever SGLang renders any tools: ``tool_choice: "none"`` still renders +none, and a named ``tool_choice`` renders every tool, as the worker does. The +sent tools are used only when each one validates to the tool SGLang parsed +from the same position, so any other body keeps SGLang's own dump. + +SGLang 0.5.20 also caches the render of a multimodal model's chat request +under a key that sorts every object's keys. Two requests whose tools or +tool-call arguments differ only in key order would then share one prompt, so +the hook turns that cache off. + +Only the SGLang child opts into this hook. It waits for the engine's normal +import of the chat serving module. +""" + +from __future__ import annotations + +import builtins +import contextvars +import sys +from collections.abc import Mapping, Sequence +from functools import wraps +from types import ModuleType +from typing import Any, cast + +_SERVING_CHAT_MODULE = "sglang.srt.entrypoints.openai.serving_chat" +_CLASS_MARKER = "_sie_chat_tools_as_sent" +_IMPORT_MARKER = "_sie_chat_tools_as_sent_deferred" +_SENT_TOOLS: contextvars.ContextVar[list[Any] | None] = contextvars.ContextVar("sie_sent_tools", default=None) + + +def tools_as_sent(sent: Any, parsed: Any, rendered: Any) -> Any: + """Return ``sent`` when it is the request's tools, otherwise ``rendered``. + + ``rendered`` is what SGLang would pass to the chat template and ``parsed`` + is the request's parsed ``tools``. Nothing rendered stays nothing. + """ + if not rendered or not isinstance(sent, list) or not isinstance(parsed, list) or len(sent) != len(parsed): + return rendered + try: + for raw, tool in zip(sent, cast("list[Any]", parsed), strict=True): + if type(tool).model_validate(raw).model_dump() != tool.model_dump(): + return rendered + except Exception: # noqa: BLE001 - an unexpected body keeps SGLang's own tools + return rendered + return sent + + +async def _sent_tools(raw_request: Any) -> list[Any] | None: + if raw_request is None: + return None + try: + body = await raw_request.json() + except Exception: # noqa: BLE001 - an unreadable body keeps SGLang's own tools + return None + tools = body.get("tools") if isinstance(body, dict) else None + return tools if isinstance(tools, list) else None + + +def _patch_serving_chat_module(module: ModuleType) -> None: + serving_class = getattr(module, "OpenAIServingChat", None) + if serving_class is None: + raise RuntimeError(f"{_SERVING_CHAT_MODULE} does not expose OpenAIServingChat") + if getattr(serving_class, _CLASS_MARKER, False): + return + original_handle = serving_class.handle_request + original_render = serving_class._apply_jinja_template + + @wraps(original_handle) + async def handle_request(self: Any, request: Any, raw_request: Any = None, *args: Any, **kwargs: Any) -> Any: + sent = await _sent_tools(raw_request) if getattr(request, "tools", None) else None + token = _SENT_TOOLS.set(sent) + try: + return await original_handle(self, request, raw_request, *args, **kwargs) + finally: + _SENT_TOOLS.reset(token) + + @wraps(original_render) + def apply_jinja_template(self: Any, request: Any, tools: Any, *args: Any, **kwargs: Any) -> Any: + tools = tools_as_sent(_SENT_TOOLS.get(), getattr(request, "tools", None), tools) + return original_render(self, request, tools, *args, **kwargs) + + serving_class.handle_request = handle_request + serving_class._apply_jinja_template = apply_jinja_template + original_encode = getattr(serving_class, "_render_and_encode_chat_template", None) + if original_encode is not None: + + @wraps(original_encode) + def render_and_encode(self: Any, *args: Any, **kwargs: Any) -> Any: + kwargs["use_cache"] = False + return original_encode(self, *args, **kwargs) + + serving_class._render_and_encode_chat_template = render_and_encode + setattr(serving_class, _CLASS_MARKER, True) + + +def install_chat_tools_as_sent() -> None: + """Patch SGLang's chat serving class now, or as soon as it is imported.""" + loaded = sys.modules.get(_SERVING_CHAT_MODULE) + if isinstance(loaded, ModuleType) and hasattr(loaded, "OpenAIServingChat"): + _patch_serving_chat_module(loaded) + return + + current_import = builtins.__import__ + if getattr(current_import, _IMPORT_MARKER, False): + return + + def deferred_import( + name: str, + globals: Mapping[str, object] | None = None, + locals: Mapping[str, object] | None = None, + fromlist: Sequence[str] | None = (), + level: int = 0, + ) -> ModuleType: + module = current_import(name, globals, locals, fromlist, level) + loaded_module = sys.modules.get(_SERVING_CHAT_MODULE) + if not isinstance(loaded_module, ModuleType) or not hasattr(loaded_module, "OpenAIServingChat"): + return module + try: + _patch_serving_chat_module(loaded_module) + finally: + if builtins.__import__ is deferred_import: + setattr(builtins, "__import__", current_import) # noqa: B010 + return module + + setattr(deferred_import, _IMPORT_MARKER, True) + setattr(builtins, "__import__", deferred_import) # noqa: B010 diff --git a/packages/sie_server/src/sie_server/adapters/sglang/_compat/sitecustomize.py b/packages/sie_server/src/sie_server/adapters/sglang/_compat/sitecustomize.py index 40969e4c7..05cb6af9b 100644 --- a/packages/sie_server/src/sie_server/adapters/sglang/_compat/sitecustomize.py +++ b/packages/sie_server/src/sie_server/adapters/sglang/_compat/sitecustomize.py @@ -25,6 +25,9 @@ ``SIE_SGLANG_LORA_COMPAT`` enables the exact-version LoRA GPU slot release backport needed for runtime eviction on the pinned SGLang 0.5.10 engine. + +``SIE_SGLANG_CHAT_TOOLS_AS_SENT`` makes the chat route render a request's +tools as sent (see ``sie_chat_tools``). """ from __future__ import annotations @@ -158,6 +161,9 @@ def deferred_import( if os.environ.get("SIE_SGLANG_LORA_COMPAT") == "1": importlib.import_module("sie_lora_compat").install_lora_compat() +if os.environ.get("SIE_SGLANG_CHAT_TOOLS_AS_SENT") == "1": + importlib.import_module("sie_chat_tools").install_chat_tools_as_sent() + # --- JSON number digit bound --------------------------------------------- # diff --git a/packages/sie_server/src/sie_server/adapters/sglang/generation.py b/packages/sie_server/src/sie_server/adapters/sglang/generation.py index 8a79aa645..9ded555f4 100644 --- a/packages/sie_server/src/sie_server/adapters/sglang/generation.py +++ b/packages/sie_server/src/sie_server/adapters/sglang/generation.py @@ -1023,6 +1023,7 @@ def _start_server(self, *, port: int, device_indices: list[int], server_url: str # auto-imports sitecustomize from PYTHONPATH during child startup. extra_env["PYTHONPATH"] = os.pathsep.join(self._compat_pythonpath_entries()) extra_env["SIE_SGLANG_MM_PROCESS_CONFIG_COMPAT"] = "1" + extra_env["SIE_SGLANG_CHAT_TOOLS_AS_SENT"] = "1" if self._json_number_max_digits is not None: extra_env[_JSON_NUMBER_MAX_DIGITS_ENV] = str(self._json_number_max_digits) logger.warning( diff --git a/packages/sie_server/src/sie_server/api/openai_local.py b/packages/sie_server/src/sie_server/api/openai_local.py index 55983f2f0..b7b2f05f5 100644 --- a/packages/sie_server/src/sie_server/api/openai_local.py +++ b/packages/sie_server/src/sie_server/api/openai_local.py @@ -829,6 +829,14 @@ def _prepare_chat_body( seed: int | None, ) -> dict[str, Any]: proxied = dict(body) + messages = body.get("messages") + if isinstance(messages, list): + proxied["messages"] = [ + {**message, "role": "system"} + if isinstance(message, dict) and message.get("role") == "developer" + else message + for message in messages + ] # SIE owns these edge-only identifiers; do not send unknown metadata into # the engine process. proxied.pop("safety_identifier", None) diff --git a/packages/sie_server/tests/adapters/test_sglang_chat_tools.py b/packages/sie_server/tests/adapters/test_sglang_chat_tools.py new file mode 100644 index 000000000..7e3fbd0ea --- /dev/null +++ b/packages/sie_server/tests/adapters/test_sglang_chat_tools.py @@ -0,0 +1,534 @@ +"""The single-server chat route and the queued worker must render one request to the same prompt.""" + +from __future__ import annotations + +import asyncio +import copy +import importlib.util +import json +import os +import subprocess +import sys +from collections.abc import AsyncIterator +from pathlib import Path +from types import ModuleType, SimpleNamespace +from typing import Any, Literal +from unittest.mock import AsyncMock, MagicMock + +import httpx +import msgpack +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient +from pydantic import BaseModel, ConfigDict, Field, model_serializer +from sie_server.adapters._generation_base import GenerationAdapter, GenerationChunk +from sie_server.adapters._spec import AdapterSpec +from sie_server.adapters.base import ModelCapabilities, ModelDims +from sie_server.adapters.sglang.generation import SGLangGenerationAdapter +from sie_server.api import openai_local +from sie_server.processors import streaming as streaming_mod +from sie_server.processors.streaming import StreamingProcessor +from tokenizers import Tokenizer, models, pre_tokenizers +from transformers import PreTrainedTokenizerFast + +_COMPAT_DIR = Path(__file__).parents[2] / "src/sie_server/adapters/sglang/_compat" +_SERVING_CHAT = "sglang.srt.entrypoints.openai.serving_chat" +_MODEL = "Qwen/Qwen3.8-27B-FP8" +_TEMPLATE_KWARGS = {"enable_thinking": False} +_UNKNOWN = "[UNK]" + +# Each template keeps only the tool and message handling of one family the +# catalog serves with tools: the Qwen3.x templates write every tool with +# ``tojson``, GLM-5.3 writes the function's keys in order without ``strict`` +# and ``defer_loading``, and Gemma 4 reads the function's fields by name. +_QWEN_TEMPLATE = """\ +{%- if tools %}<|im_start|>system +# Tools + +{% for tool in tools %} +{{ tool | tojson }}{% endfor %} +{% if messages[0].role == 'system' %} + +{{ messages[0].content }}{% endif %}<|im_end|> +{% elif messages[0].role == 'system' %}<|im_start|>system +{{ messages[0].content }}<|im_end|> +{% endif %} +{%- for message in messages %} +{%- if message.role == 'user' %}<|im_start|>user +{{ message.content }}<|im_end|> +{% elif message.role == 'assistant' %}<|im_start|>assistant +{{ message.content }}{% for call in message.tool_calls or [] %} +{% for key, value in call.function.arguments | items %} +{{ value }}{% endfor %} +{% endfor %}<|im_end|> +{% elif message.role == 'tool' %}<|im_start|>user +{{ message.content }}<|im_end|> +{% elif message.role != 'system' or not loop.first %}{{ raise_exception('Unexpected message role.') }} +{%- endif %} +{%- endfor %}<|im_start|>assistant +{% if enable_thinking is false %} + + + +{% endif %}""" + +_GLM_TEMPLATE = """\ +{%- if tools %}<|system|> +{% for tool in tools %}{% if 'function' in tool %}{% set tool = tool['function'] %}{% endif %} +{ {%- for key, value in tool.items() if key not in ('defer_loading', 'strict') %}{% if not loop.first %}, {% endif %}\ +"{{ key }}": {{ value | tojson(ensure_ascii=False) }}{% endfor %}}{% endfor %} +{% endif %} +{%- for message in messages %}<|{{ message.role }}|>{{ message.content }}{% endfor %}<|assistant|>""" + +_GEMMA_TEMPLATE = """\ +{%- for tool in tools or [] %}<|tool>declaration:{{ tool['function']['name'] }}\ +{description:{{ tool['function']['description'] }}}{% endfor %} +{%- for message in messages %}{{ message.role }} +{{ message.content }}{% endfor %}model +""" + +_WEATHER = { + "type": "function", + "function": { + "name": "get_weather", + "description": "Current weather for one city.", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string", "description": "City name, for example Paris"}}, + "required": ["city"], + }, + }, +} +_PING = {"type": "function", "function": {"name": "ping", "parameters": {"type": "object", "properties": {}}}} +_STRICT = { + "type": "function", + "function": { + "name": "lookup", + "description": "Look one key up.", + "parameters": {"type": "object", "properties": {"key": {"type": "string"}}, "required": ["key"]}, + "strict": True, + }, +} +_ASK = {"role": "user", "content": "What is the weather in Paris right now? Use the get_weather tool."} +_CALL = { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": '{"city": "Paris"}'}} + ], +} +_RESULT = {"role": "tool", "tool_call_id": "call_1", "content": "18 C and sunny"} + +_REQUESTS: dict[str, dict[str, Any]] = { + "one_tool": {"messages": [_ASK], "tools": [_WEATHER]}, + "tool_exchange": { + "messages": [{"role": "system", "content": "Answer briefly."}, _ASK, _CALL, _RESULT], + "tools": [_WEATHER, _PING], + }, + "strict_tool": {"messages": [_ASK], "tools": [_STRICT, _WEATHER]}, + "named_tool_choice": { + "messages": [_ASK], + "tools": [_PING, _WEATHER], + "tool_choice": {"type": "function", "function": {"name": "get_weather"}}, + }, + "tool_choice_none": {"messages": [_ASK], "tools": [_WEATHER], "tool_choice": "none"}, + "developer_message": {"messages": [{"role": "developer", "content": "Answer briefly."}, _ASK], "tools": [_WEATHER]}, +} + + +def _load_hook() -> ModuleType: + spec = importlib.util.spec_from_file_location("sie_chat_tools_test", _COMPAT_DIR / "sie_chat_tools.py") + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _tokenizer(template: str) -> PreTrainedTokenizerFast: + backend = Tokenizer(models.WordLevel({_UNKNOWN: 0}, unk_token=_UNKNOWN)) + backend.pre_tokenizer = pre_tokenizers.Whitespace() + tokenizer = PreTrainedTokenizerFast(tokenizer_object=backend, unk_token=_UNKNOWN) + tokenizer.chat_template = template + return tokenizer + + +def _sglang_serving_chat(version: str, tokenizer: PreTrainedTokenizerFast) -> ModuleType: + """The request models and chat-template path of the pinned SGLang releases.""" + + class Function(BaseModel): + description: str | None = Field(default=None) + name: str + parameters: object | None = None + strict: bool = False + + class Tool(BaseModel): + type: str = Field(default="function") + function: Function + + class Function0520(Function): + defer_loading: bool | None = None + + @model_serializer(mode="wrap") + def _serialize(self, handler: Any) -> Any: + data = handler(self) + if self.defer_loading is None: + data.pop("defer_loading", None) + return data + + class Tool0520(BaseModel): + type: str = Field(default="function") + function: Function0520 + defer_loading: bool | None = None + + class FunctionResponse(BaseModel): + name: str | None = None + arguments: str | dict[str, Any] | None = None + + class ToolCall(BaseModel): + id: str | None = None + index: int | None = None + type: Literal["function"] = "function" + function: FunctionResponse + + class Message(BaseModel): + role: str + content: str | None = None + tool_call_id: str | None = None + name: str | None = None + reasoning_content: str | None = None + tool_calls: list[ToolCall] | None = None + + class ToolChoiceFunction(BaseModel): + name: str + + class ToolChoice(BaseModel): + type: Literal["function"] = "function" + function: ToolChoiceFunction + + tool_model = Tool0520 if version == "0.5.20" else Tool + + class ChatCompletionRequest(BaseModel): + model_config = ConfigDict(extra="ignore") + + messages: list[Message] + tools: list[tool_model] | None = None # type: ignore[valid-type] + tool_choice: Literal["none", "auto", "required"] | ToolChoice = "auto" + chat_template_kwargs: dict[str, Any] | None = None + + class OpenAIServingChat: + def __init__(self) -> None: + self._chat_template_cache: dict[str, str] = {} + + async def handle_request(self, request: ChatCompletionRequest, raw_request: Any) -> str: + return self._convert_to_internal_request(request, raw_request) + + def _convert_to_internal_request(self, request: ChatCompletionRequest, raw_request: Any) -> str: + return self._process_messages(request, False) + + def _process_messages(self, request: ChatCompletionRequest, is_multimodal: bool) -> str: + tools = None + if request.tools and request.tool_choice != "none": + if not isinstance(request.tool_choice, str): + tools = [ + item.model_dump() + for item in request.tools + if item.function.name == request.tool_choice.function.name + ] + else: + tools = [item.model_dump() for item in request.tools] + return self._apply_jinja_template(request, tools, is_multimodal) + + def _apply_jinja_template(self, request: ChatCompletionRequest, tools: Any, is_multimodal: bool) -> str: + messages = [] + for message in request.messages: + rendered = message.model_dump() + if rendered["content"] is None: + rendered["content"] = "" + for call in rendered["tool_calls"] or []: + if isinstance(call["function"]["arguments"], str): + call["function"]["arguments"] = json.loads(call["function"]["arguments"]) + messages.append(rendered) + template_kwargs = dict(request.chat_template_kwargs or {}) + if version == "0.5.20": + return self._render_and_encode_chat_template( + messages, tools=tools, template_kwargs=template_kwargs, encode_kwargs={}, use_cache=True + ) + return tokenizer.apply_chat_template( + messages, tokenize=False, add_generation_prompt=True, tools=tools, **template_kwargs + ) + + def _render_and_encode_chat_template( + self, + messages: list[dict[str, Any]], + *, + tools: Any, + template_kwargs: dict[str, Any], + encode_kwargs: dict[str, Any], + use_cache: bool, + ) -> str: + key = json.dumps((messages, tools, template_kwargs, encode_kwargs), sort_keys=True) if use_cache else None + if key is not None and key in self._chat_template_cache: + return self._chat_template_cache[key] + rendered = tokenizer.apply_chat_template( + messages, tokenize=False, add_generation_prompt=True, tools=tools, **template_kwargs + ) + if key is not None: + self._chat_template_cache[key] = rendered + return rendered + + module = ModuleType(_SERVING_CHAT) + module.ChatCompletionRequest = ChatCompletionRequest # type: ignore[attr-defined] + module.OpenAIServingChat = OpenAIServingChat # type: ignore[attr-defined] + return module + + +class _RawRequest: + def __init__(self, body: bytes) -> None: + self._body = body + + async def json(self) -> Any: + return json.loads(self._body) + + +def _single_server_prompts( + monkeypatch: pytest.MonkeyPatch, requests: list[dict[str, Any]], serving_chat: ModuleType, *, hook: bool +) -> list[str]: + """Send the requests through the single-server route and render what reaches one child.""" + proxied: list[bytes] = [] + + def _child(child_request: httpx.Request) -> httpx.Response: + proxied.append(child_request.content) + message = {"role": "assistant", "content": "ok"} + return httpx.Response( + 200, + json={ + "id": "chatcmpl-1", + "object": "chat.completion", + "created": 1, + "model": "served", + "choices": [{"index": 0, "message": message, "logprobs": None, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + ) + + adapter = SGLangGenerationAdapter(model_name_or_path=_MODEL, served_model_name="served") + adapter._server_url = "http://127.0.0.1:30400" + profile = SimpleNamespace(runtime={"default_sampling": {}}, loadtime={}) + config = SimpleNamespace( + tasks=SimpleNamespace( + generate=SimpleNamespace(max_output_tokens=64, chat_template_kwargs=dict(_TEMPLATE_KWARGS)), score=None + ), + resolve_profile=MagicMock(return_value=profile), + inputs=SimpleNamespace(video=False), + profiles={}, + routing=None, + ) + registry = MagicMock() + registry.device = "cuda:0" + registry.has_model.return_value = True + registry.get_config.return_value = config + registry.is_failed.return_value = False + registry.is_unloading.return_value = False + registry.is_loading.return_value = False + registry.is_loaded.return_value = True + registry.get.return_value = adapter + monkeypatch.setattr( + openai_local, "_new_proxy_client", lambda: httpx.AsyncClient(transport=httpx.MockTransport(_child)) + ) + app = FastAPI() + app.include_router(openai_local.router) + app.state.registry = registry + client = TestClient(app) + for request in requests: + response = client.post("/v1/chat/completions", json={"model": _MODEL, **request}) + assert response.status_code == 200, response.text + + if hook: + monkeypatch.setitem(sys.modules, _SERVING_CHAT, serving_chat) + _load_hook().install_chat_tools_as_sent() + serving = serving_chat.OpenAIServingChat() + return [ + asyncio.run( + serving.handle_request(serving_chat.ChatCompletionRequest.model_validate_json(body), _RawRequest(body)) + ) + for body in proxied + ] + + +class _CapturingAdapter(GenerationAdapter): + spec = AdapterSpec(inputs=("text",), outputs=("tokens",), unload_fields=()) + + def __init__(self) -> None: + self._device = None + self.prompts: list[str] = [] + + def load(self, device: str) -> None: # pragma: no cover - registry-mocked + self._device = device + + @property + def capabilities(self) -> ModelCapabilities: + return ModelCapabilities(inputs=["text"], outputs=["tokens"]) + + @property + def dims(self) -> ModelDims: + return ModelDims() + + async def generate(self, prompt: str, **kwargs: Any) -> AsyncIterator[GenerationChunk]: + self.prompts.append(prompt) + yield GenerationChunk(text_delta="", done=True, finish_reason="stop") + + +def _queued_prompt(monkeypatch: pytest.MonkeyPatch, request: dict[str, Any], tokenizer: Any) -> str: + """Publish the request as the gateway does and return the prompt the worker renders.""" + messages = [] + for message in request["messages"]: + forwarded = {"role": "system" if message["role"] == "developer" else message["role"]} + forwarded["content"] = message.get("content") or "" + for key in ("tool_calls", "tool_call_id"): + if message.get(key) is not None: + forwarded[key] = copy.deepcopy(message[key]) + messages.append(forwarded) + generate: dict[str, Any] = {"messages": messages, "max_new_tokens": 64, "tools": copy.deepcopy(request["tools"])} + if "tool_choice" in request: + generate["tool_choice"] = request["tool_choice"] + work_item = { + "work_item_id": "req-1.0", + "request_id": "req-1", + "item_index": 0, + "total_items": 1, + "operation": "generate", + "model_id": _MODEL, + "profile_id": "default", + "pool_name": "default", + "machine_profile": "default", + "router_id": "router-1", + "reply_subject": "_INBOX.router-1.req-1", + "timestamp": 0.0, + "generate": generate, + } + msg = AsyncMock() + msg.data = msgpack.packb(work_item, use_bin_type=True) + + adapter = _CapturingAdapter() + registry = MagicMock() + registry.is_loaded.return_value = True + registry.get.return_value = adapter + registry.device = "cuda:0" + config = MagicMock() + config.hf_id = _MODEL + config.name = _MODEL + config.hf_revision = None + config.weights_path = None + config.tasks.generate.chat_template_kwargs = dict(_TEMPLATE_KWARGS) + config.tasks.generate.context_length = 32768 + config.tasks.generate.max_output_tokens = 512 + profile = MagicMock() + profile.loadtime = {"tool_call_parser": "qwen3_coder"} + config.resolve_profile.return_value = profile + registry.get_config.return_value = config + monkeypatch.setattr(streaming_mod, "load_tokenizer", lambda *_args, **_kwargs: tokenizer) + + processor = StreamingProcessor(nc=AsyncMock(), registry=registry, worker_id="w1") + asyncio.run(processor.process(msg, _MODEL)) + assert len(adapter.prompts) == 1, msgpack.unpackb(processor._nc.publish.await_args.args[1], raw=False) + return adapter.prompts[0] + + +@pytest.mark.parametrize("version", ["0.5.10.post1", "0.5.20"]) +@pytest.mark.parametrize("template", [_QWEN_TEMPLATE, _GLM_TEMPLATE, _GEMMA_TEMPLATE], ids=["qwen", "glm", "gemma"]) +@pytest.mark.parametrize("name", sorted(_REQUESTS)) +def test_single_server_and_queued_routes_render_the_same_prompt( + monkeypatch: pytest.MonkeyPatch, name: str, template: str, version: str +) -> None: + request = _REQUESTS[name] + tokenizer = _tokenizer(template) + + queued = _queued_prompt(monkeypatch, request, tokenizer) + (single,) = _single_server_prompts(monkeypatch, [request], _sglang_serving_chat(version, tokenizer), hook=True) + + assert single == queued + + +def test_sglang_own_tool_dump_changes_the_prompt(monkeypatch: pytest.MonkeyPatch) -> None: + tokenizer = _tokenizer(_QWEN_TEMPLATE) + request = _REQUESTS["one_tool"] + + (single,) = _single_server_prompts(monkeypatch, [request], _sglang_serving_chat("0.5.20", tokenizer), hook=False) + + assert single != _queued_prompt(monkeypatch, request, tokenizer) + assert '"strict": false}, "defer_loading": null}' in single + + +@pytest.mark.parametrize( + ("template", "reordered"), + [(_QWEN_TEMPLATE, "tools"), (_GLM_TEMPLATE, "tools"), (_QWEN_TEMPLATE, "arguments")], + ids=["qwen-tools", "glm-tools", "qwen-arguments"], +) +def test_requests_differing_only_in_key_order_keep_their_own_prompts( + monkeypatch: pytest.MonkeyPatch, template: str, reordered: str +) -> None: + tokenizer = _tokenizer(template) + first = copy.deepcopy(_REQUESTS["tool_exchange"]) + first["messages"][2]["tool_calls"][0]["function"]["arguments"] = '{"city": "Paris", "unit": "C"}' + second = copy.deepcopy(first) + if reordered == "tools": + second["tools"] = json.loads(json.dumps(first["tools"], sort_keys=True)) + else: + second["messages"][2]["tool_calls"][0]["function"]["arguments"] = '{"unit": "C", "city": "Paris"}' + + prompts = _single_server_prompts(monkeypatch, [first, second], _sglang_serving_chat("0.5.20", tokenizer), hook=True) + + assert prompts == [_queued_prompt(monkeypatch, first, tokenizer), _queued_prompt(monkeypatch, second, tokenizer)] + assert prompts[0] != prompts[1] + + +def test_tools_as_sent_keeps_sglang_tools_for_another_body() -> None: + hook = _load_hook() + serving_chat = _sglang_serving_chat("0.5.20", _tokenizer(_QWEN_TEMPLATE)) + parsed = serving_chat.ChatCompletionRequest.model_validate({"messages": [_ASK], "tools": [_WEATHER]}).tools + rendered = [tool.model_dump() for tool in parsed] + + assert hook.tools_as_sent([_WEATHER], parsed, rendered) == [_WEATHER] + assert hook.tools_as_sent([_WEATHER], parsed, None) is None + assert hook.tools_as_sent(None, parsed, rendered) is rendered + assert hook.tools_as_sent([_PING], parsed, rendered) is rendered + assert hook.tools_as_sent([_WEATHER, _PING], parsed, rendered) is rendered + anthropic = [{"name": "get_weather", "input_schema": _WEATHER["function"]["parameters"]}] + assert hook.tools_as_sent(anthropic, parsed, rendered) is rendered + + +def test_child_sitecustomize_installs_the_hook(tmp_path: Path) -> None: + package = tmp_path / "sglang/srt/entrypoints/openai" + package.mkdir(parents=True) + for directory in (package, *package.parents): + if directory == tmp_path: + break + (directory / "__init__.py").write_text("") + (package / "serving_chat.py").write_text( + "class OpenAIServingChat:\n" + " async def handle_request(self, request, raw_request):\n" + " return self._apply_jinja_template(request, ['dumped'], False)\n" + " def _apply_jinja_template(self, request, tools, is_multimodal):\n" + " return tools\n" + ) + environment = {key: value for key, value in os.environ.items() if not key.startswith("SIE_SGLANG_")} + environment["PYTHONPATH"] = os.pathsep.join([str(_COMPAT_DIR), str(tmp_path)]) + environment["SIE_SGLANG_CHAT_TOOLS_AS_SENT"] = "1" + result = subprocess.run( # noqa: S603 - executes the fixed local interpreter + [ + sys.executable, + "-c", + "import sys\n" + f"assert {_SERVING_CHAT!r} not in sys.modules\n" + f"from {_SERVING_CHAT} import OpenAIServingChat\n" + "assert OpenAIServingChat._sie_chat_tools_as_sent\n", + ], + env=environment, + capture_output=True, + text=True, + check=False, + timeout=15, + ) + assert result.returncode == 0, result.stderr + assert "Error in sitecustomize" not in result.stderr diff --git a/packages/sie_server/tests/adapters/test_sglang_generation.py b/packages/sie_server/tests/adapters/test_sglang_generation.py index dcb67759c..6f4a25695 100644 --- a/packages/sie_server/tests/adapters/test_sglang_generation.py +++ b/packages/sie_server/tests/adapters/test_sglang_generation.py @@ -205,6 +205,7 @@ def test_load_drops_is_embedding( assert adapter._server_url == "http://localhost:30005" child_env = mock_popen.call_args.kwargs["env"] assert child_env["SIE_SGLANG_MM_PROCESS_CONFIG_COMPAT"] == "1" + assert child_env["SIE_SGLANG_CHAT_TOOLS_AS_SENT"] == "1" expected_compat_dir = Path(__file__).resolve().parents[2] / "src/sie_server/adapters/sglang/_compat" assert child_env["PYTHONPATH"].split(os.pathsep)[0] == str(expected_compat_dir)