diff --git a/docs/guide/infer.md b/docs/guide/infer.md index de77e69b..a2a1767e 100644 --- a/docs/guide/infer.md +++ b/docs/guide/infer.md @@ -57,6 +57,40 @@ used, not whether the service starts. Models with a `path` field (and no `api_base`) or an `infer` section in the YAML config are automatically launched by `sieval run` and stopped after evaluation completes. +### Anthropic Messages + +To call the native Anthropic Messages API, select the dialect explicitly and +provide an external endpoint and credential: + +```yaml +models: + claude: + name: claude-sonnet-4-5 + dialect: anthropic_messages + api_base: https://api.anthropic.com/v1 + api_key: "" + args: + max_tokens: 1024 +``` + +The dialect sends JSON or SSE requests to `POST /v1/messages`. Every request +must specify `max_tokens`; set a model-wide default under `models..args`, +as above, or override it for one task under `tasks..infer_args`. + +Anthropic Messages has no per-request `seed`. Deterministic mode therefore does +not inject one for this dialect, and an explicit `seed` is rejected before any +network I/O. + +Raw request passthrough is intentionally unavailable. `dialect_options` cannot +inject arbitrary Anthropic body fields or HTTP headers; in particular, there is +currently no way to send the `anthropic-beta` header. Features that require a +beta header are unsupported until they receive an explicit, audited mapping. + +The YAML loader does not expand environment-variable expressions in `api_key`. +Replace the placeholder above through your configuration/secret-management +workflow; omit `api_key` only when the target endpoint does not require +authentication. + ## Environment Variables Custom environment variables can be passed through the YAML config's `infer.env` section. Values are injected into the inference engine process. diff --git a/sieval/core/models/dialect.py b/sieval/core/models/dialect.py index 56d05e3a..b849445a 100644 --- a/sieval/core/models/dialect.py +++ b/sieval/core/models/dialect.py @@ -627,11 +627,17 @@ def validate(self, plan: RuntimePlanView, req: Request, response: Response) -> N # must be able to serve. Only the projected request decides which # response channels this particular call must return. required = set(required_response_channels(req)) + finish_reasons = response.finish_reasons + refused = ( + finish_reasons is not None + and bool(finish_reasons) + and all(reason == "refusal" for reason in finish_reasons) + ) for name, rule in self.rules.items(): value = getattr(response, name) if rule.guarantee is Guarantee.NEVER and value is not None: raise OutputContractError(f"dialect promised {name!r} would be absent") - if name in required: + if name in required and not refused: if rule.guarantee is not Guarantee.PRESENT_OR_ERROR: raise OutputContractError( f"{name!r} is required but has {rule.guarantee.value} guarantee" @@ -643,7 +649,7 @@ def validate(self, plan: RuntimePlanView, req: Request, response: Response) -> N if value is not None: rule.validator(value) - if req.session.opaque_continuation is not None: + if req.session.opaque_continuation is not None and not refused: reasoning = response.reasoning if reasoning is None or any( item is None diff --git a/sieval/core/models/dialect_registry.py b/sieval/core/models/dialect_registry.py index 84f5008c..5724014a 100644 --- a/sieval/core/models/dialect_registry.py +++ b/sieval/core/models/dialect_registry.py @@ -30,6 +30,10 @@ from .connection_factory import CONNECTION_FACTORY_REGISTRY from .deployment import ConnectionPool, Deployment, RouteIntent, resolve_route from .dialect import Dialect, active_request_leaves +from .dialects.anthropic_messages import ( + CAPABILITY_DECISIONS as ANTHROPIC_MESSAGES_CAPABILITY_DECISIONS, +) +from .dialects.anthropic_messages import AnthropicMessagesDialect from .dialects.openai_chat import ( CAPABILITY_DECISIONS as OPENAI_CHAT_CAPABILITY_DECISIONS, ) @@ -321,6 +325,10 @@ def _bind_openai_responses(connection: Any, requested_model_id: str) -> Dialect: return OpenAIResponsesDialect(connection, requested_model_id) +def _bind_anthropic_messages(connection: Any, requested_model_id: str) -> Dialect: + return AnthropicMessagesDialect(connection, requested_model_id) + + def _binder( function: Callable[[Any, str], Dialect], decisions: Mapping[str, DialectCapabilityDecision], @@ -355,6 +363,13 @@ def _binder( OPENAI_RESPONSES_CAPABILITY_DECISIONS, ), ), + "anthropic_messages": _binder( + _bind_anthropic_messages, + cast( + Mapping[str, DialectCapabilityDecision], + ANTHROPIC_MESSAGES_CAPABILITY_DECISIONS, + ), + ), } ) @@ -395,7 +410,8 @@ def _registered_decisions( "anthropic_messages": _spec( "anthropic_messages", "async_http_json", - request_seed_support=RequestSeedSupport.RESERVED, + decisions=_registered_decisions("anthropic_messages"), + request_seed_support=RequestSeedSupport.UNSUPPORTED, input_kinds=("chat",), input_modalities=("text", "image", "tool_call", "tool_result"), ), diff --git a/sieval/core/models/dialects/anthropic_messages.py b/sieval/core/models/dialects/anthropic_messages.py new file mode 100644 index 00000000..07763f74 --- /dev/null +++ b/sieval/core/models/dialects/anthropic_messages.py @@ -0,0 +1,2544 @@ +"""Anthropic Messages dialect for the provider-neutral model IR. + +The adapter speaks the native ``POST /v1/messages`` JSON/SSE protocol through +the provider-neutral HTTP connection family. It owns Anthropic authentication, +top-level system extraction, content-block translation, extended-thinking +round trips, and strict terminal-response lifting; it does not own the HTTP +client lifetime. + +AI-Generated Code - GPT-5.6 (OpenAI) +""" + +import json +from collections.abc import AsyncIterator, Mapping, Sequence +from copy import deepcopy +from dataclasses import dataclass +from types import MappingProxyType +from typing import NoReturn, cast + +import httpx + +from sieval.core.models._shared import copy_json_value +from sieval.core.models.capabilities import ( + CAPABILITY_KEYS, + Capability, + CapabilityKey, + DialectCapabilityBinding, + DialectCapabilityDecision, + MultimodalInputOptions, + ReasoningOptions, + StructuredOutputOptions, + Supported, + Unsupported, +) +from sieval.core.models.connection_factory import AsyncHTTPJSONConnection +from sieval.core.models.deployment import BINDING_RESOURCE_KEYS +from sieval.core.models.dialect import ( + DialectError, + Guarantee, + OutputContract, + OutputContractError, + OutputRule, + PreparedRequest, + RequestAudit, + RuntimePlanView, + WireEvidence, + WireObservation, + active_request_leaves, + validate_reasoning, + validate_request_invariants, + validate_runtime_binding_plan, + validate_structured_output, + validate_tool_calls, +) +from sieval.core.models.ir import ( + ChatInput, + FunctionToolCall, + ImagePart, + OpaqueContinuation, + ReasoningOutput, + Request, + Response, + StructuredOutput, + TextPart, + ToolCallPart, + ToolResultPart, + UsageStats, +) +from sieval.core.types import JSONValue + +_ANTHROPIC_VERSION = "2023-06-01" +_MESSAGES_PATH = "messages" +_OPAQUE_CONTINUATION_VERSION = 2 +_IMAGE_MEDIA_TYPES = frozenset({"image/jpeg", "image/png", "image/gif", "image/webp"}) +_REASONING_EFFORTS = frozenset({"low", "medium", "high", "xhigh", "max"}) +_STOP_REASONS = frozenset( + { + "end_turn", + "max_tokens", + "stop_sequence", + "tool_use", + "refusal", + "model_context_window_exceeded", + } +) +_IR_OWNED_BODY_KEYS = BINDING_RESOURCE_KEYS | frozenset( + { + "model", + "messages", + "system", + "max_tokens", + "temperature", + "top_p", + "top_k", + "stop_sequences", + "stream", + "thinking", + "output_config", + "tools", + "tool_choice", + } +) +_MESSAGE_RESPONSE_KEYS = frozenset( + { + "id", + "type", + "role", + "content", + "model", + "stop_reason", + "stop_sequence", + "usage", + "container", + "stop_details", + } +) +_STREAM_EVENT_KEYS: Mapping[str, frozenset[str]] = MappingProxyType( + { + "ping": frozenset({"type"}), + "error": frozenset({"type", "error"}), + "message_start": frozenset({"type", "message"}), + "content_block_start": frozenset({"type", "index", "content_block"}), + "content_block_delta": frozenset({"type", "index", "delta"}), + "content_block_stop": frozenset({"type", "index"}), + "message_delta": frozenset({"type", "delta", "usage"}), + "message_stop": frozenset({"type"}), + } +) +_MESSAGE_DELTA_KEYS = frozenset( + {"stop_reason", "stop_sequence", "container", "stop_details"} +) +#: ``recommended_model``, ``fallback_credit_token`` and +#: ``fallback_has_prefill_claim`` are omitted: each needs an ``anthropic-beta`` +#: header, which this dialect does not send, so all three are unreachable and +#: rejected as unknown. Adding header passthrough means mapping them here. +_REFUSAL_DETAIL_KEYS = frozenset({"type", "category", "explanation"}) + + +def _validate_reasoning_config(options: object) -> None: + assert isinstance(options, ReasoningOptions) + if options.effort is not None and options.effort not in _REASONING_EFFORTS: + raise ValueError( + f"anthropic_messages does not support effort {options.effort!r}" + ) + if options.budget_tokens is not None and options.budget_tokens < 1024: + raise ValueError( + "anthropic_messages reasoning budget_tokens must be at least 1024" + ) + if options.summary not in {None, "none", "auto"}: + raise ValueError( + "anthropic_messages supports only summary='none' or summary='auto'" + ) + + +def _validate_structured_output_config(options: object) -> None: + assert isinstance(options, StructuredOutputOptions) + unsupported = set(options.formats) - {"json_schema"} + if unsupported: + raise ValueError( + "anthropic_messages supports only json_schema structured output" + ) + + +def _validate_multimodal_config(options: object) -> None: + assert isinstance(options, MultimodalInputOptions) + unsupported = set(options.modalities) - {"image"} + if unsupported: + raise ValueError( + f"anthropic_messages does not support modalities {unsupported}" + ) + + +CAPABILITY_DECISIONS: Mapping[CapabilityKey, DialectCapabilityDecision] = ( + MappingProxyType( + { + "input_scoring": Unsupported( + "Anthropic Messages cannot score input tokens" + ), + "sampled_logprobs": Unsupported( + "Anthropic Messages does not return sampled token logprobs" + ), + "top_logprobs": Unsupported( + "Anthropic Messages does not return alternative token logprobs" + ), + "reasoning": Supported( + DialectCapabilityBinding( + "reasoning", + request_leaves=( + "reasoning.effort", + "reasoning.budget_tokens", + "reasoning.summary", + ), + response_channels=("reasoning",), + _config_validator=_validate_reasoning_config, + ) + ), + "function_tools": Supported( + DialectCapabilityBinding( + "function_tools", + request_leaves=( + "tools.functions", + "tools.choice", + "tools.parallel", + ), + response_channels=("tool_calls",), + ) + ), + "hosted_tools": Unsupported( + "PR 3 does not bind Anthropic server-hosted tools" + ), + "structured_output": Supported( + DialectCapabilityBinding( + "structured_output", + request_leaves=( + "structured_output.format", + "structured_output.schema", + "structured_output.name", + "structured_output.strict", + ), + response_channels=("structured_output",), + _config_validator=_validate_structured_output_config, + ) + ), + "stateful_session": Unsupported( + "Anthropic Messages has no previous-response-id continuation" + ), + "opaque_continuation": Supported( + DialectCapabilityBinding( + "opaque_continuation", + request_leaves=("session.opaque_continuation",), + response_channels=("reasoning",), + ) + ), + "multimodal_input": Supported( + DialectCapabilityBinding( + "multimodal_input", + request_leaves=( + "input.modality.image", + "input.modality.image.media_type", + "input.modality.image.detail", + ), + _config_validator=_validate_multimodal_config, + ) + ), + # Anthropic's final assistant message is a native prefill. The + # semantic is carried by ChatInput rather than a separate Request leaf. + "prefill": Supported(DialectCapabilityBinding("prefill")), + "fim": Unsupported("Anthropic Messages has no completion suffix"), + } + ) +) + +assert set(CAPABILITY_DECISIONS) == set(CAPABILITY_KEYS) + + +OUTPUT_CONTRACT = OutputContract( + { + "reasoning": OutputRule(Guarantee.PRESENT_OR_ERROR, validate_reasoning), + "tool_calls": OutputRule(Guarantee.BEST_EFFORT, validate_tool_calls), + "server_tool_uses": OutputRule(Guarantee.NEVER), + "structured_output": OutputRule( + Guarantee.PRESENT_OR_ERROR, validate_structured_output + ), + "logprobs": OutputRule(Guarantee.NEVER), + "top_logprobs": OutputRule(Guarantee.NEVER), + "input_scoring": OutputRule(Guarantee.NEVER), + "citations": OutputRule(Guarantee.NEVER), + "grounding": OutputRule(Guarantee.NEVER), + "session_id": OutputRule(Guarantee.NEVER), + "usage": OutputRule(Guarantee.BEST_EFFORT), + } +) + + +@dataclass(frozen=True) +class _AnthropicContext: + request: Request + + +@dataclass(frozen=True) +class _ExecutionContext: + request: Request + system: tuple[Mapping[str, JSONValue], ...] + messages: tuple[Mapping[str, JSONValue], ...] + request_params: Mapping[str, JSONValue] + + +@dataclass(frozen=True) +class _DecodedContinuation: + system: tuple[dict[str, JSONValue], ...] + messages: tuple[dict[str, JSONValue], ...] + tools: tuple[dict[str, JSONValue], ...] | None + open_turn: bool + thinking: Mapping[str, JSONValue] | None + effort: str | None + + +@dataclass(frozen=True) +class _LegacyPlan: + dialect_id: str = "anthropic_messages" + available_capabilities: frozenset[str] = frozenset( + key + for key, decision in CAPABILITY_DECISIONS.items() + if isinstance(decision, Supported) + ) + capability_minimums: Mapping[str, Mapping[str, JSONValue]] = MappingProxyType({}) + required_output_channels: frozenset[str] = frozenset() + + +def _canonical_json(value: object, path: str) -> str: + return json.dumps( + copy_json_value(value, path), + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ) + + +def _json_mapping(value: object, path: str) -> dict[str, JSONValue]: + copied = copy_json_value(value, path) + if not isinstance(copied, dict): + raise OutputContractError(f"{path} must be an object") + return cast(dict[str, JSONValue], copied) + + +def _reject_unknown_fields( + value: Mapping[str, object], allowed: frozenset[str], path: str +) -> None: + unknown = set(value) - allowed + if unknown: + raise OutputContractError( + f"{path} has unsupported field(s): " + ", ".join(sorted(unknown)) + ) + + +def _required_string(value: object, path: str) -> str: + if not isinstance(value, str) or not value: + raise OutputContractError(f"{path} must be a non-empty string") + return value + + +def _optional_string(value: object, path: str) -> str | None: + if value is None: + return None + return _required_string(value, path) + + +def _validate_stop_details(stop_reason: str, raw: object) -> None: + if stop_reason != "refusal": + if raw is not None: + raise OutputContractError( + "response.stop_details must be null unless stop_reason is 'refusal'" + ) + return + + details = _json_mapping(raw, "response.stop_details") + _reject_unknown_fields(details, _REFUSAL_DETAIL_KEYS, "response.stop_details") + if details.get("type") != "refusal": + raise OutputContractError("response.stop_details.type must be 'refusal'") + for field in ("category", "explanation"): + value = details.get(field) + if value is not None and not isinstance(value, str): + raise OutputContractError( + f"response.stop_details.{field} must be a string or null" + ) + + +def _sequence(value: object, path: str) -> Sequence[object]: + if not isinstance(value, list | tuple): + raise OutputContractError(f"{path} must be a list") + return cast(Sequence[object], value) + + +def _nonnegative_int(value: object, path: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise OutputContractError(f"{path} must be a non-negative integer") + return value + + +def _json_text(value: JSONValue) -> str: + if isinstance(value, str): + return value + return json.dumps( + copy_json_value(value, "tool result"), + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ) + + +def _reject_nonstandard_json_constant(value: str) -> NoReturn: + raise ValueError(f"non-standard JSON constant {value!r}") + + +def _unique_json_object( + pairs: list[tuple[str, JSONValue]], +) -> dict[str, JSONValue]: + result: dict[str, JSONValue] = {} + for key, value in pairs: + if key in result: + raise ValueError(f"duplicate JSON object key {key!r}") + result[key] = value + return result + + +def _strict_json_loads(value: str | bytes) -> JSONValue: + return cast( + JSONValue, + json.loads( + value, + object_pairs_hook=_unique_json_object, + parse_constant=_reject_nonstandard_json_constant, + ), + ) + + +def _image_rejection(part: ImagePart) -> tuple[str, str] | None: + if part.url == "" or part.data == "": + return ( + "input.modality.image", + "Anthropic image URL/data payloads must not be empty", + ) + if part.detail is not None: + return ( + "input.modality.image.detail", + "Anthropic Messages has no provider-neutral image detail control", + ) + if part.url is not None and part.media_type is not None: + return ( + "input.modality.image.media_type", + "Anthropic URL image sources do not carry a media_type", + ) + if part.data is not None and part.media_type is None: + return ( + "input.modality.image", + "Anthropic base64 image sources require media_type", + ) + if part.data is not None and part.media_type not in _IMAGE_MEDIA_TYPES: + return ( + "input.modality.image.media_type", + f"Anthropic base64 images require one of {sorted(_IMAGE_MEDIA_TYPES)}", + ) + return None + + +def _image_to_wire(part: ImagePart) -> dict[str, JSONValue]: + if part.url is not None: + return { + "type": "image", + "source": {"type": "url", "url": part.url}, + } + assert part.data is not None + assert part.media_type is not None + return { + "type": "image", + "source": { + "type": "base64", + "media_type": part.media_type, + "data": part.data, + }, + } + + +def _part_to_wire(part: object) -> dict[str, JSONValue]: + if isinstance(part, TextPart): + if not isinstance(part.text, str): + raise DialectError("Anthropic text content must be a string") + return {"type": "text", "text": part.text} + if isinstance(part, ImagePart): + return _image_to_wire(part) + if isinstance(part, ToolCallPart): + if not isinstance(part.arguments, Mapping): + raise DialectError("Anthropic tool_use input must be a JSON object") + return { + "type": "tool_use", + "id": part.call_id, + "name": part.name, + "input": copy_json_value(part.arguments, "tool call arguments"), + } + if isinstance(part, ToolResultPart): + return { + "type": "tool_result", + "tool_use_id": part.call_id, + "content": _json_text(part.result), + "is_error": part.is_error, + } + raise DialectError(f"unsupported Anthropic message part {type(part).__name__}") + + +def _lower_chat_input( + input_: ChatInput, +) -> tuple[list[dict[str, JSONValue]], list[dict[str, JSONValue]]]: + system: list[dict[str, JSONValue]] = [] + messages: list[dict[str, JSONValue]] = [] + for message in input_.messages: + if message.role == "system": + for part in message.content: + if not isinstance(part, TextPart): + raise DialectError( + "Anthropic top-level system accepts text content only" + ) + system.append(_part_to_wire(part)) + continue + role = "user" if message.role == "tool" else message.role + messages.append( + { + "role": role, + "content": [_part_to_wire(part) for part in message.content], + } + ) + return system, messages + + +def _function_tool_to_wire(raw: Mapping[str, JSONValue]) -> dict[str, JSONValue]: + data = dict(raw) + if data.get("type") != "function": + raise DialectError("function tool declarations require type='function'") + nested = data.get("function") + if nested is not None: + if set(data) != {"type", "function"} or not isinstance(nested, Mapping): + raise DialectError("invalid Chat-shaped function tool declaration") + source = dict(nested) + else: + source = data + source.pop("type", None) + + allowed = {"name", "description", "parameters", "strict"} + unknown = set(source) - allowed + if unknown: + raise DialectError( + "unsupported function tool field(s): " + ", ".join(sorted(unknown)) + ) + name = source.get("name") + if not isinstance(name, str) or not name: + raise DialectError("function tool name must be a non-empty string") + parameters = source.get("parameters") + if not isinstance(parameters, Mapping): + raise DialectError("function tool parameters must be a JSON object") + result: dict[str, JSONValue] = { + "name": name, + "input_schema": copy_json_value(parameters, "function tool parameters"), + } + if "description" in source: + description = source["description"] + if not isinstance(description, str): + raise DialectError("function tool description must be a string") + result["description"] = description + if "strict" in source: + strict = source["strict"] + if not isinstance(strict, bool): + raise DialectError("function tool strict must be a boolean") + result["strict"] = strict + return result + + +def _tool_choice_to_wire(raw: JSONValue) -> dict[str, JSONValue]: + if isinstance(raw, str): + choices = {"auto": "auto", "required": "any", "none": "none"} + try: + return {"type": choices[raw]} + except KeyError as exc: + raise DialectError(f"unsupported Anthropic tool choice {raw!r}") from exc + if not isinstance(raw, Mapping): + raise DialectError("Anthropic tool choice must be a string or mapping") + data = dict(raw) + if data.get("type") != "function": + raise DialectError("only function-specific mapped tool choices are supported") + nested = data.get("function") + if nested is not None: + if set(data) != {"type", "function"} or not isinstance(nested, Mapping): + raise DialectError("invalid Chat-shaped function tool choice") + if set(nested) != {"name"}: + raise DialectError("function tool choice requires only a name") + name = nested.get("name") + else: + if set(data) != {"type", "name"}: + raise DialectError("invalid function tool choice") + name = data.get("name") + if not isinstance(name, str) or not name: + raise DialectError("function tool choice requires a non-empty name") + return {"type": "tool", "name": name} + + +def _tool_choice( + req: Request, +) -> dict[str, JSONValue] | None: + choice = ( + _tool_choice_to_wire(req.tools.choice) if req.tools.choice is not None else None + ) + if req.tools.parallel is not None: + if choice is None: + choice = {"type": "auto"} + choice["disable_parallel_tool_use"] = not req.tools.parallel + return choice + + +def _thinking_config(req: Request) -> dict[str, JSONValue] | None: + # ``display`` lives only on the ``thinking`` object, so "no visible + # summary" cannot be said without also saying "think": ``summary='none'`` + # enables thinking on a model where it defaults off, and those tokens bill + # as output. Leaving ``summary`` unset is what sends no thinking at all. + # (On Responses ``summary='none'`` is a no-op -- the same field, a + # different meaning.) + thinking: dict[str, JSONValue] | None = None + if req.reasoning.budget_tokens is not None: + thinking = { + "type": "enabled", + "budget_tokens": req.reasoning.budget_tokens, + } + if req.reasoning.summary is not None: + if thinking is None: + thinking = {"type": "adaptive"} + thinking["display"] = ( + "summarized" if req.reasoning.summary == "auto" else "omitted" + ) + return thinking + + +def _has_assistant_prefill(input_: ChatInput) -> bool: + return input_.messages[-1].role == "assistant" + + +def _validate_continuation_thinking(value: object) -> Mapping[str, JSONValue] | None: + if value is None: + return None + if not isinstance(value, Mapping): + raise DialectError("Anthropic continuation thinking state must be an object") + thinking = cast(Mapping[str, JSONValue], value) + thinking_type = thinking.get("type") + if thinking_type == "enabled": + allowed = {"type", "budget_tokens", "display"} + if set(thinking) - allowed: + raise DialectError("Anthropic continuation thinking state is malformed") + budget = thinking.get("budget_tokens") + if isinstance(budget, bool) or not isinstance(budget, int) or budget < 1024: + raise DialectError("Anthropic continuation thinking budget is invalid") + elif thinking_type == "adaptive": + if set(thinking) - {"type", "display"}: + raise DialectError("Anthropic continuation thinking state is malformed") + else: + raise DialectError("Anthropic continuation thinking type is invalid") + if thinking.get("display") not in {None, "summarized", "omitted"}: + raise DialectError("Anthropic continuation thinking display is invalid") + return thinking + + +def _decode_continuation( + value: str, +) -> _DecodedContinuation: + try: + raw = _strict_json_loads(value) + except (UnicodeDecodeError, ValueError) as exc: + raise DialectError("Anthropic opaque continuation is not valid JSON") from exc + expected_keys = { + "version", + "system", + "messages", + "tools", + "open_turn", + "reasoning", + } + if not isinstance(raw, dict) or set(raw) != expected_keys: + raise DialectError("Anthropic opaque continuation has an invalid envelope") + if raw["version"] != _OPAQUE_CONTINUATION_VERSION: + raise DialectError("Anthropic opaque continuation version is unsupported") + raw_system = raw["system"] + raw_messages = raw["messages"] + if not isinstance(raw_system, list) or not isinstance(raw_messages, list): + raise DialectError("Anthropic opaque continuation history must be lists") + open_turn = raw["open_turn"] + if not isinstance(open_turn, bool): + raise DialectError("Anthropic continuation open_turn must be a boolean") + raw_reasoning = raw["reasoning"] + if not isinstance(raw_reasoning, dict) or set(raw_reasoning) != { + "thinking", + "effort", + }: + raise DialectError("Anthropic continuation reasoning state is malformed") + effort = raw_reasoning["effort"] + if effort is not None and ( + not isinstance(effort, str) or effort not in _REASONING_EFFORTS + ): + raise DialectError("Anthropic continuation effort is invalid") + thinking = _validate_continuation_thinking(raw_reasoning["thinking"]) + try: + system = [ + _json_mapping(item, f"opaque.system[{index}]") + for index, item in enumerate(raw_system) + ] + messages = [ + _json_mapping(item, f"opaque.messages[{index}]") + for index, item in enumerate(raw_messages) + ] + except OutputContractError as exc: + raise DialectError("Anthropic opaque continuation is malformed") from exc + _validate_replay_system(system) + _validate_replay_messages(messages) + tool_history_rejection, pending_tool_calls = _tool_history_state(messages) + if tool_history_rejection is not None: + raise DialectError(tool_history_rejection) + if open_turn != bool(pending_tool_calls): + raise DialectError( + "Anthropic continuation open_turn disagrees with its terminal " + "tool-use state" + ) + raw_tools = raw["tools"] + if raw_tools is None: + tools = None + elif isinstance(raw_tools, list) and raw_tools: + try: + tools = [ + _json_mapping(item, f"opaque.tools[{index}]") + for index, item in enumerate(raw_tools) + ] + except OutputContractError as exc: + raise DialectError("Anthropic continuation tools are malformed") from exc + _validate_replay_tools(tools) + else: + raise DialectError("Anthropic continuation tools must be null or non-empty") + has_signed_thinking = any( + block.get("type") in {"thinking", "redacted_thinking"} + for message in messages + if message.get("role") == "assistant" + for block in cast(list[Mapping[str, JSONValue]], message.get("content", [])) + ) + if not has_signed_thinking and ( + not open_turn or (thinking is None and effort is None) + ): + raise DialectError( + "Anthropic opaque continuation contains no replayable thinking/tool state" + ) + return _DecodedContinuation( + system=tuple(system), + messages=tuple(messages), + tools=tuple(tools) if tools is not None else None, + open_turn=open_turn, + thinking=thinking, + effort=effort, + ) + + +def _continuation_reasoning_rejection(req: Request) -> str | None: + continuation = req.session.opaque_continuation + if continuation is None: + return None + decoded = _decode_continuation(continuation.value) + if not decoded.open_turn: + return None + current_thinking = _thinking_config(req) + if _canonical_json( + current_thinking, "current thinking configuration" + ) != _canonical_json(decoded.thinking, "continued thinking configuration"): + return ( + "an open Anthropic tool turn must reuse the exact thinking mode, " + "budget, and display from the prior request" + ) + if req.reasoning.effort != decoded.effort: + return ( + "an open Anthropic tool turn must reuse the exact reasoning effort " + "from the prior request" + ) + return None + + +def _continuation_tools_mismatch( + continuation: OpaqueContinuation, + body: Mapping[str, JSONValue], +) -> str | None: + decoded = _decode_continuation(continuation.value) + if not decoded.open_turn: + return None + expected: JSONValue = ( + [dict(tool) for tool in decoded.tools] if decoded.tools is not None else None + ) + if _canonical_json(body.get("tools"), "body.tools") != _canonical_json( + expected, "continued tools" + ): + return ( + "body.tools must exactly match the tool definitions from the " + "continued Anthropic thinking history" + ) + return None + + +def _validate_replay_system(system: Sequence[Mapping[str, JSONValue]]) -> None: + for index, block in enumerate(system): + if set(block) != {"type", "text"} or block.get("type") != "text": + raise DialectError( + f"opaque.system[{index}] must be an Anthropic text block" + ) + if not isinstance(block.get("text"), str): + raise DialectError(f"opaque.system[{index}].text must be a string") + + +def _validate_replay_messages(messages: Sequence[Mapping[str, JSONValue]]) -> None: + for index, message in enumerate(messages): + if set(message) != {"role", "content"}: + raise DialectError(f"opaque.messages[{index}] has unsupported fields") + if message.get("role") not in {"user", "assistant"}: + raise DialectError(f"opaque.messages[{index}].role is invalid") + content = message.get("content") + if not isinstance(content, list) or not content: + raise DialectError(f"opaque.messages[{index}].content must be non-empty") + for block_index, block in enumerate(content): + if not isinstance(block, Mapping): + raise DialectError( + f"opaque.messages[{index}].content[{block_index}] must be an object" + ) + _validate_history_block( + block, + f"opaque.messages[{index}].content[{block_index}]", + role=cast(str, message["role"]), + ) + + +def _tool_history_state( + messages: Sequence[Mapping[str, JSONValue]], +) -> tuple[str | None, frozenset[str]]: + """Validate Anthropic's immediate, one-result-per-call tool protocol.""" + + seen_call_ids: set[str] = set() + pending_call_ids: frozenset[str] = frozenset() + for message_index, message in enumerate(messages): + content = message.get("content") + if not isinstance(content, list): + return f"messages[{message_index}].content must be a list", frozenset() + + call_ids: list[str] = [] + result_ids: list[str] = [] + saw_non_result = False + for block_index, block in enumerate(content): + if not isinstance(block, Mapping): + return ( + f"messages[{message_index}].content[{block_index}] must be an " + "object", + frozenset(), + ) + block_type = block.get("type") + if block_type == "tool_result": + if saw_non_result: + return ( + f"messages[{message_index}] tool_result blocks must precede " + "all non-tool-result content", + frozenset(), + ) + tool_use_id = block.get("tool_use_id") + if isinstance(tool_use_id, str): + result_ids.append(tool_use_id) + else: + saw_non_result = True + if block_type == "tool_use": + call_id = block.get("id") + if isinstance(call_id, str): + call_ids.append(call_id) + + if result_ids: + if message.get("role") != "user": + return ( + f"messages[{message_index}] tool results require user role", + frozenset(), + ) + if len(result_ids) != len(set(result_ids)): + return ( + f"messages[{message_index}] duplicates a tool_result id", + frozenset(), + ) + if not pending_call_ids: + return ( + f"messages[{message_index}] tool results do not immediately " + "follow tool uses", + frozenset(), + ) + actual_results = frozenset(result_ids) + if actual_results != pending_call_ids: + return ( + f"messages[{message_index}] tool_result ids must exactly match " + "the preceding assistant tool_use ids", + frozenset(), + ) + pending_call_ids = frozenset() + elif pending_call_ids: + return ( + f"messages[{message_index}] must immediately return every preceding " + "assistant tool result", + frozenset(), + ) + + if call_ids: + if message.get("role") != "assistant": + return ( + f"messages[{message_index}] tool uses require assistant role", + frozenset(), + ) + if len(call_ids) != len(set(call_ids)) or any( + call_id in seen_call_ids for call_id in call_ids + ): + return ( + f"messages[{message_index}] duplicates a tool_use id", + frozenset(), + ) + seen_call_ids.update(call_ids) + pending_call_ids = frozenset(call_ids) + + return None, pending_call_ids + + +def _validate_replay_tools(tools: Sequence[Mapping[str, JSONValue]]) -> None: + for index, tool in enumerate(tools): + path = f"opaque.tools[{index}]" + allowed = {"name", "description", "input_schema", "strict"} + if set(tool) - allowed: + raise DialectError(f"{path} has unsupported fields") + name = tool.get("name") + if not isinstance(name, str) or not name: + raise DialectError(f"{path}.name must be a non-empty string") + if not isinstance(tool.get("input_schema"), Mapping): + raise DialectError(f"{path}.input_schema must be an object") + if "description" in tool and not isinstance(tool["description"], str): + raise DialectError(f"{path}.description must be a string") + if "strict" in tool and not isinstance(tool["strict"], bool): + raise DialectError(f"{path}.strict must be a boolean") + + +def _validate_history_block( + block: Mapping[str, JSONValue], path: str, *, role: str +) -> None: + block_type = block.get("type") + if block_type == "text": + if set(block) - {"type", "text", "citations"} or not isinstance( + block.get("text"), str + ): + raise DialectError(f"{path} is not a canonical text block") + citations = block.get("citations") + if "citations" in block and citations is not None and citations != []: + raise DialectError(f"{path}.citations cannot be replayed losslessly") + return + if block_type == "image": + if role != "user": + raise DialectError(f"{path} image block requires user role") + _validate_history_image(block, path) + return + if block_type == "tool_use": + if role != "assistant": + raise DialectError(f"{path} tool_use block requires assistant role") + allowed = {"type", "id", "name", "input"} + if set(block) != allowed: + raise DialectError(f"{path} is not a canonical tool_use block") + _required_history_string(block.get("id"), f"{path}.id") + _required_history_string(block.get("name"), f"{path}.name") + if not isinstance(block.get("input"), Mapping): + raise DialectError(f"{path}.input must be an object") + return + if block_type == "tool_result": + if role != "user": + raise DialectError(f"{path} tool_result block requires user role") + allowed = {"type", "tool_use_id", "content", "is_error"} + if set(block) != allowed: + raise DialectError(f"{path} is not a canonical tool_result block") + _required_history_string(block.get("tool_use_id"), f"{path}.tool_use_id") + if not isinstance(block.get("content"), str): + raise DialectError(f"{path}.content must be a string") + if not isinstance(block.get("is_error"), bool): + raise DialectError(f"{path}.is_error must be a boolean") + return + if block_type == "thinking": + if role != "assistant" or set(block) != { + "type", + "thinking", + "signature", + }: + raise DialectError(f"{path} is not a canonical thinking block") + if not isinstance(block.get("thinking"), str): + raise DialectError(f"{path}.thinking must be a string") + _required_history_string(block.get("signature"), f"{path}.signature") + return + if block_type == "redacted_thinking": + if role != "assistant" or set(block) != {"type", "data"}: + raise DialectError(f"{path} is not a canonical redacted-thinking block") + _required_history_string(block.get("data"), f"{path}.data") + return + raise DialectError(f"{path} has unsupported Anthropic block type {block_type!r}") + + +def _validate_history_image(block: Mapping[str, JSONValue], path: str) -> None: + if set(block) != {"type", "source"} or not isinstance(block.get("source"), Mapping): + raise DialectError(f"{path} is not a canonical image block") + source = cast(Mapping[str, JSONValue], block["source"]) + source_type = source.get("type") + if source_type == "url": + if set(source) != {"type", "url"}: + raise DialectError(f"{path}.source is not a canonical URL source") + _required_history_string(source.get("url"), f"{path}.source.url") + return + if source_type == "base64": + if set(source) != {"type", "media_type", "data"}: + raise DialectError(f"{path}.source is not a canonical base64 source") + if source.get("media_type") not in _IMAGE_MEDIA_TYPES: + raise DialectError(f"{path}.source.media_type is unsupported") + _required_history_string(source.get("data"), f"{path}.source.data") + return + raise DialectError(f"{path}.source.type is unsupported") + + +def _required_history_string(value: object, path: str) -> str: + if not isinstance(value, str) or not value: + raise DialectError(f"{path} must be a non-empty string") + return value + + +def _verification_image(part: ImagePart) -> dict[str, JSONValue]: + if part.url is not None: + return {"type": "image", "source": {"type": "url", "url": part.url}} + return { + "type": "image", + "source": { + "type": "base64", + "media_type": cast(str, part.media_type), + "data": cast(str, part.data), + }, + } + + +def _verification_part(part: object) -> dict[str, JSONValue]: + if isinstance(part, TextPart): + return {"type": "text", "text": part.text} + if isinstance(part, ImagePart): + return _verification_image(part) + if isinstance(part, ToolCallPart): + return { + "type": "tool_use", + "id": part.call_id, + "name": part.name, + "input": copy_json_value(part.arguments, "verified tool call"), + } + if isinstance(part, ToolResultPart): + content = ( + part.result + if isinstance(part.result, str) + else json.dumps( + copy_json_value(part.result, "verified tool result"), + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ) + ) + return { + "type": "tool_result", + "tool_use_id": part.call_id, + "content": content, + "is_error": part.is_error, + } + raise AssertionError(f"unclassified verification part {type(part).__name__}") + + +def _verification_current_input( + input_: ChatInput, +) -> tuple[list[dict[str, JSONValue]], list[dict[str, JSONValue]]]: + system: list[dict[str, JSONValue]] = [] + messages: list[dict[str, JSONValue]] = [] + for message in input_.messages: + if message.role == "system": + for part in message.content: + assert isinstance(part, TextPart) + system.append({"type": "text", "text": part.text}) + continue + messages.append( + { + "role": "user" if message.role == "tool" else message.role, + "content": [_verification_part(part) for part in message.content], + } + ) + return system, messages + + +def _verification_function_tool( + raw: Mapping[str, JSONValue], index: int +) -> dict[str, JSONValue]: + """Independently derive one expected tool wire shape for audit evidence.""" + + data = dict(raw) + nested = data.get("function") + if nested is not None: + if data.get("type") != "function" or not isinstance(nested, Mapping): + raise AssertionError(f"validated tool {index} changed shape") + source = dict(nested) + else: + source = {key: value for key, value in data.items() if key != "type"} + name = source.get("name") + parameters = source.get("parameters") + if not isinstance(name, str) or not isinstance(parameters, Mapping): + raise AssertionError(f"validated tool {index} changed shape") + result: dict[str, JSONValue] = { + "name": name, + "input_schema": copy_json_value( + parameters, f"verified function tool {index} parameters" + ), + } + if "description" in source: + result["description"] = cast(str, source["description"]) + if "strict" in source: + result["strict"] = cast(bool, source["strict"]) + return result + + +def _verification_tool_choice(raw: JSONValue) -> dict[str, JSONValue]: + """Independently derive Anthropic tool choice from provider-neutral input.""" + + if isinstance(raw, str): + return { + "type": { + "auto": "auto", + "required": "any", + "none": "none", + }[raw] + } + assert isinstance(raw, Mapping) + raw_mapping = cast(Mapping[str, object], raw) + nested = raw_mapping.get("function") + name = ( + cast(Mapping[str, object], nested).get("name") + if isinstance(nested, Mapping) + else raw_mapping.get("name") + ) + assert isinstance(name, str) + return {"type": "tool", "name": name} + + +@dataclass(frozen=True) +class _AnthropicInputVerifier: + source: ChatInput + continuation: OpaqueContinuation | None + + def __post_init__(self) -> None: + object.__setattr__(self, "source", deepcopy(self.source)) + object.__setattr__(self, "continuation", deepcopy(self.continuation)) + + def verify(self, body: Mapping[str, JSONValue]) -> str | None: + current_system, current_messages = _verification_current_input(self.source) + history_system: list[dict[str, JSONValue]] = [] + history_messages: list[dict[str, JSONValue]] = [] + if self.continuation is not None: + decoded = _decode_continuation(self.continuation.value) + history_system = list(decoded.system) + history_messages = list(decoded.messages) + expected_system = [*history_system, *current_system] + expected_messages = [*history_messages, *current_messages] + actual_system = body.get("system") + if expected_system: + if actual_system is None: + return "body.system is missing" + if _canonical_json(actual_system, "body.system") != _canonical_json( + expected_system, "expected system" + ): + return "body.system changed" + elif actual_system is not None: + return "body.system was added" + actual_messages = body.get("messages") + if _canonical_json(actual_messages, "body.messages") != _canonical_json( + expected_messages, "expected messages" + ): + return "body.messages changed" + if self.continuation is not None: + mismatch = _continuation_tools_mismatch(self.continuation, body) + if mismatch is not None: + return mismatch + return None + + +@dataclass(frozen=True) +class _AnthropicToolsVerifier: + source: tuple[Mapping[str, JSONValue], ...] + + def __post_init__(self) -> None: + object.__setattr__(self, "source", deepcopy(self.source)) + + def verify(self, body: Mapping[str, JSONValue]) -> str | None: + actual = body.get("tools") + if not isinstance(actual, list) or len(actual) != len(self.source): + return "body.tools count changed" + for index, (source, wire) in enumerate(zip(self.source, actual, strict=True)): + expected = _verification_function_tool(source, index) + if _canonical_json(wire, f"body.tools[{index}]") != _canonical_json( + expected, f"expected tools[{index}]" + ): + return f"body.tools[{index}] changed" + return None + + +@dataclass(frozen=True) +class _AnthropicToolChoiceVerifier: + choice: JSONValue + parallel: bool | None + + def __post_init__(self) -> None: + object.__setattr__(self, "choice", deepcopy(self.choice)) + + def verify(self, body: Mapping[str, JSONValue]) -> str | None: + expected: dict[str, JSONValue] | None = None + if self.choice is not None: + expected = _verification_tool_choice(self.choice) + if self.parallel is not None: + if expected is None: + expected = {"type": "auto"} + expected["disable_parallel_tool_use"] = not self.parallel + if _canonical_json(body.get("tool_choice"), "body.tool_choice") != ( + _canonical_json(expected, "expected tool choice") + ): + return "body.tool_choice changed" + return None + + +@dataclass(frozen=True) +class _ReasoningSummaryVerifier: + summary: str + has_budget: bool + + def verify(self, body: Mapping[str, JSONValue]) -> str | None: + thinking = body.get("thinking") + if not isinstance(thinking, Mapping): + return "body.thinking is missing" + thinking_mapping = cast(Mapping[str, object], thinking) + expected = "summarized" if self.summary == "auto" else "omitted" + if thinking_mapping.get("display") != expected: + return "body.thinking.display changed" + expected_type = "enabled" if self.has_budget else "adaptive" + if thinking_mapping.get("type") != expected_type: + return "body.thinking.type changed" + return None + + +def _message_rejection(input_: ChatInput) -> str | None: + saw_conversation = False + for index, message in enumerate(input_.messages): + if not message.content: + return f"messages[{index}] has empty content" + for part_index, part in enumerate(message.content): + if isinstance(part, TextPart) and not isinstance(part.text, str): + return f"messages[{index}].content[{part_index}].text must be a string" + if message.name is not None: + return "Anthropic Messages has no message name field" + if message.role == "developer": + return "Anthropic Messages has no developer-role authority level" + if message.role == "system": + if saw_conversation: + return "Anthropic system messages must precede conversation turns" + if not all(isinstance(part, TextPart) for part in message.content): + return "Anthropic top-level system accepts text content only" + continue + saw_conversation = True + if message.role == "tool" and not all( + isinstance(part, ToolResultPart) for part in message.content + ): + return "Anthropic tool-role messages require only tool results" + if message.role == "user" and any( + isinstance(part, ToolCallPart) for part in message.content + ): + return "Anthropic tool_use blocks require assistant role" + if message.role == "assistant" and any( + isinstance(part, ImagePart | ToolResultPart) for part in message.content + ): + return "Anthropic assistant messages cannot contain images or tool results" + for part in message.content: + if isinstance(part, ToolCallPart): + if not part.call_id or not part.name: + return "Anthropic tool_use requires non-empty id and name" + if not isinstance(part.arguments, Mapping): + return "Anthropic tool_use input must be a JSON object" + if isinstance(part, ToolResultPart) and not part.call_id: + return "Anthropic tool_result requires a non-empty tool_use_id" + if not saw_conversation: + return "Anthropic Messages requires at least one conversational message" + return None + + +def _tool_history_rejection(req: Request) -> str | None: + if not isinstance(req.input, ChatInput): + return None + history_messages: list[dict[str, JSONValue]] = [] + continuation = req.session.opaque_continuation + if continuation is not None: + history_messages.extend(_decode_continuation(continuation.value).messages) + _, current_messages = _lower_chat_input(req.input) + rejection, pending_call_ids = _tool_history_state( + [*history_messages, *current_messages] + ) + if rejection is not None: + return rejection + if pending_call_ids: + return "Anthropic request leaves tool_use blocks without immediate results" + return None + + +def _has_system(input_: ChatInput) -> bool: + return any(message.role == "system" for message in input_.messages) + + +def _validate_function_tools(req: Request) -> str | None: + if (req.tools.choice is not None or req.tools.parallel is not None) and not ( + req.tools.functions + ): + return "tool choice and parallel controls require function tools" + try: + for tool in req.tools.functions: + _function_tool_to_wire(tool) + if req.tools.choice is not None: + choice = _tool_choice_to_wire(req.tools.choice) + if choice.get("type") == "none" and req.tools.parallel is not None: + return "parallel tool control is meaningless with tool_choice='none'" + except DialectError as exc: + return str(exc) + return None + + +def _structured_output(req: Request, text: str) -> StructuredOutput | None: + if req.structured_output.format is None: + return None + try: + return StructuredOutput(_strict_json_loads(text)) + except (TypeError, ValueError) as exc: + raise OutputContractError("structured output is not valid JSON") from exc + + +def _usage_stats(raw: object) -> UsageStats: + usage = _json_mapping(raw, "response.usage") + allowed = { + "input_tokens", + "output_tokens", + "cache_creation_input_tokens", + "cache_read_input_tokens", + "cache_creation", + "server_tool_use", + "service_tier", + "inference_geo", + "output_tokens_details", + } + unknown = set(usage) - allowed + if unknown: + # Unknown accounting fields can change totals. Refuse to pretend the + # persisted usage record is complete until they have an explicit mapping. + raise OutputContractError( + "response.usage has unsupported field(s): " + ", ".join(sorted(unknown)) + ) + uncached = _nonnegative_int( + usage.get("input_tokens"), "response.usage.input_tokens" + ) + output = _nonnegative_int( + usage.get("output_tokens"), "response.usage.output_tokens" + ) + reported_cache_creation = usage.get("cache_creation_input_tokens") + reported_cache_read = usage.get("cache_read_input_tokens") + cache_creation = 0 if reported_cache_creation is None else reported_cache_creation + cache_read = 0 if reported_cache_read is None else reported_cache_read + cache_creation_count = _nonnegative_int( + cache_creation, "response.usage.cache_creation_input_tokens" + ) + cache_read_count = _nonnegative_int( + cache_read, "response.usage.cache_read_input_tokens" + ) + cache_creation_details = usage.get("cache_creation") + if cache_creation_details is not None: + details = _json_mapping(cache_creation_details, "response.usage.cache_creation") + expected_detail_keys = { + "ephemeral_1h_input_tokens", + "ephemeral_5m_input_tokens", + } + if set(details) != expected_detail_keys: + raise OutputContractError( + "response.usage.cache_creation has unsupported fields" + ) + cache_detail_total = sum( + _nonnegative_int(details[key], f"response.usage.cache_creation.{key}") + for key in sorted(expected_detail_keys) + ) + if reported_cache_creation is None: + cache_creation_count = cache_detail_total + elif cache_detail_total != cache_creation_count: + raise OutputContractError( + "response.usage cache-creation breakdown disagrees with its total" + ) + server_tool_usage = usage.get("server_tool_use") + if server_tool_usage is not None: + server_details = _json_mapping( + server_tool_usage, "response.usage.server_tool_use" + ) + expected_server_keys = {"web_fetch_requests", "web_search_requests"} + if set(server_details) != expected_server_keys: + raise OutputContractError( + "response.usage.server_tool_use has unsupported fields" + ) + server_counts = [ + _nonnegative_int( + server_details[key], f"response.usage.server_tool_use.{key}" + ) + for key in sorted(expected_server_keys) + ] + if any(server_counts): + raise OutputContractError( + "Anthropic response reports server-tool usage that has no " + "shared-IR mapping" + ) + service_tier = usage.get("service_tier") + if service_tier is not None and service_tier not in { + "standard", + "priority", + "batch", + }: + raise OutputContractError("response.usage.service_tier is invalid") + inference_geo = usage.get("inference_geo") + if inference_geo is not None and not isinstance(inference_geo, str): + raise OutputContractError("response.usage.inference_geo must be a string") + reasoning_tokens: int | None = None + raw_output_details = usage.get("output_tokens_details") + if raw_output_details is not None: + details_mapping = _json_mapping( + raw_output_details, "response.usage.output_tokens_details" + ) + if set(details_mapping) != {"thinking_tokens"}: + raise OutputContractError( + "response.usage.output_tokens_details has unsupported fields" + ) + reasoning_tokens = _nonnegative_int( + details_mapping.get("thinking_tokens"), + "response.usage.output_tokens_details.thinking_tokens", + ) + if reasoning_tokens > output: + raise OutputContractError( + "response.usage thinking_tokens exceeds inclusive output_tokens" + ) + input_total = uncached + cache_creation_count + cache_read_count + return UsageStats( + input_tokens=input_total, + output_tokens=output, + total_tokens=input_total + output, + reasoning_tokens=reasoning_tokens, + cached_tokens=cache_read_count if reported_cache_read is not None else None, + ) + + +def _validated_content_block( + raw: object, index: int +) -> tuple[dict[str, JSONValue], str | None, FunctionToolCall | None, bool]: + path = f"response.content[{index}]" + block = _json_mapping(raw, path) + block_type = block.get("type") + if block_type == "text": + allowed = {"type", "text", "citations"} + if set(block) - allowed: + raise OutputContractError(f"{path} has unsupported text fields") + text = block.get("text") + if not isinstance(text, str): + raise OutputContractError(f"{path}.text must be a string") + citations = block.get("citations") + if citations is not None and citations != []: + raise OutputContractError( + "Anthropic citations have no lossless provider-neutral mapping" + ) + canonical: dict[str, JSONValue] = {"type": "text", "text": text} + if "citations" in block: + canonical["citations"] = copy_json_value(citations, f"{path}.citations") + return canonical, text, None, False + if block_type == "thinking": + if set(block) != {"type", "thinking", "signature"}: + raise OutputContractError(f"{path} has unsupported thinking fields") + thinking = block.get("thinking") + if not isinstance(thinking, str): + raise OutputContractError(f"{path}.thinking must be a string") + signature = _required_string(block.get("signature"), f"{path}.signature") + return ( + {"type": "thinking", "thinking": thinking, "signature": signature}, + thinking, + None, + True, + ) + if block_type == "redacted_thinking": + if set(block) != {"type", "data"}: + raise OutputContractError( + f"{path} has unsupported redacted-thinking fields" + ) + data = _required_string(block.get("data"), f"{path}.data") + return {"type": "redacted_thinking", "data": data}, None, None, True + if block_type == "tool_use": + allowed = {"type", "id", "name", "input", "caller", "toolset_name"} + if set(block) - allowed: + raise OutputContractError(f"{path} has unsupported tool_use fields") + if "caller" in block or "toolset_name" in block: + raise OutputContractError( + "Anthropic delegated/toolset tool calls have no shared-IR mapping" + ) + call_id = _required_string(block.get("id"), f"{path}.id") + name = _required_string(block.get("name"), f"{path}.name") + arguments = block.get("input") + if not isinstance(arguments, Mapping): + raise OutputContractError(f"{path}.input must be an object") + copied_arguments = copy_json_value(arguments, f"{path}.input") + canonical = { + "type": "tool_use", + "id": call_id, + "name": name, + "input": copied_arguments, + } + return ( + canonical, + None, + FunctionToolCall(call_id, name, copied_arguments), + False, + ) + raise OutputContractError( + f"Anthropic content block type {block_type!r} has no shared-IR mapping" + ) + + +def _opaque_history_bundle( + context: _ExecutionContext, + output_content: Sequence[Mapping[str, JSONValue]], + *, + saw_opaque_reasoning: bool, + stop_reason: str, +) -> str | None: + raw_thinking = context.request_params.get("thinking") + thinking: JSONValue = None + if raw_thinking is not None: + thinking = copy_json_value(raw_thinking, "request.thinking") + try: + _validate_continuation_thinking(thinking) + except DialectError as exc: + raise OutputContractError( + "final Anthropic request had invalid thinking state" + ) from exc + effort: JSONValue = None + raw_output_config = context.request_params.get("output_config") + if raw_output_config is not None: + output_config = _json_mapping(raw_output_config, "request.output_config") + effort = output_config.get("effort") + if effort is not None and ( + not isinstance(effort, str) or effort not in _REASONING_EFFORTS + ): + raise OutputContractError("final Anthropic request had invalid effort") + raw_tools = context.request_params.get("tools") + tools: JSONValue = None + if raw_tools is not None: + if not isinstance(raw_tools, list) or not raw_tools: + raise OutputContractError("final Anthropic request had invalid tools") + tools = copy_json_value(raw_tools, "request.tools") + try: + _validate_replay_tools( + [ + _json_mapping(tool, f"request.tools[{index}]") + for index, tool in enumerate(raw_tools) + ] + ) + except (DialectError, OutputContractError) as exc: + raise OutputContractError( + "final Anthropic request had invalid tools" + ) from exc + # ``pause_turn`` is intentionally rejected before this helper because PR 3 + # does not bind the server-hosted tools that produce it. + open_turn = stop_reason == "tool_use" + should_preserve = ( + saw_opaque_reasoning + or context.request.session.opaque_continuation is not None + or (open_turn and (thinking is not None or effort is not None)) + ) + if not should_preserve: + return None + messages: list[JSONValue] = [ + copy_json_value(message, f"request.messages[{index}]") + for index, message in enumerate(context.messages) + ] + # Anthropic documents an empty ``end_turn`` response as a valid outcome and + # instructs callers to append a new user prompt without replaying that empty + # assistant response. Keeping ``content: []`` here would also create an + # opaque continuation that this dialect's own replay validator rejects. + if output_content: + messages.append( + { + "role": "assistant", + "content": [ + copy_json_value(block, f"response.content[{index}]") + for index, block in enumerate(output_content) + ], + } + ) + return json.dumps( + { + "version": _OPAQUE_CONTINUATION_VERSION, + "system": [ + copy_json_value(block, f"request.system[{index}]") + for index, block in enumerate(context.system) + ], + "messages": messages, + "tools": tools, + "open_turn": open_turn, + "reasoning": {"thinking": thinking, "effort": effort}, + }, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ) + + +def _headers(connection: AsyncHTTPJSONConnection) -> dict[str, str]: + headers = { + "anthropic-version": _ANTHROPIC_VERSION, + "accept": "application/json", + } + if connection.credential is not None: + headers["x-api-key"] = connection.credential + return headers + + +def _execution_context( + prepared: PreparedRequest, + context: _AnthropicContext, + expected_model: str, +) -> tuple[dict[str, JSONValue], bool, _ExecutionContext]: + body = prepared.thaw_body() + model = body.get("model") + if not isinstance(model, str) or not model: + raise DialectError("Anthropic prepared body has an invalid model") + if model != expected_model: + raise DialectError( + "Anthropic prepared body model does not match the bound model" + ) + stream = body.get("stream") + if not isinstance(stream, bool): + raise DialectError("Anthropic prepared body has an invalid stream flag") + raw_messages = body.get("messages") + if not isinstance(raw_messages, list): + raise DialectError("Anthropic prepared body has invalid messages") + raw_system = body.get("system", []) + if not isinstance(raw_system, list): + raise DialectError("Anthropic prepared body has invalid system content") + messages = tuple( + _json_mapping(item, f"prepared.messages[{index}]") + for index, item in enumerate(raw_messages) + ) + system = tuple( + _json_mapping(item, f"prepared.system[{index}]") + for index, item in enumerate(raw_system) + ) + continuation = context.request.session.opaque_continuation + if continuation is not None: + mismatch = _continuation_tools_mismatch(continuation, body) + if mismatch is not None: + raise DialectError(mismatch) + request_params = { + key: copy_json_value(value, f"prepared.{key}") + for key, value in body.items() + if key not in {"model", "messages", "system"} + } + return ( + body, + stream, + _ExecutionContext( + request=deepcopy(context.request), + system=system, + messages=messages, + request_params=request_params, + ), + ) + + +def _parse_json_response(response: httpx.Response) -> dict[str, JSONValue]: + try: + raw = _strict_json_loads(response.content) + except (UnicodeDecodeError, ValueError) as exc: + raise OutputContractError("Anthropic response is not valid JSON") from exc + return _json_mapping(raw, "response") + + +async def _sse_events(response: httpx.Response) -> AsyncIterator[dict[str, JSONValue]]: + event_name: str | None = None + data_lines: list[str] = [] + + def finish_event() -> dict[str, JSONValue] | None: + nonlocal event_name, data_lines + if not data_lines: + event_name = None + return None + data = "\n".join(data_lines) + data_lines = [] + try: + parsed = _strict_json_loads(data) + except ValueError as exc: + raise OutputContractError("Anthropic SSE data is not valid JSON") from exc + event = _json_mapping(parsed, "stream event") + payload_type = event.get("type") + if event_name is not None and event_name != payload_type: + raise OutputContractError( + "Anthropic SSE event name disagrees with payload type" + ) + event_name = None + return event + + async for line in response.aiter_lines(): + if line == "": + event = finish_event() + if event is not None: + yield event + continue + if line.startswith(":"): + continue + field, separator, value = line.partition(":") + if separator and value.startswith(" "): + value = value[1:] + if field == "event": + event_name = value + elif field == "data": + data_lines.append(value) + elif field not in {"id", "retry"}: + raise OutputContractError(f"unsupported Anthropic SSE field {field!r}") + event = finish_event() + if event is not None: + yield event + + +def _apply_stream_delta( + block: dict[str, JSONValue], delta: Mapping[str, JSONValue], path: str +) -> None: + delta_type = delta.get("type") + block_type = block.get("type") + if delta_type == "text_delta" and block_type == "text": + _reject_unknown_fields(delta, frozenset({"type", "text"}), path) + text = delta.get("text") + if not isinstance(text, str): + raise OutputContractError(f"{path}.text must be a string") + block["text"] = cast(str, block.get("text", "")) + text + return + if delta_type == "thinking_delta" and block_type == "thinking": + _reject_unknown_fields(delta, frozenset({"type", "thinking"}), path) + if block.get("_signature_delta_seen") is True: + raise OutputContractError(f"{path} followed the terminal signature delta") + thinking = delta.get("thinking") + if not isinstance(thinking, str): + raise OutputContractError(f"{path}.thinking must be a string") + block["thinking"] = cast(str, block.get("thinking", "")) + thinking + return + if delta_type == "signature_delta" and block_type == "thinking": + _reject_unknown_fields(delta, frozenset({"type", "signature"}), path) + if block.get("_signature_delta_seen") is True: + raise OutputContractError(f"{path} duplicated the signature delta") + signature = delta.get("signature") + if not isinstance(signature, str): + raise OutputContractError(f"{path}.signature must be a string") + block["signature"] = cast(str, block.get("signature", "")) + signature + block["_signature_delta_seen"] = True + return + if delta_type == "input_json_delta" and block_type == "tool_use": + _reject_unknown_fields(delta, frozenset({"type", "partial_json"}), path) + partial = delta.get("partial_json") + if not isinstance(partial, str): + raise OutputContractError(f"{path}.partial_json must be a string") + block["_partial_json"] = cast(str, block.get("_partial_json", "")) + partial + return + raise OutputContractError( + f"{path} type {delta_type!r} does not match block type {block_type!r}" + ) + + +def _finish_stream_block(block: dict[str, JSONValue], path: str) -> None: + block.pop("_signature_delta_seen", None) + if block.get("type") != "tool_use": + return + partial = block.pop("_partial_json", "") + if not isinstance(partial, str): + raise OutputContractError(f"{path} tool JSON state is invalid") + if not partial: + existing = block.get("input") + if isinstance(existing, Mapping): + return + raise OutputContractError(f"{path}.input is absent") + existing = block.get("input") + if not isinstance(existing, Mapping) or existing: + raise OutputContractError( + f"{path}.input must start as an empty object before JSON deltas" + ) + try: + parsed = _strict_json_loads(partial) + except ValueError as exc: + raise OutputContractError(f"{path}.input is not valid JSON") from exc + if not isinstance(parsed, dict): + raise OutputContractError(f"{path}.input must be an object") + block["input"] = copy_json_value(parsed, f"{path}.input") + + +async def _terminal_stream_message(response: httpx.Response) -> dict[str, JSONValue]: + message: dict[str, JSONValue] | None = None + blocks: dict[int, dict[str, JSONValue]] = {} + stopped_blocks: set[int] = set() + saw_delta = False + saw_stop = False + async for event in _sse_events(response): + event_type = event.get("type") + if not isinstance(event_type, str) or event_type not in _STREAM_EVENT_KEYS: + raise OutputContractError( + f"unsupported Anthropic stream event {event_type!r}" + ) + _reject_unknown_fields(event, _STREAM_EVENT_KEYS[event_type], "stream event") + if saw_stop: + raise OutputContractError( + "Anthropic stream emitted data after message_stop" + ) + if event_type == "ping": + continue + if event_type == "error": + error = event.get("error") + detail = "" + if isinstance(error, Mapping): + error_message = cast(Mapping[str, object], error).get("message") + if isinstance(error_message, str): + detail = f": {error_message}" + raise OutputContractError(f"Anthropic stream error{detail}") + if event_type == "message_start": + if message is not None: + raise OutputContractError( + "Anthropic stream emitted two message_start events" + ) + message = _json_mapping(event.get("message"), "message_start.message") + initial_content = message.get("content") + if initial_content is not None and initial_content != []: + raise OutputContractError( + "Anthropic message_start content must be empty" + ) + if message.get("stop_reason") is not None: + raise OutputContractError( + "Anthropic message_start stop_reason must be null" + ) + if message.get("stop_sequence") is not None: + raise OutputContractError( + "Anthropic message_start stop_sequence must be null" + ) + message["content"] = [] + continue + if message is None: + raise OutputContractError("Anthropic stream event preceded message_start") + if event_type == "content_block_start": + if saw_delta: + raise OutputContractError( + "Anthropic stream emitted content after message_delta" + ) + index = _nonnegative_int(event.get("index"), "content_block_start.index") + if index in blocks: + raise OutputContractError( + f"Anthropic stream duplicated content block {index}" + ) + blocks[index] = _json_mapping( + event.get("content_block"), f"content_block_start[{index}]" + ) + continue + if event_type == "content_block_delta": + if saw_delta: + raise OutputContractError( + "Anthropic stream emitted content after message_delta" + ) + index = _nonnegative_int(event.get("index"), "content_block_delta.index") + if index not in blocks or index in stopped_blocks: + raise OutputContractError( + f"Anthropic delta targets inactive content block {index}" + ) + delta = _json_mapping(event.get("delta"), f"content_block_delta[{index}]") + _apply_stream_delta(blocks[index], delta, f"content_block_delta[{index}]") + continue + if event_type == "content_block_stop": + if saw_delta: + raise OutputContractError( + "Anthropic stream emitted content after message_delta" + ) + index = _nonnegative_int(event.get("index"), "content_block_stop.index") + if index not in blocks or index in stopped_blocks: + raise OutputContractError( + f"Anthropic stop targets inactive content block {index}" + ) + _finish_stream_block(blocks[index], f"content_block[{index}]") + stopped_blocks.add(index) + continue + if event_type == "message_delta": + if saw_delta: + raise OutputContractError( + "Anthropic stream emitted two message_delta events" + ) + delta = _json_mapping(event.get("delta"), "message_delta.delta") + _reject_unknown_fields(delta, _MESSAGE_DELTA_KEYS, "message_delta.delta") + stop_reason = delta.get("stop_reason") + if not isinstance(stop_reason, str) or not stop_reason: + raise OutputContractError( + "message_delta.delta.stop_reason must be a non-empty string" + ) + _validate_stop_details(stop_reason, delta.get("stop_details")) + saw_delta = True + if stop_reason == "refusal": + # Refusal is a terminal non-success outcome. Deliberately + # discard every earlier block, whether still open or already + # closed, so streamed and non-streamed refusals both lift to an + # empty scoreable answer and blocked partial content cannot + # enter task scoring. This is a normalization policy, not an + # implication that Anthropic guarantees either stream shape. + blocks.clear() + stopped_blocks.clear() + elif set(blocks) != stopped_blocks: + raise OutputContractError( + "Anthropic message_delta preceded content_block_stop" + ) + if delta.get("container") is not None: + raise OutputContractError( + "Anthropic response container has no shared-IR mapping" + ) + message["stop_reason"] = stop_reason + message["stop_sequence"] = delta.get("stop_sequence") + message["stop_details"] = delta.get("stop_details") + initial_usage = _json_mapping( + message.get("usage"), "message_start.message.usage" + ) + final_usage = _json_mapping(event.get("usage"), "message_delta.usage") + _nonnegative_int( + final_usage.get("output_tokens"), + "message_delta.usage.output_tokens", + ) + message["usage"] = { + **initial_usage, + **{ + key: value + for key, value in final_usage.items() + if value is not None + }, + } + continue + if event_type == "message_stop": + saw_stop = True + continue + if message is None: + raise OutputContractError("Anthropic stream omitted message_start") + if not saw_delta: + raise OutputContractError("Anthropic stream omitted message_delta") + if not saw_stop: + raise OutputContractError("Anthropic stream omitted message_stop") + if set(blocks) != stopped_blocks: + raise OutputContractError("Anthropic stream left a content block unfinished") + if set(blocks) != set(range(len(blocks))): + raise OutputContractError( + "Anthropic stream content block indexes are not dense" + ) + message["content"] = [blocks[index] for index in range(len(blocks))] + return message + + +class AnthropicMessagesDialect: + """Executable native Anthropic Messages adapter.""" + + dialect_id = "anthropic_messages" + connection_family = "async_http_json" + capability_decisions = CAPABILITY_DECISIONS + output_contract = OUTPUT_CONTRACT + CAPABILITIES = frozenset( + { + Capability.Chat, + Capability.FunctionCalling, + Capability.Reasoning, + Capability.ReasoningEffort, + Capability.StructuredOutput, + Capability.Prefill, + } + ) + + def __init__( + self, connection: AsyncHTTPJSONConnection, requested_model_id: str + ) -> None: + if not isinstance(connection, AsyncHTTPJSONConnection): + raise TypeError("anthropic_messages requires an AsyncHTTPJSONConnection") + if not requested_model_id: + raise ValueError("requested_model_id must not be empty") + self._connection = connection + self._requested_model_id = requested_model_id + + @property + def capabilities(self) -> frozenset[Capability]: + """Deprecated enum view retained during the compatibility cycle.""" + + return self.CAPABILITIES + + def validate_request( + self, req: Request, audit: RequestAudit, plan: RuntimePlanView + ) -> None: + audit.require_matches_request(req) + del plan + if req.sampling.max_tokens is None: + raise DialectError("Anthropic Messages requires sampling.max_tokens") + input_rejection = ( + _message_rejection(req.input) if isinstance(req.input, ChatInput) else None + ) + image_rejection = None + if isinstance(req.input, ChatInput): + image_rejection = next( + ( + rejection + for message in req.input.messages + for part in message.content + if isinstance(part, ImagePart) + if (rejection := _image_rejection(part)) is not None + ), + None, + ) + tool_rejection = _validate_function_tools(req) + tool_history_rejection = ( + _tool_history_rejection(req) + if input_rejection is None and image_rejection is None + else None + ) + thinking_config = _thinking_config(req) + has_prefill = isinstance(req.input, ChatInput) and _has_assistant_prefill( + req.input + ) + forced_tool_choice = False + if req.tools.choice is not None and tool_rejection is None: + forced_tool_choice = _tool_choice_to_wire(req.tools.choice).get("type") in { + "any", + "tool", + } + continuation_reasoning_rejection = _continuation_reasoning_rejection(req) + continuation_with_system = ( + req.session.opaque_continuation is not None + and isinstance(req.input, ChatInput) + and _has_system(req.input) + ) + for path in audit.active_paths: + if path == "dialect_options": + audit.noop(path, "an empty matching options mapping has no effect") + elif path in {"input.completion", "input.completion.suffix"}: + audit.rejected(path, "anthropic_messages requires ChatInput") + elif path == "input.chat" and input_rejection is not None: + audit.rejected(path, input_rejection) + elif path == "input.chat" and tool_history_rejection is not None: + audit.rejected(path, tool_history_rejection) + elif path == "input.chat" and continuation_with_system: + audit.rejected( + path, + "opaque continuation already carries the prior system prompt; " + "new system messages would make authority ambiguous", + ) + elif ( + path == "input.chat" + and has_prefill + and req.structured_output.format is not None + ): + audit.rejected( + path, + "Anthropic JSON-schema output is incompatible with assistant " + "prefill", + ) + elif path == "input.chat" and has_prefill and thinking_config is not None: + audit.rejected( + path, + "Anthropic thinking is incompatible with assistant prefill", + ) + elif image_rejection is not None and path == image_rejection[0]: + audit.rejected(path, image_rejection[1]) + elif path in { + "scoring.input_scoring", + "scoring.sampled_logprobs", + "scoring.top_logprobs", + }: + audit.rejected(path, "Anthropic Messages does not provide logprobs") + elif path == "sampling.n": + audit.rejected(path, "Anthropic Messages returns one choice per call") + elif path == "sampling.seed": + audit.rejected(path, "Anthropic Messages has no per-request seed") + elif path in {"sampling.frequency_penalty", "sampling.presence_penalty"}: + audit.rejected(path, "Anthropic Messages has no equivalent penalty") + elif ( + path == "sampling.max_tokens" + and req.sampling.max_tokens is not None + and req.sampling.max_tokens < 0 + ): + audit.rejected(path, "Anthropic max_tokens must be non-negative") + elif ( + path == "sampling.temperature" + and req.sampling.temperature is not None + and not 0 <= req.sampling.temperature <= 1 + ): + audit.rejected(path, "Anthropic temperature must be between 0 and 1") + elif path == "sampling.temperature" and thinking_config is not None: + audit.rejected( + path, + "Anthropic sampling temperature is incompatible with explicit " + "thinking", + ) + elif ( + path == "sampling.top_p" + and req.sampling.top_p is not None + and not 0 <= req.sampling.top_p <= 1 + ): + audit.rejected(path, "Anthropic top_p must be between 0 and 1") + elif ( + path == "sampling.top_p" + and thinking_config is not None + and req.sampling.top_p is not None + and req.sampling.top_p < 0.95 + ): + audit.rejected( + path, + "Anthropic top_p must be between 0.95 and 1 with explicit thinking", + ) + elif ( + path == "sampling.top_k" + and req.sampling.top_k is not None + and req.sampling.top_k < 0 + ): + audit.rejected(path, "Anthropic top_k must be non-negative") + elif path == "sampling.top_k" and thinking_config is not None: + audit.rejected( + path, + "Anthropic top_k is incompatible with explicit thinking", + ) + elif ( + path == "sampling.stop" + and req.sampling.stop is not None + and any(not item for item in req.sampling.stop) + ): + audit.rejected(path, "Anthropic stop sequences must not be empty") + elif ( + path == "reasoning.effort" + and req.reasoning.effort not in _REASONING_EFFORTS + ): + audit.rejected( + path, + f"unsupported Anthropic reasoning effort {req.reasoning.effort!r}", + ) + elif ( + path == "reasoning.budget_tokens" + and req.reasoning.budget_tokens is not None + and req.reasoning.budget_tokens < 1024 + ): + audit.rejected(path, "Anthropic thinking budget must be at least 1024") + elif ( + path == "sampling.max_tokens" + and req.reasoning.budget_tokens is not None + and req.sampling.max_tokens is not None + and req.reasoning.budget_tokens >= req.sampling.max_tokens + ): + audit.rejected( + path, + "Anthropic max_tokens must exceed reasoning.budget_tokens", + ) + elif path == "reasoning.summary" and req.reasoning.summary not in { + "none", + "auto", + }: + audit.rejected( + path, + "Anthropic Messages supports only summary='none' or 'auto'", + ) + elif ( + path in {"tools.functions", "tools.choice", "tools.parallel"} + and tool_rejection is not None + ): + audit.rejected(path, tool_rejection) + elif ( + path == "tools.choice" + and req.reasoning.budget_tokens is not None + and forced_tool_choice + ): + audit.rejected( + path, + "manual Anthropic thinking supports only tool_choice='auto' " + "or 'none'", + ) + elif path == "tools.hosted": + audit.rejected(path, "Anthropic hosted tools are not active in PR 3") + elif ( + path == "structured_output.format" + and req.structured_output.format != "json_schema" + ): + audit.rejected(path, "Anthropic structured output requires json_schema") + elif path == "structured_output.name": + audit.rejected(path, "Anthropic output_config has no schema name field") + elif ( + path == "structured_output.strict" + and req.structured_output.strict is False + ): + audit.rejected( + path, "Anthropic JSON-schema output is always schema constrained" + ) + elif path == "session.previous_response_id": + audit.rejected( + path, "Anthropic Messages has no previous-response-id continuation" + ) + elif ( + path == "session.opaque_continuation" + and continuation_reasoning_rejection is not None + ): + audit.rejected(path, continuation_reasoning_rejection) + elif path.startswith("dialect_options."): + key = path.removeprefix("dialect_options.") + if not key: + audit.rejected(path, "dialect option keys must be non-empty") + elif key in _IR_OWNED_BODY_KEYS: + audit.rejected( + path, + f"{key!r} must use its canonical provider-neutral field", + ) + else: + audit.rejected( + path, + "PR 3 exposes no raw Anthropic request passthrough", + ) + + def prepare(self, req: Request, audit: RequestAudit) -> PreparedRequest: + audit.require_matches_request(req) + audit.raise_rejections() + if not isinstance(req.input, ChatInput): + raise DialectError("anthropic_messages requires ChatInput") + if req.sampling.max_tokens is None: + raise DialectError("Anthropic Messages requires sampling.max_tokens") + + active_paths = frozenset(audit.active_paths) + + def consume(path: str, *wire: WireEvidence) -> None: + if path in active_paths and path not in audit.decisions: + audit.consumed(path, *wire) + + continuation = req.session.opaque_continuation + input_verifier = _AnthropicInputVerifier(req.input, continuation) + history_system: list[dict[str, JSONValue]] = [] + history_messages: list[dict[str, JSONValue]] = [] + if continuation is not None: + decoded = _decode_continuation(continuation.value) + history_system = list(decoded.system) + history_messages = list(decoded.messages) + current_system, current_messages = _lower_chat_input(req.input) + system = [*history_system, *current_system] + messages = [*history_messages, *current_messages] + for path in ( + "input.chat", + "input.chat.message.name", + "input.modality.text", + "input.modality.image", + "input.modality.image.media_type", + "input.modality.tool_call", + "input.modality.tool_result", + "input.modality.tool_result.is_error", + ): + consume(path, input_verifier) + + body: dict[str, JSONValue] = { + "model": self._requested_model_id, + "max_tokens": req.sampling.max_tokens, + "messages": cast(JSONValue, messages), + "stream": req.scheduling.stream, + } + if system: + body["system"] = cast(JSONValue, system) + consume("sampling.max_tokens", WireObservation(("max_tokens",))) + for name in ("temperature", "top_p", "top_k"): + value = getattr(req.sampling, name) + if value is not None: + body[name] = value + consume(f"sampling.{name}", WireObservation((name,))) + if req.sampling.stop is not None: + body["stop_sequences"] = list(req.sampling.stop) + consume("sampling.stop", WireObservation(("stop_sequences",))) + + output_config: dict[str, JSONValue] = {} + thinking = _thinking_config(req) + if req.reasoning.budget_tokens is not None: + consume( + "reasoning.budget_tokens", + WireObservation(("thinking", "budget_tokens")), + ) + if req.reasoning.effort is not None: + output_config["effort"] = req.reasoning.effort + consume( + "reasoning.effort", + WireObservation(("output_config", "effort")), + ) + if req.reasoning.summary == "auto": + assert thinking is not None + consume( + "reasoning.summary", + _ReasoningSummaryVerifier( + "auto", req.reasoning.budget_tokens is not None + ), + ) + elif req.reasoning.summary == "none": + assert thinking is not None + consume( + "reasoning.summary", + _ReasoningSummaryVerifier( + "none", req.reasoning.budget_tokens is not None + ), + ) + if thinking is not None: + body["thinking"] = thinking + + if req.tools.functions: + body["tools"] = [ + _function_tool_to_wire(tool) for tool in req.tools.functions + ] + consume("tools.functions", _AnthropicToolsVerifier(req.tools.functions)) + choice = _tool_choice(req) + if choice is not None: + body["tool_choice"] = choice + choice_verifier = _AnthropicToolChoiceVerifier( + req.tools.choice, req.tools.parallel + ) + consume("tools.choice", choice_verifier) + consume("tools.parallel", choice_verifier) + + if req.structured_output.format is not None: + output_config["format"] = { + "type": "json_schema", + "schema": dict(req.structured_output.schema or {}), + } + consume( + "structured_output.format", + WireObservation(("output_config", "format", "type")), + ) + consume( + "structured_output.schema", + WireObservation(("output_config", "format", "schema")), + ) + if req.structured_output.strict is True: + audit.noop( + "structured_output.strict", + "Anthropic JSON-schema output is always schema constrained", + ) + if output_config: + body["output_config"] = output_config + + if continuation is not None: + consume("session.opaque_continuation", input_verifier) + if req.scheduling.stream: + consume("scheduling.stream", WireObservation(("stream",))) + + return PreparedRequest( + operation="messages.create", + body=body, + context=_AnthropicContext(deepcopy(req)), + ) + + def _lift(self, raw: object, context: _ExecutionContext) -> Response: + message = _json_mapping(raw, "response") + _reject_unknown_fields(message, _MESSAGE_RESPONSE_KEYS, "response") + if message.get("type") != "message": + raise OutputContractError("response.type must be 'message'") + if message.get("role") != "assistant": + raise OutputContractError("response.role must be 'assistant'") + _required_string(message.get("id"), "response.id") + response_model = _required_string(message.get("model"), "response.model") + stop_reason = _required_string( + message.get("stop_reason"), "response.stop_reason" + ) + if stop_reason not in _STOP_REASONS: + raise OutputContractError( + f"response.stop_reason {stop_reason!r} is unsupported" + ) + if message.get("container") is not None: + raise OutputContractError( + "Anthropic response container has no shared-IR mapping" + ) + _validate_stop_details(stop_reason, message.get("stop_details")) + stop_sequence = message.get("stop_sequence") + if stop_reason == "stop_sequence": + _required_string(stop_sequence, "response.stop_sequence") + elif stop_sequence is not None: + _optional_string(stop_sequence, "response.stop_sequence") + raise OutputContractError( + "response.stop_sequence must be null unless stop_reason is " + "'stop_sequence'" + ) + + usage = _usage_stats(message.get("usage")) + if stop_reason == "refusal": + # Content is discarded: Anthropic documents a refusal as arriving + # before any output or mid-stream, and says to treat partial output + # as incomplete either way. Validated first anyway, so a shape this + # dialect cannot parse still raises here as it would anywhere else. + for index, raw_block in enumerate( + _sequence(message.get("content"), "response.content") + ): + _validated_content_block(raw_block, index) + return Response( + texts=("",), + finish_reasons=(stop_reason,), + usage=usage, + request_params=context.request_params, + response_model=response_model, + ) + + text_parts: list[str] = [] + thinking_parts: list[str] = [] + tool_calls: list[FunctionToolCall] = [] + canonical_content: list[dict[str, JSONValue]] = [] + saw_opaque_reasoning = False + for index, raw_block in enumerate( + _sequence(message.get("content"), "response.content") + ): + block, text, call, is_opaque_reasoning = _validated_content_block( + raw_block, index + ) + canonical_content.append(block) + if block["type"] == "text" and text is not None: + text_parts.append(text) + elif block["type"] == "thinking" and text is not None: + thinking_parts.append(text) + if call is not None: + tool_calls.append(call) + saw_opaque_reasoning = saw_opaque_reasoning or is_opaque_reasoning + + if ( + context.request.reasoning.budget_tokens is not None + and context.request.session.opaque_continuation is None + and saw_opaque_reasoning + and canonical_content + and canonical_content[0].get("type") + not in {"thinking", "redacted_thinking"} + ): + raise OutputContractError( + "Anthropic manual-thinking response must begin with a thinking block" + ) + + if ( + context.request.reasoning.budget_tokens is not None + and not saw_opaque_reasoning + and context.request.session.opaque_continuation is None + ): + raise OutputContractError( + "Anthropic reply omitted required manual-thinking state" + ) + if stop_reason == "tool_use" and not tool_calls: + raise OutputContractError( + "Anthropic stop_reason='tool_use' requires a tool_use content block" + ) + if tool_calls and stop_reason != "tool_use": + raise OutputContractError( + "Anthropic tool_use content requires stop_reason='tool_use'" + ) + call_ids = [call.call_id for call in tool_calls] + if len(call_ids) != len(set(call_ids)): + raise OutputContractError("Anthropic reply duplicated a tool_use id") + response_message: Mapping[str, JSONValue] = { + "role": "assistant", + "content": cast(JSONValue, canonical_content), + } + history_rejection, _ = _tool_history_state( + (*context.messages, response_message) + ) + if history_rejection is not None: + raise OutputContractError( + f"Anthropic reply produced invalid tool history: {history_rejection}" + ) + text = "".join(text_parts) + if ( + context.request.reasoning.summary == "auto" + and stop_reason not in {"max_tokens", "model_context_window_exceeded"} + and any(block.get("type") == "thinking" for block in canonical_content) + and not any(part.strip() for part in thinking_parts) + ): + raise OutputContractError( + "Anthropic reply omitted the requested visible thinking summary" + ) + opaque = _opaque_history_bundle( + context, + canonical_content, + saw_opaque_reasoning=saw_opaque_reasoning, + stop_reason=stop_reason, + ) + reasoning: tuple[ReasoningOutput | None, ...] | None = None + if opaque is not None: + reasoning = ( + ReasoningOutput( + text="\n".join(thinking_parts) or None, + opaque_roundtrip=opaque, + thinking_tokens=( + usage.reasoning_tokens + if usage.reasoning_tokens is not None + else 0 + ), + ), + ) + elif context.request.reasoning.summary == "auto": + # Adaptive thinking is optional per turn. Keep the choice-aligned + # channel present without inventing reasoning when Claude skips it. + reasoning = (None,) + return Response( + texts=(text,), + reasoning=reasoning, + finish_reasons=(stop_reason,), + tool_calls=tuple(tool_calls) or None, + structured_output=_structured_output(context.request, text), + usage=usage, + request_params=context.request_params, + response_model=response_model, + ) + + async def execute(self, prepared: PreparedRequest) -> Response: + if prepared.operation != "messages.create": + raise DialectError(f"unexpected Anthropic operation {prepared.operation!r}") + context = prepared.context + if not isinstance(context, _AnthropicContext): + raise DialectError("prepared Anthropic request has invalid context") + body, stream, execution = _execution_context( + prepared, + context, + self._requested_model_id, + ) + headers = _headers(self._connection) + if not stream: + response = await self._connection.client.post( + _MESSAGES_PATH, + json=body, + headers=headers, + ) + response.raise_for_status() + return self._lift(_parse_json_response(response), execution) + + stream_headers = {**headers, "accept": "text/event-stream"} + async with self._connection.client.stream( + "POST", + _MESSAGES_PATH, + json=body, + headers=stream_headers, + ) as response: + if not response.is_success: + await response.aread() + response.raise_for_status() + terminal = await _terminal_stream_message(response) + return self._lift(terminal, execution) + + async def arun(self, req: Request) -> Response: + """One-cycle direct path; canonical ``Model`` uses the split contract.""" + + plan = _LegacyPlan() + validate_request_invariants(req) + validate_runtime_binding_plan(plan, req) + audit = RequestAudit(active_request_leaves(req)) + self.validate_request(req, audit, plan) + audit.raise_rejections() + prepared = self.prepare(req, audit) + audit.finish(prepared) + response = await self.execute(prepared) + self.output_contract.validate(plan, req, response) + return response diff --git a/sieval/core/utils/meta.py b/sieval/core/utils/meta.py index f09cc711..01d962e3 100644 --- a/sieval/core/utils/meta.py +++ b/sieval/core/utils/meta.py @@ -15,13 +15,19 @@ #: Every spelling, not one canonical name: the IR does not normalize these, so a #: set holding only ``length`` (OpenAI-compatible Chat/Completions and SGLang #: native) would read zero against ``max_output_tokens`` (OpenAI Responses) or -#: ``max_tokens`` (Anthropic). +#: ``max_tokens`` or ``model_context_window_exceeded`` (Anthropic). #: ``content_filter`` is in because the output is cut short the same way; #: separating causes is what the raw ``finish_reasons`` are for. Shared with #: :func:`sieval.core.tasks.anomaly.detect_truncated_output` so the rule and the #: report key cannot drift apart. TRUNCATION_FINISH_REASONS = frozenset( - {"length", "max_output_tokens", "max_tokens", "content_filter"} + { + "length", + "max_output_tokens", + "max_tokens", + "model_context_window_exceeded", + "content_filter", + } ) diff --git a/tests/unit/cli/leaderboard/test_session.py b/tests/unit/cli/leaderboard/test_session.py index dd71a918..5b19803a 100644 --- a/tests/unit/cli/leaderboard/test_session.py +++ b/tests/unit/cli/leaderboard/test_session.py @@ -6865,12 +6865,14 @@ def test_present_decision_replaces_existing_argument_seed(self): assert args == {"seed": 0, "temperature": 0.5} - def test_reserved_seed_policy_fails_loudly(self): - with pytest.raises(ValueError, match="anthropic_messages.*has not declared"): - _resolve_deterministic_request_seed( - dialect_id="anthropic_messages", - explicit_seed_present=False, - ) + def test_anthropic_automatic_request_seed_is_explicitly_unsupported(self): + decision = _resolve_deterministic_request_seed( + dialect_id="anthropic_messages", + explicit_seed_present=False, + ) + + assert decision.seed_present is False + assert decision.support is RequestSeedSupport.UNSUPPORTED @pytest.mark.parametrize("seed", [None, 0, 42]) def test_explicit_seed_is_never_rewritten(self, seed): diff --git a/tests/unit/cli/test_validation.py b/tests/unit/cli/test_validation.py index bf9c6004..674f762e 100644 --- a/tests/unit/cli/test_validation.py +++ b/tests/unit/cli/test_validation.py @@ -461,10 +461,25 @@ def test_active_canonical_dialects_and_typed_options_are_accepted(self): }, ) ) + anthropic = validate_eval_config( + self._config( + dialect="anthropic_messages", + capabilities={ + "reasoning": { + "budget_tokens": 2048, + "summary": "auto", + }, + "structured_output": {"formats": ["json_schema"]}, + "multimodal_input": {"modalities": ["image"]}, + "input_scoring": False, + }, + ) + ) assert chat.ok, chat.errors assert completions.ok, completions.errors assert responses.ok, responses.errors + assert anthropic.ok, anthropic.errors @pytest.mark.parametrize("dialect", [None, "", 7, ["openai_chat"]]) def test_explicit_malformed_dialect_is_rejected(self, dialect): @@ -1657,6 +1672,43 @@ def test_active_responses_dialect_reconciles_to_its_runtime_plan(self, tmp_path) assert len(binding_plans) == 1 assert next(iter(binding_plans.values()))["dialect_id"] == "openai_responses" + def test_active_anthropic_dialect_reconciles_to_its_runtime_plan(self, tmp_path): + """Dry-run carries native Messages through schema and reconciliation.""" + from sieval.cli.validation import run_dry_run + + config = tmp_path / "anthropic-messages.yaml" + config.write_text( + "models:\n" + " m:\n" + " name: claude-sonnet-4-6\n" + " type: chat\n" + " dialect: anthropic_messages\n" + " api_base: https://api.anthropic.com/v1\n" + "datasets:\n" + " d:\n" + " class: sieval.datasets.aime_2024.AIME2024Dataset\n" + "tasks:\n" + " t:\n" + " class: sieval.tasks.aime_2024_0shot_gen.AIME2024ZeroShotGenTask\n" + " dataset: d\n" + " model: m\n", + encoding="utf-8", + ) + content = config.read_text(encoding="utf-8") + + direct = validate_eval_config(yaml.safe_load(content)) + dry_run = run_dry_run(config) + checks = {check["name"]: check for check in dry_run["checks"]} + + assert direct.ok, direct.errors + assert checks["schema"]["ok"] is True + assert checks["capability_reconcile"]["ok"] is True + binding_plans = checks["capability_reconcile"]["plan"]["binding_plans"] + assert len(binding_plans) == 1 + assert next(iter(binding_plans.values()))["dialect_id"] == ( + "anthropic_messages" + ) + def test_schema_validation_failure(self, tmp_path): """Config with bad schema returns schema check failure.""" from sieval.cli.validation import run_dry_run diff --git a/tests/unit/core/models/dialects/test_anthropic_messages.py b/tests/unit/core/models/dialects/test_anthropic_messages.py new file mode 100644 index 00000000..9f0fe3fc --- /dev/null +++ b/tests/unit/core/models/dialects/test_anthropic_messages.py @@ -0,0 +1,3938 @@ +"""Tests for the native Anthropic Messages dialect. + +AI-Generated Code - GPT-5.6 (OpenAI) +""" + +import json +from collections.abc import AsyncIterator, Callable, Mapping +from typing import Any, cast +from unittest.mock import AsyncMock + +import httpx +import pytest + +import sieval.core.models.dialects.anthropic_messages as anthropic_messages_module +from sieval.core.models.capabilities import ( + CAPABILITY_KEYS, + DialectCapabilityStatus, + MultimodalInputOptions, + ReasoningOptions, + StructuredOutputOptions, + Supported, + Unsupported, +) +from sieval.core.models.connection_factory import AsyncHTTPJSONConnection +from sieval.core.models.dialect import ( + DialectError, + OutputContractError, + PreparedRequest, + Rejected, + RequestAudit, + RequestAuditError, + active_request_leaves, +) +from sieval.core.models.dialect_registry import DIALECT_SPECS +from sieval.core.models.dialects.anthropic_messages import ( + CAPABILITY_DECISIONS, + AnthropicMessagesDialect, + _AnthropicInputVerifier, + _AnthropicToolChoiceVerifier, + _AnthropicToolsVerifier, + _decode_continuation, + _finish_stream_block, + _LegacyPlan, + _lower_chat_input, + _part_to_wire, + _ReasoningSummaryVerifier, + _tool_history_state, + _validate_history_block, + _validate_replay_system, + _validate_replay_tools, +) +from sieval.core.models.ir import ( + ChatInput, + ChatMessage, + CompletionInput, + DialectOptions, + HostedToolSpec, + ImagePart, + OpaqueContinuation, + ReasoningParams, + Request, + SamplingParams, + SchedulingParams, + ScoringParams, + SessionParams, + StructuredOutputParams, + TextPart, + ToolCallPart, + ToolParams, + ToolResultPart, +) + + +def _chat(*messages: ChatMessage) -> ChatInput: + return ChatInput(messages or (ChatMessage("user", (TextPart("Hello"),)),)) + + +def _request(**changes: object) -> Request: + values: dict[str, object] = { + "input": _chat(), + "sampling": SamplingParams(max_tokens=128), + } + values.update(changes) + return Request(**cast(Any, values)) + + +def _response( + content: list[dict[str, object]] | None = None, + *, + stop_reason: object = "end_turn", + usage: object | None = None, + model: object = "claude-sonnet-4-6", +) -> dict[str, object]: + return { + "id": "msg_123", + "type": "message", + "role": "assistant", + "content": content if content is not None else [{"type": "text", "text": "Hi"}], + "model": model, + "stop_reason": stop_reason, + "stop_sequence": None, + "usage": usage + if usage is not None + else { + "input_tokens": 7, + "output_tokens": 3, + }, + } + + +def _connection( + handler: Callable[[httpx.Request], httpx.Response], + *, + credential: str | None = "anthropic-secret", +) -> AsyncHTTPJSONConnection: + return AsyncHTTPJSONConnection( + httpx.AsyncClient( + base_url="https://api.anthropic.com/v1/", + transport=httpx.MockTransport(handler), + ), + credential, + ) + + +async def _run( + handler: Callable[[httpx.Request], httpx.Response], + request: Request | None = None, + *, + credential: str | None = "anthropic-secret", +): + connection = _connection(handler, credential=credential) + try: + dialect = AnthropicMessagesDialect(connection, "claude-sonnet-4-6") + return await dialect.arun(request or _request()) + finally: + await connection.aclose() + + +def _sse(*events: Mapping[str, object]) -> bytes: + lines: list[str] = [] + for event in events: + event_type = cast(str, event["type"]) + lines.extend( + ( + f"event: {event_type}", + "data: " + json.dumps(event, separators=(",", ":")), + "", + ) + ) + return "\n".join(lines).encode() + + +def _stream_start(*, usage: Mapping[str, object] | None = None) -> dict[str, object]: + return { + "type": "message_start", + "message": { + **_response(content=[], stop_reason=None, usage=usage), + "stop_sequence": None, + }, + } + + +def _stream_delta( + *, + stop_reason: str = "end_turn", + usage: Mapping[str, object] | None = None, + stop_details: Mapping[str, object] | None = None, +) -> dict[str, object]: + delta: dict[str, object] = { + "stop_reason": stop_reason, + "stop_sequence": None, + } + if stop_details is not None: + delta["stop_details"] = stop_details + return { + "type": "message_delta", + "delta": delta, + "usage": usage if usage is not None else {"output_tokens": 3}, + } + + +class _UnbufferedAsyncByteStream(httpx.AsyncByteStream): + def __init__(self, content: bytes) -> None: + self._content = content + + async def __aiter__(self) -> AsyncIterator[bytes]: + yield self._content + + +class TestCapabilities: + def test_decision_row_is_complete_and_logprobs_are_explicitly_absent(self) -> None: + assert set(CAPABILITY_DECISIONS) == set(CAPABILITY_KEYS) + assert isinstance(CAPABILITY_DECISIONS["input_scoring"], Unsupported) + assert isinstance(CAPABILITY_DECISIONS["sampled_logprobs"], Unsupported) + assert isinstance(CAPABILITY_DECISIONS["top_logprobs"], Unsupported) + assert "does not return" in CAPABILITY_DECISIONS["sampled_logprobs"].reason + + def test_supported_capabilities_are_first_class_registry_outcomes(self) -> None: + spec = DIALECT_SPECS["anthropic_messages"] + + assert spec.implementation_status.value == "active" + assert spec.request_seed_support.value == "unsupported" + for key in ( + "reasoning", + "function_tools", + "structured_output", + "opaque_continuation", + "multimodal_input", + "prefill", + ): + assert isinstance(CAPABILITY_DECISIONS[key], Supported) + assert spec.capability_outcomes[key] is DialectCapabilityStatus.SUPPORTED + + @pytest.mark.parametrize( + ("key", "options", "message"), + [ + ("reasoning", ReasoningOptions(effort="minimal"), "effort"), + ("reasoning", ReasoningOptions(budget_tokens=100), "1024"), + ("reasoning", ReasoningOptions(summary="detailed"), "summary"), + ( + "structured_output", + StructuredOutputOptions(("json_object",)), + "json_schema", + ), + ( + "multimodal_input", + cast(Any, MultimodalInputOptions.__new__(MultimodalInputOptions)), + "modalities", + ), + ], + ) + def test_capability_domains_are_dialect_validated( + self, key: str, options: object, message: str + ) -> None: + decision = cast(Supported, CAPABILITY_DECISIONS[cast(Any, key)]) + if key == "multimodal_input": + object.__setattr__(options, "modalities", ("audio",)) + + with pytest.raises((TypeError, ValueError), match=message): + decision.binding.validate_config(cast(Any, options)) + + +class TestLoweringAndLift: + @pytest.mark.anyio + async def test_native_nonstream_request_uses_headers_system_and_final_body( + self, + ) -> None: + seen: dict[str, object] = {} + + def handler(request: httpx.Request) -> httpx.Response: + seen["url"] = str(request.url) + seen["headers"] = dict(request.headers) + seen["body"] = json.loads(request.content) + return httpx.Response(200, json=_response()) + + req = _request( + input=_chat( + ChatMessage("system", (TextPart("Be concise."),)), + ChatMessage("user", (TextPart("Hello"),)), + ), + sampling=SamplingParams( + max_tokens=64, + temperature=0.2, + top_p=0.9, + top_k=40, + stop=("END",), + ), + ) + result = await _run(handler, req) + + assert seen["url"] == "https://api.anthropic.com/v1/messages" + headers = cast(dict[str, str], seen["headers"]) + assert headers["x-api-key"] == "anthropic-secret" + assert headers["anthropic-version"] == "2023-06-01" + assert seen["body"] == { + "model": "claude-sonnet-4-6", + "max_tokens": 64, + "messages": [ + { + "role": "user", + "content": [{"type": "text", "text": "Hello"}], + } + ], + "stream": False, + "system": [{"type": "text", "text": "Be concise."}], + "temperature": 0.2, + "top_p": 0.9, + "top_k": 40, + "stop_sequences": ["END"], + } + assert result.texts == ("Hi",) + assert result.finish_reasons == ("end_turn",) + assert result.response_model == "claude-sonnet-4-6" + assert result.request_params == { + "max_tokens": 64, + "stream": False, + "temperature": 0.2, + "top_p": 0.9, + "top_k": 40, + "stop_sequences": ["END"], + } + + @pytest.mark.anyio + async def test_missing_credential_does_not_fabricate_auth_header(self) -> None: + seen_headers: dict[str, str] = {} + + def handler(request: httpx.Request) -> httpx.Response: + seen_headers.update(request.headers) + return httpx.Response(200, json=_response()) + + await _run(handler, credential=None) + + assert "x-api-key" not in seen_headers + assert seen_headers["anthropic-version"] == "2023-06-01" + + @pytest.mark.anyio + async def test_usage_includes_cache_partitions_and_reasoning_breakdown( + self, + ) -> None: + usage = { + "input_tokens": 5, + "cache_creation_input_tokens": 7, + "cache_read_input_tokens": 11, + "output_tokens": 13, + "output_tokens_details": {"thinking_tokens": 3}, + "service_tier": "standard", + } + + result = await _run(lambda _: httpx.Response(200, json=_response(usage=usage))) + + assert result.usage is not None + assert result.usage.input_tokens == 23 + assert result.usage.output_tokens == 13 + assert result.usage.total_tokens == 36 + assert result.usage.cached_tokens == 11 + assert result.usage.reasoning_tokens == 3 + + @pytest.mark.anyio + async def test_context_window_limit_reason_is_preserved_verbatim(self) -> None: + result = await _run( + lambda _: httpx.Response( + 200, + json=_response(stop_reason="model_context_window_exceeded"), + ) + ) + + assert result.finish_reasons == ("model_context_window_exceeded",) + + @pytest.mark.anyio + async def test_zero_max_tokens_is_preserved_for_cache_warming(self) -> None: + seen: dict[str, object] = {} + + def handler(request: httpx.Request) -> httpx.Response: + seen.update(json.loads(request.content)) + return httpx.Response( + 200, + json=_response( + [], + stop_reason="max_tokens", + usage={"input_tokens": 7, "output_tokens": 0}, + ), + ) + + result = await _run( + handler, + _request(sampling=SamplingParams(max_tokens=0)), + ) + + assert seen["max_tokens"] == 0 + assert result.texts == ("",) + assert result.usage is not None + assert result.usage.output_tokens == 0 + + @pytest.mark.anyio + async def test_image_and_tool_history_lower_without_reordering(self) -> None: + seen: dict[str, object] = {} + + def handler(request: httpx.Request) -> httpx.Response: + seen.update(json.loads(request.content)) + return httpx.Response(200, json=_response()) + + req = _request( + input=_chat( + ChatMessage( + "user", + ( + TextPart("look"), + ImagePart( + data="aGVsbG8=", + media_type="image/png", + ), + ), + ), + ChatMessage( + "assistant", + (ToolCallPart("toolu_1", "weather", {"city": "Paris"}),), + ), + ChatMessage( + "tool", + (ToolResultPart("toolu_1", {"temp": 20}, is_error=True),), + ), + ) + ) + + await _run(handler, req) + + assert seen["messages"] == [ + { + "role": "user", + "content": [ + {"type": "text", "text": "look"}, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "aGVsbG8=", + }, + }, + ], + }, + { + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "toolu_1", + "name": "weather", + "input": {"city": "Paris"}, + } + ], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_1", + "content": '{"temp":20}', + "is_error": True, + } + ], + }, + ] + + @pytest.mark.anyio + async def test_supported_prefill_lowers_final_assistant_message(self) -> None: + decision = CAPABILITY_DECISIONS["prefill"] + assert isinstance(decision, Supported) + + seen: dict[str, object] = {} + + def handler(request: httpx.Request) -> httpx.Response: + seen.update(json.loads(request.content)) + return httpx.Response(200, json=_response()) + + await _run( + handler, + _request( + input=_chat( + ChatMessage("user", (TextPart("Complete this"),)), + ChatMessage("assistant", (TextPart("Partial answer"),)), + ) + ), + ) + + assert cast(list[object], seen["messages"])[-1] == { + "role": "assistant", + "content": [{"type": "text", "text": "Partial answer"}], + } + + @pytest.mark.anyio + async def test_functions_choice_parallel_and_tool_use_lift(self) -> None: + seen: dict[str, object] = {} + + def handler(request: httpx.Request) -> httpx.Response: + seen.update(json.loads(request.content)) + return httpx.Response( + 200, + json=_response( + [ + {"type": "text", "text": "Checking"}, + { + "type": "tool_use", + "id": "toolu_2", + "name": "weather", + "input": {"city": "Tokyo"}, + }, + ], + stop_reason="tool_use", + ), + ) + + req = _request( + tools=ToolParams( + functions=( + { + "type": "function", + "function": { + "name": "weather", + "description": "Get weather", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + }, + "strict": True, + }, + }, + ), + choice={"type": "function", "function": {"name": "weather"}}, + parallel=False, + ) + ) + + result = await _run(handler, req) + + assert seen["tools"] == [ + { + "name": "weather", + "description": "Get weather", + "input_schema": { + "type": "object", + "properties": {"city": {"type": "string"}}, + }, + "strict": True, + } + ] + assert seen["tool_choice"] == { + "type": "tool", + "name": "weather", + "disable_parallel_tool_use": True, + } + assert result.finish_reasons == ("tool_use",) + assert result.tool_calls is not None + assert result.tool_calls[0].call_id == "toolu_2" + assert result.tool_calls[0].arguments == {"city": "Tokyo"} + + @pytest.mark.anyio + async def test_reasoning_budget_summary_and_opaque_history(self) -> None: + seen: dict[str, object] = {} + + def handler(request: httpx.Request) -> httpx.Response: + seen.update(json.loads(request.content)) + return httpx.Response( + 200, + json=_response( + [ + { + "type": "thinking", + "thinking": "I should answer briefly.", + "signature": "signed-state", + }, + {"type": "text", "text": "42"}, + ], + usage={ + "input_tokens": 8, + "output_tokens": 9, + "output_tokens_details": {"thinking_tokens": 6}, + }, + ), + ) + + req = _request( + sampling=SamplingParams(max_tokens=4096), + reasoning=ReasoningParams(budget_tokens=2048, summary="auto"), + ) + result = await _run(handler, req) + + assert seen["thinking"] == { + "type": "enabled", + "budget_tokens": 2048, + "display": "summarized", + } + assert result.reasoning is not None + assert result.reasoning[0] is not None + assert result.reasoning[0].text == "I should answer briefly." + assert result.reasoning[0].thinking_tokens == 6 + opaque = result.reasoning[0].opaque_roundtrip + assert opaque is not None + bundle = json.loads(opaque) + assert bundle["messages"][-1]["content"][0]["signature"] == "signed-state" + + @pytest.mark.anyio + async def test_opaque_continuation_replays_complete_history_untouched(self) -> None: + first = await _run( + lambda _: httpx.Response( + 200, + json=_response( + [ + { + "type": "thinking", + "thinking": "secret reasoning", + "signature": "sig-1", + }, + { + "type": "text", + "text": "first answer", + "citations": [], + }, + ] + ), + ), + _request( + input=_chat( + ChatMessage("system", (TextPart("Stable system"),)), + ChatMessage("user", (TextPart("first question"),)), + ), + sampling=SamplingParams(max_tokens=4096), + reasoning=ReasoningParams(budget_tokens=2048), + ), + ) + assert first.reasoning is not None and first.reasoning[0] is not None + opaque = cast(str, first.reasoning[0].opaque_roundtrip) + seen: dict[str, object] = {} + + def second_handler(request: httpx.Request) -> httpx.Response: + seen.update(json.loads(request.content)) + return httpx.Response( + 200, + json=_response( + [ + { + "type": "redacted_thinking", + "data": "encrypted-next", + }, + {"type": "text", "text": "second answer"}, + ] + ), + ) + + second = await _run( + second_handler, + _request( + input=_chat(ChatMessage("user", (TextPart("follow up"),))), + session=SessionParams( + opaque_continuation=OpaqueContinuation("anthropic_messages", opaque) + ), + ), + ) + + assert seen["system"] == [{"type": "text", "text": "Stable system"}] + assert cast(list[object], seen["messages"])[-2] == { + "role": "assistant", + "content": [ + { + "type": "thinking", + "thinking": "secret reasoning", + "signature": "sig-1", + }, + {"type": "text", "text": "first answer", "citations": []}, + ], + } + assert cast(list[object], seen["messages"])[-1] == { + "role": "user", + "content": [{"type": "text", "text": "follow up"}], + } + assert second.reasoning is not None + assert second.reasoning[0] is not None + assert second.reasoning[0].opaque_roundtrip is not None + + @pytest.mark.anyio + async def test_continuation_remains_replayable_without_new_thinking_block( + self, + ) -> None: + opaque = json.dumps( + { + "version": 2, + "system": [], + "messages": [ + { + "role": "assistant", + "content": [ + { + "type": "thinking", + "thinking": "prior", + "signature": "prior-signature", + }, + {"type": "text", "text": "prior answer"}, + ], + } + ], + "tools": None, + "open_turn": False, + "reasoning": {"thinking": None, "effort": None}, + } + ) + result = await _run( + lambda _: httpx.Response( + 200, + json=_response([{"type": "text", "text": "next answer"}]), + ), + _request( + session=SessionParams( + opaque_continuation=OpaqueContinuation("anthropic_messages", opaque) + ) + ), + ) + + assert result.reasoning is not None and result.reasoning[0] is not None + next_opaque = cast(str, result.reasoning[0].opaque_roundtrip) + next_history = json.loads(next_opaque)["messages"] + assert next_history[-1] == { + "role": "assistant", + "content": [{"type": "text", "text": "next answer"}], + } + + @pytest.mark.anyio + async def test_json_schema_output_uses_output_config_and_parses_null(self) -> None: + seen: dict[str, object] = {} + + def handler(request: httpx.Request) -> httpx.Response: + seen.update(json.loads(request.content)) + return httpx.Response( + 200, + json=_response([{"type": "text", "text": "null"}]), + ) + + req = _request( + structured_output=StructuredOutputParams( + format="json_schema", + schema={"type": ["object", "null"]}, + strict=True, + ) + ) + result = await _run(handler, req) + + assert seen["output_config"] == { + "format": { + "type": "json_schema", + "schema": {"type": ["object", "null"]}, + } + } + assert result.structured_output is not None + assert result.structured_output.value is None + + @pytest.mark.anyio + async def test_effort_and_schema_share_one_output_config(self) -> None: + seen: dict[str, object] = {} + + def handler(request: httpx.Request) -> httpx.Response: + seen.update(json.loads(request.content)) + return httpx.Response( + 200, + json=_response([{"type": "text", "text": '{"ok":true}'}]), + ) + + await _run( + handler, + _request( + reasoning=ReasoningParams(effort="medium"), + structured_output=StructuredOutputParams( + format="json_schema", + schema={"type": "object"}, + ), + ), + ) + + assert seen["output_config"] == { + "effort": "medium", + "format": {"type": "json_schema", "schema": {"type": "object"}}, + } + + @pytest.mark.anyio + async def test_summary_none_explicitly_requests_omitted_adaptive_thinking( + self, + ) -> None: + seen: dict[str, object] = {} + + def handler(request: httpx.Request) -> httpx.Response: + seen.update(json.loads(request.content)) + return httpx.Response(200, json=_response()) + + await _run(handler, _request(reasoning=ReasoningParams(summary="none"))) + + assert seen["thinking"] == {"type": "adaptive", "display": "omitted"} + + @pytest.mark.anyio + async def test_open_tool_turn_requires_exact_prior_reasoning_configuration( + self, + ) -> None: + first = await _run( + lambda _: httpx.Response( + 200, + json=_response( + [ + { + "type": "thinking", + "thinking": "choose a tool", + "signature": "signed-tool-turn", + }, + { + "type": "tool_use", + "id": "toolu_1", + "name": "lookup", + "input": {"key": "x"}, + }, + ], + stop_reason="tool_use", + ), + ), + _request( + sampling=SamplingParams(max_tokens=2048), + reasoning=ReasoningParams(budget_tokens=1024, summary="none"), + ), + ) + assert first.reasoning is not None and first.reasoning[0] is not None + opaque = cast(str, first.reasoning[0].opaque_roundtrip) + bundle = json.loads(opaque) + assert bundle["open_turn"] is True + assert bundle["reasoning"] == { + "thinking": { + "type": "enabled", + "budget_tokens": 1024, + "display": "omitted", + }, + "effort": None, + } + + calls = 0 + + def rejected_handler(_: httpx.Request) -> httpx.Response: + nonlocal calls + calls += 1 + return httpx.Response(200, json=_response()) + + with pytest.raises(RequestAuditError, match="reuse the exact thinking"): + await _run( + rejected_handler, + _request( + sampling=SamplingParams(max_tokens=2048), + input=_chat( + ChatMessage("tool", (ToolResultPart("toolu_1", {"value": 1}),)) + ), + session=SessionParams( + opaque_continuation=OpaqueContinuation( + "anthropic_messages", opaque + ) + ), + ), + ) + assert calls == 0 + + seen: dict[str, object] = {} + + def accepted_handler(request: httpx.Request) -> httpx.Response: + seen.update(json.loads(request.content)) + return httpx.Response(200, json=_response()) + + await _run( + accepted_handler, + _request( + sampling=SamplingParams(max_tokens=2048), + input=_chat( + ChatMessage("tool", (ToolResultPart("toolu_1", {"value": 1}),)) + ), + reasoning=ReasoningParams(budget_tokens=1024, summary="none"), + session=SessionParams( + opaque_continuation=OpaqueContinuation("anthropic_messages", opaque) + ), + ), + ) + assert seen["thinking"] == { + "type": "enabled", + "budget_tokens": 1024, + "display": "omitted", + } + + @pytest.mark.anyio + async def test_open_tool_turn_preserves_reasoning_config_without_new_thinking( + self, + ) -> None: + result = await _run( + lambda _: httpx.Response( + 200, + json=_response( + [ + { + "type": "tool_use", + "id": "toolu_1", + "name": "lookup", + "input": {}, + } + ], + stop_reason="tool_use", + ), + ), + _request(reasoning=ReasoningParams(summary="none")), + ) + + assert result.reasoning is not None and result.reasoning[0] is not None + bundle = json.loads(cast(str, result.reasoning[0].opaque_roundtrip)) + assert bundle["open_turn"] is True + assert bundle["reasoning"] == { + "thinking": {"type": "adaptive", "display": "omitted"}, + "effort": None, + } + + @pytest.mark.anyio + async def test_continuation_requires_exact_final_wire_tools(self) -> None: + functions = ( + { + "type": "function", + "name": "lookup", + "description": "Look up a value", + "parameters": {"type": "object"}, + }, + { + "type": "function", + "name": "calculate", + "parameters": {"type": "object"}, + }, + ) + first = await _run( + lambda _: httpx.Response( + 200, + json=_response( + [ + { + "type": "tool_use", + "id": "toolu_1", + "name": "lookup", + "input": {}, + } + ], + stop_reason="tool_use", + ), + ), + _request( + reasoning=ReasoningParams(summary="none"), + tools=ToolParams(functions=functions), + ), + ) + assert first.reasoning is not None and first.reasoning[0] is not None + opaque = cast(str, first.reasoning[0].opaque_roundtrip) + assert json.loads(opaque)["tools"] == [ + { + "name": "lookup", + "description": "Look up a value", + "input_schema": {"type": "object"}, + }, + { + "name": "calculate", + "input_schema": {"type": "object"}, + }, + ] + + follow_up = _request( + input=_chat( + ChatMessage("tool", (ToolResultPart("toolu_1", {"value": 1}),)) + ), + reasoning=ReasoningParams(summary="none"), + tools=ToolParams(functions=functions), + session=SessionParams( + opaque_continuation=OpaqueContinuation("anthropic_messages", opaque) + ), + ) + await _run( + lambda _: httpx.Response(200, json=_response()), + follow_up, + ) + + changed = dict(functions[0]) + changed["description"] = "Changed" + mismatches = ( + ToolParams(), + ToolParams(functions=(changed, functions[1])), + ToolParams(functions=tuple(reversed(functions))), + ) + for tools in mismatches: + calls = 0 + + def rejected_handler(_: httpx.Request) -> httpx.Response: + nonlocal calls + calls += 1 + return httpx.Response(200, json=_response()) + + with pytest.raises(RequestAuditError, match="must exactly match"): + await _run( + rejected_handler, + _request( + input=follow_up.input, + reasoning=follow_up.reasoning, + tools=tools, + session=follow_up.session, + ), + ) + assert calls == 0 + + +class TestStreaming: + @pytest.mark.anyio + async def test_streaming_reconstructs_text_thinking_signature_and_tool_json( + self, + ) -> None: + content = _sse( + _stream_start(usage={"input_tokens": 7, "output_tokens": 1}), + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "thinking", "thinking": "", "signature": ""}, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "thinking_delta", "thinking": "think"}, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "signature_delta", "signature": "sig"}, + }, + {"type": "content_block_stop", "index": 0}, + { + "type": "content_block_start", + "index": 1, + "content_block": {"type": "text", "text": ""}, + }, + { + "type": "content_block_delta", + "index": 1, + "delta": {"type": "text_delta", "text": "answer"}, + }, + {"type": "content_block_stop", "index": 1}, + { + "type": "content_block_start", + "index": 2, + "content_block": { + "type": "tool_use", + "id": "toolu_stream", + "name": "lookup", + "input": {}, + }, + }, + { + "type": "content_block_delta", + "index": 2, + "delta": {"type": "input_json_delta", "partial_json": '{"x"'}, + }, + { + "type": "content_block_delta", + "index": 2, + "delta": {"type": "input_json_delta", "partial_json": ":1}"}, + }, + {"type": "content_block_stop", "index": 2}, + _stream_delta( + stop_reason="tool_use", + usage={ + "output_tokens": 12, + "output_tokens_details": {"thinking_tokens": 4}, + }, + ), + {"type": "message_stop"}, + ) + + result = await _run( + lambda _: httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + content=content, + ), + _request( + sampling=SamplingParams(max_tokens=4096), + reasoning=ReasoningParams(budget_tokens=2048, summary="auto"), + scheduling=SchedulingParams(stream=True), + ), + ) + + assert result.texts == ("answer",) + assert result.finish_reasons == ("tool_use",) + assert result.reasoning is not None and result.reasoning[0] is not None + assert result.reasoning[0].text == "think" + assert result.reasoning[0].thinking_tokens == 4 + assert result.tool_calls is not None + assert result.tool_calls[0].arguments == {"x": 1} + assert result.usage is not None + assert result.usage.input_tokens == 7 + assert result.usage.output_tokens == 12 + + @pytest.mark.anyio + async def test_streaming_refusal_is_a_scoreable_terminal_outcome(self) -> None: + content = _sse( + _stream_start(), + _stream_delta( + stop_reason="refusal", + stop_details={ + "type": "refusal", + "category": "safety", + "explanation": "Request refused", + }, + ), + {"type": "message_stop"}, + ) + + result = await _run( + lambda _: httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + content=content, + ), + _request( + reasoning=ReasoningParams(summary="auto"), + scheduling=SchedulingParams(stream=True), + ), + ) + + assert result.texts == ("",) + assert result.finish_reasons == ("refusal",) + assert result.reasoning is None + + @pytest.mark.anyio + async def test_streaming_refusal_discards_unfinished_partial_output(self) -> None: + content = _sse( + _stream_start(), + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "text", "text": ""}, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "text_delta", "text": "partial"}, + }, + _stream_delta( + stop_reason="refusal", + stop_details={"type": "refusal"}, + ), + {"type": "message_stop"}, + ) + + result = await _run( + lambda _: httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + content=content, + ), + _request(scheduling=SchedulingParams(stream=True)), + ) + + assert result.texts == ("",) + assert result.finish_reasons == ("refusal",) + + @pytest.mark.anyio + async def test_streaming_refusal_discards_completed_partial_text(self) -> None: + content = _sse( + _stream_start(), + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "text", "text": ""}, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "text_delta", "text": "completed partial"}, + }, + {"type": "content_block_stop", "index": 0}, + _stream_delta( + stop_reason="refusal", + stop_details={"type": "refusal"}, + ), + {"type": "message_stop"}, + ) + + result = await _run( + lambda _: httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + content=content, + ), + _request(scheduling=SchedulingParams(stream=True)), + ) + + assert result.texts == ("",) + assert result.finish_reasons == ("refusal",) + + @pytest.mark.anyio + @pytest.mark.parametrize( + ("events", "message"), + [ + ((_stream_delta(), {"type": "message_stop"}), "preceded message_start"), + ((_stream_start(), {"type": "message_stop"}), "omitted message_delta"), + ((_stream_start(), _stream_delta()), "omitted message_stop"), + ( + ( + _stream_start(), + {"type": "error", "error": {"message": "overloaded"}}, + ), + "overloaded", + ), + ( + ( + _stream_start(), + _stream_delta(), + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "text", "text": ""}, + }, + {"type": "message_stop"}, + ), + "content after message_delta", + ), + ( + ( + _stream_start(), + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "text", "text": ""}, + }, + _stream_delta(), + {"type": "message_stop"}, + ), + "preceded content_block_stop", + ), + ( + ( + _stream_start(), + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "text", "text": ""}, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": { + "type": "text_delta", + "text": "x", + "future": True, + }, + }, + ), + "unsupported field", + ), + (({"type": "ping", "future": True},), "unsupported field"), + ( + ( + _stream_start(usage={"input_tokens": 7, "output_tokens": 1}), + _stream_delta(usage={}), + {"type": "message_stop"}, + ), + "message_delta.usage.output_tokens", + ), + ( + ( + _stream_start(usage={"input_tokens": 7, "output_tokens": 1}), + _stream_delta(usage={"output_tokens": None}), + {"type": "message_stop"}, + ), + "message_delta.usage.output_tokens", + ), + ], + ) + async def test_malformed_streams_fail_loudly( + self, events: tuple[Mapping[str, object], ...], message: str + ) -> None: + with pytest.raises(OutputContractError, match=message): + await _run( + lambda _: httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + content=_sse(*events), + ), + _request(scheduling=SchedulingParams(stream=True)), + ) + + +class TestPreIOGuards: + @pytest.mark.anyio + @pytest.mark.parametrize( + ("req", "message"), + [ + (Request(_chat()), "requires sampling.max_tokens"), + ( + _request(input=_chat(ChatMessage("developer", (TextPart("hidden"),)))), + "developer-role", + ), + ( + _request(input=_chat(ChatMessage("system", (TextPart("rules"),)))), + "at least one conversational message", + ), + ( + _request( + input=_chat( + ChatMessage("user", (TextPart("u"),)), + ChatMessage("system", (TextPart("late"),)), + ) + ), + "must precede", + ), + ( + _request( + input=_chat( + ChatMessage("user", (TextPart(cast(Any, 7)),)), + ) + ), + r"content\[0\]\.text must be a string", + ), + ( + _request( + input=_chat(ChatMessage("assistant", (ImagePart(url="https://x"),))) + ), + "assistant messages", + ), + ( + _request( + input=_chat( + ChatMessage("user", (ImagePart(data="x", media_type=None),)) + ) + ), + "require media_type", + ), + ( + _request(sampling=SamplingParams(max_tokens=10, seed=1)), + "per-request seed", + ), + ( + _request(sampling=SamplingParams(max_tokens=10, n=2)), + "one choice", + ), + ( + _request(sampling=SamplingParams(max_tokens=10, temperature=1.1)), + "between 0 and 1", + ), + ( + _request( + sampling=SamplingParams(max_tokens=2048), + reasoning=ReasoningParams(budget_tokens=2048), + ), + "max_tokens must exceed reasoning.budget_tokens", + ), + ( + _request( + sampling=SamplingParams(max_tokens=2048, temperature=0.5), + reasoning=ReasoningParams(budget_tokens=1024), + ), + "temperature is incompatible", + ), + ( + _request( + sampling=SamplingParams(max_tokens=2048, top_p=0.9), + reasoning=ReasoningParams(budget_tokens=1024), + ), + "between 0.95 and 1", + ), + ( + _request( + sampling=SamplingParams(max_tokens=2048, top_k=20), + reasoning=ReasoningParams(budget_tokens=1024), + ), + "top_k is incompatible", + ), + ( + _request(reasoning=ReasoningParams(summary="detailed")), + "only summary", + ), + ( + _request(tools=ToolParams(choice="required")), + "require function tools", + ), + ( + _request( + sampling=SamplingParams(max_tokens=2048), + reasoning=ReasoningParams(budget_tokens=1024), + tools=ToolParams( + functions=( + { + "type": "function", + "name": "lookup", + "parameters": {"type": "object"}, + }, + ), + choice="required", + ), + ), + "manual Anthropic thinking supports only", + ), + ( + _request(tools=ToolParams(hosted=(HostedToolSpec("web_search"),))), + "unavailable capability|not active", + ), + ( + _request( + structured_output=StructuredOutputParams(format="json_object") + ), + "requires json_schema", + ), + ( + _request( + structured_output=StructuredOutputParams( + format="json_schema", + schema={"type": "object"}, + name="named", + ) + ), + "no schema name", + ), + ( + _request( + input=_chat( + ChatMessage("user", (TextPart("complete"),)), + ChatMessage("assistant", (TextPart("{"),)), + ), + structured_output=StructuredOutputParams( + format="json_schema", schema={"type": "object"} + ), + ), + "incompatible with assistant prefill", + ), + ( + _request( + input=_chat( + ChatMessage("user", (TextPart("continue"),)), + ChatMessage("assistant", (TextPart("partial"),)), + ), + sampling=SamplingParams(max_tokens=2048), + reasoning=ReasoningParams(budget_tokens=1024), + ), + "thinking is incompatible with assistant prefill", + ), + ( + _request(session=SessionParams(previous_response_id="msg_previous")), + "unavailable capability|no previous-response-id", + ), + ( + _request( + dialect_options=DialectOptions( + "anthropic_messages", {"service_tier": "auto"} + ) + ), + "no raw Anthropic request passthrough", + ), + ( + _request( + session=SessionParams( + opaque_continuation=OpaqueContinuation( + "anthropic_messages", '{"version":1}' + ) + ) + ), + "invalid envelope", + ), + ], + ) + async def test_invalid_requests_make_zero_http_calls( + self, req: Request, message: str + ) -> None: + calls = 0 + + def handler(_: httpx.Request) -> httpx.Response: + nonlocal calls + calls += 1 + return httpx.Response(200, json=_response()) + + with pytest.raises( + (DialectError, RequestAuditError, ValueError), match=message + ): + await _run(handler, req) + + assert calls == 0 + + @pytest.mark.anyio + async def test_continuation_cannot_be_combined_with_new_system_authority( + self, + ) -> None: + opaque = json.dumps( + { + "version": 2, + "system": [], + "messages": [ + { + "role": "assistant", + "content": [ + { + "type": "thinking", + "thinking": "x", + "signature": "sig", + } + ], + } + ], + "tools": None, + "open_turn": False, + "reasoning": {"thinking": None, "effort": None}, + } + ) + req = _request( + input=_chat( + ChatMessage("system", (TextPart("new"),)), + ChatMessage("user", (TextPart("next"),)), + ), + session=SessionParams( + opaque_continuation=OpaqueContinuation("anthropic_messages", opaque) + ), + ) + + with pytest.raises(RequestAuditError, match="authority ambiguous"): + await _run( + lambda _: pytest.fail("HTTP must not run"), + req, + ) + + @pytest.mark.anyio + @pytest.mark.parametrize( + ("tools", "message"), + [ + ([], "null or non-empty"), + (["not-an-object"], "tools are malformed"), + ( + [{"name": "lookup", "input_schema": {}, "future": True}], + "unsupported fields", + ), + ], + ) + async def test_malformed_continuation_tools_fail_before_http( + self, tools: object, message: str + ) -> None: + opaque = json.dumps( + { + "version": 2, + "system": [], + "messages": [ + { + "role": "assistant", + "content": [ + { + "type": "thinking", + "thinking": "prior", + "signature": "signature", + } + ], + } + ], + "tools": tools, + "open_turn": False, + "reasoning": {"thinking": None, "effort": None}, + } + ) + + with pytest.raises(DialectError, match=message): + await _run( + lambda _: pytest.fail("HTTP must not run"), + _request( + session=SessionParams( + opaque_continuation=OpaqueContinuation( + "anthropic_messages", opaque + ) + ) + ), + ) + + +class TestResponseGuards: + @pytest.mark.anyio + @pytest.mark.parametrize( + ("response", "message"), + [ + ({**_response(), "type": "completion"}, "response.type"), + ({**_response(), "role": "user"}, "response.role"), + ({**_response(), "model": None}, "response.model"), + ({**_response(), "stop_reason": "unknown"}, "unsupported"), + ({**_response(), "future_semantic": {}}, "unsupported field"), + ( + {**_response(), "container": {"id": "container_1"}}, + "container has no shared-IR mapping", + ), + ( + {**_response(), "stop_reason": "refusal"}, + "response.stop_details must be an object", + ), + ( + { + **_response(), + "stop_reason": "refusal", + "stop_details": {"type": "future"}, + }, + "stop_details.type must be 'refusal'", + ), + ( + { + **_response(), + "stop_reason": "refusal", + "stop_details": {"type": "refusal", "category": 1}, + }, + "stop_details.category must be a string or null", + ), + ( + { + **_response(), + "stop_reason": "refusal", + "stop_details": {"type": "refusal", "future": True}, + }, + "stop_details has unsupported field", + ), + ( + { + **_response(), + "stop_details": {"type": "refusal"}, + }, + "stop_details must be null unless stop_reason is 'refusal'", + ), + ( + {**_response(), "stop_sequence": "unexpected"}, + "must be null", + ), + ({**_response(), "usage": None}, "response.usage"), + ( + _response( + usage={ + "input_tokens": 1, + "output_tokens": 1, + "server_tool_use": { + "web_fetch_requests": 0, + "web_search_requests": 1, + }, + } + ), + "server-tool usage", + ), + ( + _response( + usage={ + "input_tokens": 1, + "output_tokens": 1, + "cache_creation_input_tokens": 3, + "cache_creation": { + "ephemeral_1h_input_tokens": 1, + "ephemeral_5m_input_tokens": 1, + }, + } + ), + "breakdown disagrees", + ), + ( + _response([{"type": "server_tool_use", "id": "x"}]), + "no shared-IR mapping", + ), + ( + _response( + [ + { + "type": "tool_use", + "id": "toolu_1", + "name": "lookup", + "input": {}, + "caller": None, + } + ], + stop_reason="tool_use", + ), + "delegated/toolset", + ), + ( + _response( + [ + { + "type": "thinking", + "thinking": "x", + "signature": "", + } + ] + ), + "signature", + ), + ( + _response( + [ + { + "type": "text", + "text": "x", + "citations": [{"type": "page_location"}], + } + ] + ), + "citations", + ), + ], + ) + async def test_malformed_native_response_fails_lift( + self, response: Mapping[str, object], message: str + ) -> None: + with pytest.raises(OutputContractError, match=message): + await _run(lambda _: httpx.Response(200, json=response)) + + @pytest.mark.anyio + async def test_refusal_is_a_scoreable_terminal_outcome(self) -> None: + req = _request( + reasoning=ReasoningParams(summary="auto"), + structured_output=StructuredOutputParams( + format="json_schema", schema={"type": "object"} + ), + ) + response = { + **_response( + [ + {"type": "thinking", "thinking": "partial", "signature": "sig"}, + {"type": "text", "text": "partial answer"}, + ], + stop_reason="refusal", + ), + "stop_details": { + "type": "refusal", + "category": "reasoning_extraction", + "explanation": None, + }, + } + + result = await _run(lambda _: httpx.Response(200, json=response), req) + + assert result.texts == ("",) + assert result.finish_reasons == ("refusal",) + assert result.reasoning is None + assert result.structured_output is None + assert result.tool_calls is None + assert result.usage is not None + + @pytest.mark.anyio + @pytest.mark.parametrize( + ("content", "message"), + [ + ([{"type": "quantum_widget"}], "has no shared-IR mapping"), + ([{"type": "text", "text": 5}], "must be a string"), + (["not an object"], "must be an object"), + ], + ) + async def test_refusal_content_blocks_are_still_validated( + self, content: list[object], message: str + ) -> None: + """Discarding a refusal's content does not mean skipping its checks.""" + response = { + **_response(cast(Any, content), stop_reason="refusal"), + "stop_details": {"type": "refusal"}, + } + + with pytest.raises(OutputContractError, match=message): + await _run(lambda _: httpx.Response(200, json=response)) + + @pytest.mark.anyio + async def test_adaptive_summary_allows_a_turn_without_thinking(self) -> None: + req = _request(reasoning=ReasoningParams(summary="auto")) + + result = await _run( + lambda _: httpx.Response(200, json=_response()), + req, + ) + + assert result.texts == ("Hi",) + assert result.reasoning == (None,) + + @pytest.mark.anyio + async def test_low_effort_adaptive_summary_allows_no_thinking(self) -> None: + req = _request(reasoning=ReasoningParams(effort="low", summary="auto")) + + result = await _run( + lambda _: httpx.Response(200, json=_response()), + req, + ) + + assert result.reasoning == (None,) + + @pytest.mark.anyio + @pytest.mark.parametrize( + "stop_reason", ["max_tokens", "model_context_window_exceeded"] + ) + async def test_truncated_adaptive_summary_allows_no_thinking( + self, stop_reason: str + ) -> None: + req = _request(reasoning=ReasoningParams(summary="auto")) + + result = await _run( + lambda _: httpx.Response(200, json=_response(stop_reason=stop_reason)), + req, + ) + + assert result.finish_reasons == (stop_reason,) + assert result.reasoning == (None,) + + @pytest.mark.anyio + async def test_truncated_adaptive_summary_allows_incomplete_thinking(self) -> None: + req = _request(reasoning=ReasoningParams(summary="auto")) + + result = await _run( + lambda _: httpx.Response( + 200, + json=_response( + [{"type": "thinking", "thinking": " ", "signature": "sig"}], + stop_reason="max_tokens", + ), + ), + req, + ) + + assert result.finish_reasons == ("max_tokens",) + assert result.reasoning is not None + assert result.reasoning[0] is not None + assert result.reasoning[0].text == " " + + @pytest.mark.anyio + async def test_adaptive_summary_accepts_redacted_thinking(self) -> None: + req = _request(reasoning=ReasoningParams(summary="auto")) + + result = await _run( + lambda _: httpx.Response( + 200, + json=_response( + [ + {"type": "redacted_thinking", "data": "opaque"}, + {"type": "text", "text": "answer"}, + ] + ), + ), + req, + ) + + assert result.texts == ("answer",) + assert result.reasoning is not None + assert result.reasoning[0] is not None + assert result.reasoning[0].text is None + assert result.reasoning[0].opaque_roundtrip is not None + + @pytest.mark.anyio + async def test_manual_thinking_requires_replayable_thinking_state(self) -> None: + req = _request( + sampling=SamplingParams(max_tokens=2048), + reasoning=ReasoningParams(budget_tokens=1024), + ) + + with pytest.raises(OutputContractError, match="manual-thinking state"): + await _run(lambda _: httpx.Response(200, json=_response()), req) + + @pytest.mark.anyio + @pytest.mark.parametrize( + ("response", "message"), + [ + ( + _response(stop_reason="tool_use"), + "requires a tool_use content block", + ), + ( + _response( + [ + { + "type": "tool_use", + "id": "toolu_1", + "name": "lookup", + "input": {}, + } + ] + ), + "requires stop_reason='tool_use'", + ), + ( + _response(stop_reason="pause_turn"), + "pause_turn.*unsupported", + ), + ], + ) + async def test_stop_reason_and_content_state_must_agree( + self, response: Mapping[str, object], message: str + ) -> None: + with pytest.raises(OutputContractError, match=message): + await _run(lambda _: httpx.Response(200, json=response)) + + @pytest.mark.anyio + async def test_structured_output_rejects_non_json_text(self) -> None: + req = _request( + structured_output=StructuredOutputParams( + format="json_schema", schema={"type": "object"} + ) + ) + + with pytest.raises(OutputContractError, match="not valid JSON"): + await _run( + lambda _: httpx.Response( + 200, + json=_response([{"type": "text", "text": "not-json"}]), + ), + req, + ) + + @pytest.mark.anyio + async def test_http_error_is_not_lifted_as_success(self) -> None: + with pytest.raises(httpx.HTTPStatusError): + await _run( + lambda _: httpx.Response( + 401, + json={"type": "error", "error": {"message": "bad key"}}, + ) + ) + + @pytest.mark.anyio + async def test_stream_http_error_is_raised_before_sse_lift( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + lift = AsyncMock(side_effect=AssertionError("SSE lift must not run")) + monkeypatch.setattr( + anthropic_messages_module, + "_terminal_stream_message", + lift, + ) + + with pytest.raises(httpx.HTTPStatusError) as exc_info: + await _run( + lambda _: httpx.Response( + 429, + stream=_UnbufferedAsyncByteStream( + b'{"type":"error","error":{"type":"overloaded_error"}}' + ), + headers={"content-type": "text/event-stream"}, + ), + _request(scheduling=SchedulingParams(stream=True)), + ) + + assert exc_info.value.response.status_code == 429 + assert "overloaded_error" in exc_info.value.response.text + lift.assert_not_awaited() + + +class TestAuditAndLowLevelContract: + def test_low_level_system_lowering_rejects_non_text_content(self) -> None: + input_ = _chat( + ChatMessage("system", (ImagePart(url="https://example.com/image.png"),)), + ChatMessage("user", (TextPart("Hello"),)), + ) + + with pytest.raises(DialectError, match="system accepts text content only"): + _lower_chat_input(input_) + + def test_low_level_text_lowering_rejects_non_string_content(self) -> None: + with pytest.raises(DialectError, match="text content must be a string"): + _part_to_wire(TextPart(cast(Any, 7))) + + def test_budget_conflict_is_attributed_to_sampling_max_tokens(self) -> None: + connection = _connection(lambda _: httpx.Response(200, json=_response())) + dialect = AnthropicMessagesDialect(connection, "claude") + req = _request( + sampling=SamplingParams(max_tokens=2048), + reasoning=ReasoningParams(budget_tokens=2048), + ) + audit = RequestAudit(active_request_leaves(req)) + + dialect.validate_request(req, audit, _LegacyPlan()) + + decision = audit.decisions["sampling.max_tokens"] + assert isinstance(decision, Rejected) + assert decision.reason == ( + "Anthropic max_tokens must exceed reasoning.budget_tokens" + ) + assert "reasoning.budget_tokens" not in audit.decisions + + def test_mutating_lowered_input_is_detected_by_independent_verifier(self) -> None: + connection = _connection(lambda _: httpx.Response(200, json=_response())) + dialect = AnthropicMessagesDialect(connection, "claude") + req = _request() + audit = RequestAudit(active_request_leaves(req)) + dialect.validate_request(req, audit, cast(Any, object())) + prepared = dialect.prepare(req, audit) + body = prepared.thaw_body() + messages = cast(list[dict[str, object]], body["messages"]) + cast(list[dict[str, object]], messages[0]["content"])[0]["text"] = "changed" + mutated = PreparedRequest(prepared.operation, body, prepared.context) + + with pytest.raises(RequestAuditError, match="body.messages changed"): + audit.finish(mutated) + + def test_mutating_lowered_tools_is_detected_by_independent_verifier(self) -> None: + connection = _connection(lambda _: httpx.Response(200, json=_response())) + dialect = AnthropicMessagesDialect(connection, "claude") + req = _request( + tools=ToolParams( + functions=( + { + "type": "function", + "name": "lookup", + "parameters": {"type": "object"}, + }, + ) + ) + ) + audit = RequestAudit(active_request_leaves(req)) + dialect.validate_request(req, audit, cast(Any, object())) + prepared = dialect.prepare(req, audit) + body = prepared.thaw_body() + tools = cast(list[dict[str, object]], body["tools"]) + tools[0]["name"] = "changed" + + with pytest.raises(RequestAuditError, match=r"body.tools\[0\] changed"): + audit.finish(PreparedRequest(prepared.operation, body, prepared.context)) + + def test_mutating_lowered_tool_choice_is_detected_by_independent_verifier( + self, + ) -> None: + connection = _connection(lambda _: httpx.Response(200, json=_response())) + dialect = AnthropicMessagesDialect(connection, "claude") + req = _request( + tools=ToolParams( + functions=( + { + "type": "function", + "name": "lookup", + "parameters": {"type": "object"}, + }, + ), + choice="auto", + parallel=False, + ) + ) + audit = RequestAudit(active_request_leaves(req)) + dialect.validate_request(req, audit, cast(Any, object())) + prepared = dialect.prepare(req, audit) + body = prepared.thaw_body() + choice = cast(dict[str, object], body["tool_choice"]) + choice["disable_parallel_tool_use"] = False + + with pytest.raises(RequestAuditError, match="body.tool_choice changed"): + audit.finish(PreparedRequest(prepared.operation, body, prepared.context)) + + @pytest.mark.anyio + async def test_request_params_are_derived_from_actual_prepared_body(self) -> None: + connection = _connection(lambda _: httpx.Response(200, json=_response())) + try: + dialect = AnthropicMessagesDialect(connection, "claude") + req = _request(sampling=SamplingParams(max_tokens=10, temperature=0.2)) + audit = RequestAudit(active_request_leaves(req)) + dialect.validate_request(req, audit, cast(Any, object())) + prepared = dialect.prepare(req, audit) + body = prepared.thaw_body() + body["temperature"] = 0.9 + final = PreparedRequest(prepared.operation, body, prepared.context) + + response = await dialect.execute(final) + + assert response.request_params is not None + assert response.request_params["temperature"] == 0.9 + finally: + await connection.aclose() + + @pytest.mark.anyio + async def test_opaque_bundle_is_credential_stable_and_tool_sensitive(self) -> None: + async def opaque_for( + credential: str, + description: str, + ) -> str: + connection = _connection( + lambda _: httpx.Response( + 200, + json=_response( + [ + { + "type": "redacted_thinking", + "data": "stable-opaque-state", + }, + {"type": "text", "text": "answer"}, + ] + ), + ), + credential=credential, + ) + try: + dialect = AnthropicMessagesDialect(connection, "claude") + response = await dialect.arun( + _request( + reasoning=ReasoningParams(summary="none"), + tools=ToolParams( + functions=( + { + "type": "function", + "name": "lookup", + "description": description, + "parameters": {"type": "object"}, + }, + ) + ), + ) + ) + assert response.reasoning is not None + assert response.reasoning[0] is not None + return cast(str, response.reasoning[0].opaque_roundtrip) + finally: + await connection.aclose() + + first = await opaque_for("SECRET-A", "Stable description") + second = await opaque_for("SECRET-B", "Stable description") + changed = await opaque_for("SECRET-A", "Changed description") + + assert first == second + assert first != changed + assert "SECRET-A" not in first + assert "SECRET-B" not in second + + @pytest.mark.anyio + async def test_execute_rejects_wrong_operation_and_context_before_http( + self, + ) -> None: + connection = _connection(lambda _: pytest.fail("HTTP must not run")) + try: + dialect = AnthropicMessagesDialect(connection, "claude") + with pytest.raises(DialectError, match="unexpected Anthropic operation"): + await dialect.execute( + PreparedRequest("responses.create", {"stream": False}) + ) + with pytest.raises(DialectError, match="invalid context"): + await dialect.execute( + PreparedRequest("messages.create", {"stream": False}) + ) + finally: + await connection.aclose() + + @pytest.mark.anyio + @pytest.mark.parametrize("model", [None, 7, "", "other-model"]) + async def test_execute_rejects_invalid_prepared_model_before_http( + self, model: object + ) -> None: + connection = _connection(lambda _: pytest.fail("HTTP must not run")) + try: + dialect = AnthropicMessagesDialect(connection, "claude") + req = _request() + audit = RequestAudit(active_request_leaves(req)) + dialect.validate_request(req, audit, _LegacyPlan()) + audit.raise_rejections() + prepared = dialect.prepare(req, audit) + audit.finish(prepared) + body = prepared.thaw_body() + if model is None: + body.pop("model") + else: + body["model"] = cast(Any, model) + + with pytest.raises(DialectError, match="prepared body.*model"): + await dialect.execute( + PreparedRequest(prepared.operation, body, prepared.context) + ) + finally: + await connection.aclose() + + @pytest.mark.anyio + async def test_execute_revalidates_continuation_tools_after_audit(self) -> None: + functions = ( + { + "type": "function", + "name": "lookup", + "parameters": {"type": "object"}, + }, + ) + first = await _run( + lambda _: httpx.Response( + 200, + json=_response( + [ + { + "type": "tool_use", + "id": "toolu_1", + "name": "lookup", + "input": {}, + } + ], + stop_reason="tool_use", + ), + ), + _request( + reasoning=ReasoningParams(summary="none"), + tools=ToolParams(functions=functions), + ), + ) + assert first.reasoning is not None and first.reasoning[0] is not None + opaque = cast(str, first.reasoning[0].opaque_roundtrip) + req = _request( + input=_chat(ChatMessage("tool", (ToolResultPart("toolu_1", "ok"),))), + reasoning=ReasoningParams(summary="none"), + tools=ToolParams(functions=functions), + session=SessionParams( + opaque_continuation=OpaqueContinuation("anthropic_messages", opaque) + ), + ) + calls = 0 + + def handler(_: httpx.Request) -> httpx.Response: + nonlocal calls + calls += 1 + return httpx.Response(200, json=_response()) + + connection = _connection(handler) + try: + dialect = AnthropicMessagesDialect(connection, "claude") + audit = RequestAudit(active_request_leaves(req)) + dialect.validate_request(req, audit, cast(Any, object())) + prepared = dialect.prepare(req, audit) + audit.finish(prepared) + body = prepared.thaw_body() + cast(list[dict[str, object]], body["tools"])[0]["name"] = "changed" + + with pytest.raises(DialectError, match="must exactly match"): + await dialect.execute( + PreparedRequest(prepared.operation, body, prepared.context) + ) + finally: + await connection.aclose() + + assert calls == 0 + + def test_constructor_requires_right_connection_and_model(self) -> None: + with pytest.raises(TypeError, match="AsyncHTTPJSONConnection"): + AnthropicMessagesDialect(cast(Any, object()), "claude") + connection = _connection(lambda _: httpx.Response(200, json=_response())) + with pytest.raises(ValueError, match="requested_model_id"): + AnthropicMessagesDialect(connection, "") + + +class TestProviderContractRegressions: + @pytest.mark.anyio + @pytest.mark.parametrize( + ("input_", "message"), + [ + ( + _chat( + ChatMessage( + "user", + (TextPart("result follows"), ToolResultPart("toolu_1", "ok")), + ) + ), + "must precede", + ), + ( + _chat( + ChatMessage( + "assistant", + (ToolCallPart("toolu_1", "lookup", {}),), + ), + ChatMessage("tool", (ToolResultPart("wrong", "ok"),)), + ), + "must exactly match", + ), + ( + _chat( + ChatMessage( + "assistant", + ( + ToolCallPart("toolu_1", "lookup", {}), + ToolCallPart("toolu_1", "lookup", {}), + ), + ), + ChatMessage("tool", (ToolResultPart("toolu_1", "ok"),)), + ), + "duplicates a tool_use id", + ), + ], + ) + async def test_invalid_tool_history_is_rejected_before_http( + self, input_: ChatInput, message: str + ) -> None: + calls = 0 + + def handler(_: httpx.Request) -> httpx.Response: + nonlocal calls + calls += 1 + return httpx.Response(200, json=_response()) + + with pytest.raises(RequestAuditError, match=message): + await _run(handler, _request(input=input_)) + + assert calls == 0 + + @pytest.mark.anyio + async def test_continuation_rejects_a_second_result_for_answered_call(self) -> None: + opaque = json.dumps( + { + "version": 2, + "system": [], + "messages": [ + { + "role": "assistant", + "content": [ + { + "type": "thinking", + "thinking": "call once", + "signature": "sig", + }, + { + "type": "tool_use", + "id": "toolu_1", + "name": "lookup", + "input": {}, + }, + ], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_1", + "content": "done", + "is_error": False, + } + ], + }, + { + "role": "assistant", + "content": [{"type": "text", "text": "finished"}], + }, + ], + "tools": None, + "open_turn": False, + "reasoning": {"thinking": None, "effort": None}, + } + ) + req = _request( + input=_chat(ChatMessage("tool", (ToolResultPart("toolu_1", "again"),))), + session=SessionParams( + opaque_continuation=OpaqueContinuation("anthropic_messages", opaque) + ), + ) + + with pytest.raises(RequestAuditError, match="do not immediately follow"): + await _run(lambda _: pytest.fail("HTTP must not run"), req) + + @pytest.mark.anyio + async def test_empty_end_turn_emits_a_reusable_continuation(self) -> None: + opaque = json.dumps( + { + "version": 2, + "system": [], + "messages": [ + { + "role": "user", + "content": [{"type": "text", "text": "Use the tool"}], + }, + { + "role": "assistant", + "content": [ + { + "type": "thinking", + "thinking": "call once", + "signature": "sig", + }, + { + "type": "tool_use", + "id": "toolu_1", + "name": "lookup", + "input": {}, + }, + ], + }, + ], + "tools": [{"name": "lookup", "input_schema": {"type": "object"}}], + "open_turn": True, + "reasoning": {"thinking": None, "effort": None}, + } + ) + functions = ( + { + "type": "function", + "name": "lookup", + "parameters": {"type": "object"}, + }, + ) + first = await _run( + lambda _: httpx.Response(200, json=_response(content=[])), + _request( + input=_chat(ChatMessage("tool", (ToolResultPart("toolu_1", "done"),))), + tools=ToolParams(functions=functions), + session=SessionParams( + opaque_continuation=OpaqueContinuation("anthropic_messages", opaque) + ), + ), + ) + assert first.reasoning is not None and first.reasoning[0] is not None + continued = cast(str, first.reasoning[0].opaque_roundtrip) + seen: dict[str, object] = {} + + def handler(request: httpx.Request) -> httpx.Response: + seen.update(json.loads(request.content)) + return httpx.Response(200, json=_response()) + + second = await _run( + handler, + _request( + input=_chat(ChatMessage("user", (TextPart("Continue"),))), + session=SessionParams( + opaque_continuation=OpaqueContinuation( + "anthropic_messages", continued + ) + ), + ), + ) + + assert second.texts == ("Hi",) + messages = cast(list[dict[str, object]], seen["messages"]) + assert all(message["content"] != [] for message in messages) + assert [message["role"] for message in messages[-2:]] == ["user", "user"] + + @pytest.mark.anyio + async def test_closed_continuation_may_use_new_tool_definitions(self) -> None: + opaque = json.dumps( + { + "version": 2, + "system": [], + "messages": [ + { + "role": "assistant", + "content": [ + { + "type": "thinking", + "thinking": "finished turn", + "signature": "sig", + }, + {"type": "text", "text": "done"}, + ], + } + ], + "tools": [{"name": "old", "input_schema": {"type": "object"}}], + "open_turn": False, + "reasoning": {"thinking": None, "effort": None}, + } + ) + seen: dict[str, object] = {} + + def handler(request: httpx.Request) -> httpx.Response: + seen.update(json.loads(request.content)) + return httpx.Response(200, json=_response()) + + await _run( + handler, + _request( + session=SessionParams( + opaque_continuation=OpaqueContinuation("anthropic_messages", opaque) + ), + tools=ToolParams( + functions=( + { + "type": "function", + "name": "new", + "parameters": {"type": "object"}, + }, + ) + ), + ), + ) + + assert cast(list[dict[str, object]], seen["tools"])[0]["name"] == "new" + + @pytest.mark.anyio + async def test_manual_thinking_response_must_start_with_thinking(self) -> None: + req = _request( + sampling=SamplingParams(max_tokens=2048), + reasoning=ReasoningParams(budget_tokens=1024), + ) + + with pytest.raises(OutputContractError, match="must begin"): + await _run( + lambda _: httpx.Response( + 200, + json=_response( + [ + {"type": "text", "text": "premature"}, + { + "type": "thinking", + "thinking": "late", + "signature": "sig", + }, + ] + ), + ), + req, + ) + + @pytest.mark.anyio + async def test_visible_summary_rejects_whitespace_only_thinking(self) -> None: + req = _request(reasoning=ReasoningParams(summary="auto")) + + with pytest.raises(OutputContractError, match="visible thinking summary"): + await _run( + lambda _: httpx.Response( + 200, + json=_response( + [ + { + "type": "thinking", + "thinking": " ", + "signature": "sig", + }, + {"type": "text", "text": "answer"}, + ] + ), + ), + req, + ) + + @pytest.mark.anyio + async def test_response_rejects_duplicate_tool_use_ids(self) -> None: + with pytest.raises(OutputContractError, match="duplicated a tool_use id"): + await _run( + lambda _: httpx.Response( + 200, + json=_response( + [ + { + "type": "tool_use", + "id": "toolu_1", + "name": "one", + "input": {}, + }, + { + "type": "tool_use", + "id": "toolu_1", + "name": "two", + "input": {}, + }, + ], + stop_reason="tool_use", + ), + ) + ) + + @pytest.mark.anyio + async def test_response_rejects_tool_use_id_reused_from_prior_turn(self) -> None: + opaque = json.dumps( + { + "version": 2, + "system": [], + "messages": [ + { + "role": "assistant", + "content": [ + { + "type": "thinking", + "thinking": "call once", + "signature": "sig", + }, + { + "type": "tool_use", + "id": "toolu_1", + "name": "lookup", + "input": {}, + }, + ], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_1", + "content": "done", + "is_error": False, + } + ], + }, + ], + "tools": None, + "open_turn": False, + "reasoning": {"thinking": None, "effort": None}, + } + ) + req = _request( + tools=ToolParams( + functions=( + { + "type": "function", + "name": "lookup", + "parameters": {"type": "object"}, + }, + ) + ), + session=SessionParams( + opaque_continuation=OpaqueContinuation("anthropic_messages", opaque) + ), + ) + + with pytest.raises(OutputContractError, match="duplicates a tool_use id"): + await _run( + lambda _: httpx.Response( + 200, + json=_response( + [ + { + "type": "tool_use", + "id": "toolu_1", + "name": "lookup", + "input": {}, + } + ], + stop_reason="tool_use", + ), + ), + req, + ) + + @pytest.mark.anyio + async def test_pause_turn_fails_as_unrepresentable(self) -> None: + with pytest.raises(OutputContractError, match="pause_turn.*unsupported"): + await _run( + lambda _: httpx.Response( + 200, + json=_response(stop_reason="pause_turn"), + ) + ) + + @pytest.mark.anyio + async def test_cache_creation_breakdown_supplies_missing_total(self) -> None: + result = await _run( + lambda _: httpx.Response( + 200, + json=_response( + usage={ + "input_tokens": 2, + "output_tokens": 3, + "cache_creation": { + "ephemeral_1h_input_tokens": 5, + "ephemeral_5m_input_tokens": 7, + }, + } + ), + ) + ) + + assert result.usage is not None + assert result.usage.input_tokens == 14 + assert result.usage.total_tokens == 17 + + @pytest.mark.anyio + async def test_reasoning_breakdown_cannot_exceed_inclusive_output(self) -> None: + with pytest.raises(OutputContractError, match="exceeds inclusive"): + await _run( + lambda _: httpx.Response( + 200, + json=_response( + usage={ + "input_tokens": 2, + "output_tokens": 3, + "output_tokens_details": {"thinking_tokens": 4}, + } + ), + ) + ) + + @pytest.mark.anyio + @pytest.mark.parametrize( + "body", + [ + b'{"id":"first","id":"second","type":"message"}', + b'{"id":"msg","type":"message","role":"assistant",' + b'"content":[],"model":"claude","stop_reason":"end_turn",' + b'"stop_sequence":null,"usage":{"input_tokens":NaN,' + b'"output_tokens":1}}', + ], + ) + async def test_native_response_json_must_be_unambiguous(self, body: bytes) -> None: + with pytest.raises(OutputContractError, match="not valid JSON"): + await _run(lambda _: httpx.Response(200, content=body)) + + @pytest.mark.anyio + async def test_structured_output_rejects_duplicate_json_keys(self) -> None: + req = _request( + structured_output=StructuredOutputParams( + format="json_schema", schema={"type": "object"} + ) + ) + + with pytest.raises(OutputContractError, match="not valid JSON"): + await _run( + lambda _: httpx.Response( + 200, + json=_response([{"type": "text", "text": '{"value":1,"value":2}'}]), + ), + req, + ) + + @pytest.mark.anyio + async def test_continuation_rejects_duplicate_envelope_keys(self) -> None: + opaque = ( + '{"version":2,"version":2,"system":[],"messages":[],' + '"tools":null,"open_turn":false,' + '"reasoning":{"thinking":null,"effort":null}}' + ) + + with pytest.raises(DialectError, match="not valid JSON"): + await _run( + lambda _: pytest.fail("HTTP must not run"), + _request( + session=SessionParams( + opaque_continuation=OpaqueContinuation( + "anthropic_messages", opaque + ) + ) + ), + ) + + @pytest.mark.anyio + async def test_streamed_tool_json_rejects_duplicate_keys(self) -> None: + payload = _sse( + _stream_start(), + { + "type": "content_block_start", + "index": 0, + "content_block": { + "type": "tool_use", + "id": "toolu_1", + "name": "lookup", + "input": {}, + }, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": { + "type": "input_json_delta", + "partial_json": '{"x":1,"x":2}', + }, + }, + {"type": "content_block_stop", "index": 0}, + _stream_delta(stop_reason="tool_use"), + {"type": "message_stop"}, + ) + + with pytest.raises(OutputContractError, match="input is not valid JSON"): + await _run( + lambda _: httpx.Response( + 200, + content=payload, + headers={"content-type": "text/event-stream"}, + ), + _request(scheduling=SchedulingParams(stream=True)), + ) + + +class TestReachableInputBoundaries: + @pytest.mark.anyio + @pytest.mark.parametrize( + ("req", "message"), + [ + (_request(input=CompletionInput("prompt")), "requires ChatInput"), + ( + _request(input=_chat(ChatMessage("user", ()))), + "empty content", + ), + ( + _request( + input=_chat(ChatMessage("user", (TextPart("x"),), name="alice")) + ), + "no message name", + ), + ( + _request( + input=_chat( + ChatMessage("system", (cast(Any, ImagePart(url="x")),)), + ChatMessage("user", (TextPart("x"),)), + ) + ), + "system accepts text", + ), + ( + _request(input=_chat(ChatMessage("tool", (TextPart("x"),)))), + "tool-role messages", + ), + ( + _request( + input=_chat(ChatMessage("user", (ToolCallPart("id", "tool", {}),))) + ), + "tool_use blocks require assistant", + ), + ( + _request( + input=_chat(ChatMessage("assistant", (ToolResultPart("id", "x"),))) + ), + "assistant messages", + ), + ( + _request( + input=_chat( + ChatMessage("assistant", (ToolCallPart("", "tool", {}),)) + ) + ), + "non-empty id and name", + ), + ( + _request( + input=_chat(ChatMessage("assistant", (ToolCallPart("id", "", {}),))) + ), + "non-empty id and name", + ), + ( + _request( + input=_chat( + ChatMessage("assistant", (ToolCallPart("id", "tool", [1]),)) + ) + ), + "JSON object", + ), + ( + _request(input=_chat(ChatMessage("tool", (ToolResultPart("", "x"),)))), + "non-empty tool_use_id", + ), + ( + _request(input=_chat(ChatMessage("user", (ImagePart(url=""),)))), + "must not be empty", + ), + ( + _request( + input=_chat( + ChatMessage("user", (ImagePart(url="x", detail="high"),)) + ) + ), + "image detail", + ), + ( + _request( + input=_chat( + ChatMessage( + "user", (ImagePart(url="x", media_type="image/png"),) + ) + ) + ), + "URL image sources", + ), + ( + _request( + input=_chat( + ChatMessage( + "user", (ImagePart(data="x", media_type="image/bmp"),) + ) + ) + ), + "base64 images require one of", + ), + ( + _request(sampling=SamplingParams(max_tokens=-1)), + "max_tokens must be non-negative", + ), + ( + _request(sampling=SamplingParams(max_tokens=10, top_p=-0.1)), + "top_p must be between", + ), + ( + _request(sampling=SamplingParams(max_tokens=10, top_k=-1)), + "top_k must be non-negative", + ), + ( + _request(sampling=SamplingParams(max_tokens=10, stop=("",))), + "stop sequences must not be empty", + ), + ( + _request(sampling=SamplingParams(max_tokens=10, frequency_penalty=0.1)), + "no equivalent penalty", + ), + ( + _request(sampling=SamplingParams(max_tokens=10, presence_penalty=0.1)), + "no equivalent penalty", + ), + ( + _request(reasoning=ReasoningParams(effort="minimal")), + "unsupported Anthropic reasoning effort", + ), + ( + _request( + sampling=SamplingParams(max_tokens=2048), + reasoning=ReasoningParams(budget_tokens=512), + ), + "budget must be at least 1024", + ), + ( + _request( + structured_output=StructuredOutputParams( + format="json_schema", schema={}, strict=False + ) + ), + "always schema constrained", + ), + ( + _request(dialect_options=DialectOptions("anthropic_messages", {"": 1})), + "keys must be non-empty", + ), + ( + _request( + dialect_options=DialectOptions( + "anthropic_messages", {"max_tokens": 1} + ) + ), + "canonical provider-neutral field", + ), + ], + ) + async def test_invalid_request_leaf_is_rejected_before_http( + self, req: Request, message: str + ) -> None: + calls = 0 + + def handler(_: httpx.Request) -> httpx.Response: + nonlocal calls + calls += 1 + return httpx.Response(200, json=_response()) + + with pytest.raises( + (DialectError, RequestAuditError, ValueError), match=message + ): + await _run(handler, req) + + assert calls == 0 + + @pytest.mark.parametrize( + "scoring", + [ + ScoringParams(input_scoring=True), + ScoringParams(sampled_logprobs=True), + ScoringParams(sampled_logprobs=True, top_logprobs=2), + ], + ) + def test_low_level_dialect_validation_also_rejects_logprob_requests( + self, scoring: ScoringParams + ) -> None: + connection = _connection(lambda _: pytest.fail("HTTP must not run")) + dialect = AnthropicMessagesDialect(connection, "claude") + req = _request(scoring=scoring) + audit = RequestAudit(active_request_leaves(req)) + + dialect.validate_request(req, audit, _LegacyPlan()) + + with pytest.raises(RequestAuditError, match="does not provide logprobs"): + audit.raise_rejections() + + @pytest.mark.anyio + @pytest.mark.parametrize( + ("tools", "message"), + [ + ( + ToolParams(functions=({"type": "hosted", "name": "x"},)), + "require type='function'", + ), + ( + ToolParams(functions=({"type": "function", "function": "wrong"},)), + "invalid Chat-shaped", + ), + ( + ToolParams( + functions=( + { + "type": "function", + "name": "x", + "parameters": {}, + "future": True, + }, + ) + ), + "unsupported function tool field", + ), + ( + ToolParams( + functions=({"type": "function", "name": "", "parameters": {}},) + ), + "non-empty string", + ), + ( + ToolParams( + functions=({"type": "function", "name": "x", "parameters": []},) + ), + "parameters must be a JSON object", + ), + ( + ToolParams( + functions=( + { + "type": "function", + "name": "x", + "parameters": {}, + "description": 1, + }, + ) + ), + "description must be a string", + ), + ( + ToolParams( + functions=( + { + "type": "function", + "name": "x", + "parameters": {}, + "strict": "yes", + }, + ) + ), + "strict must be a boolean", + ), + ( + ToolParams( + functions=({"type": "function", "name": "x", "parameters": {}},), + choice="sometimes", + ), + "unsupported Anthropic tool choice", + ), + ( + ToolParams( + functions=({"type": "function", "name": "x", "parameters": {}},), + choice=1, + ), + "string or mapping", + ), + ( + ToolParams( + functions=({"type": "function", "name": "x", "parameters": {}},), + choice={"type": "hosted", "name": "x"}, + ), + "only function-specific", + ), + ( + ToolParams( + functions=({"type": "function", "name": "x", "parameters": {}},), + choice={"type": "function", "function": "wrong"}, + ), + "invalid Chat-shaped function tool choice", + ), + ( + ToolParams( + functions=({"type": "function", "name": "x", "parameters": {}},), + choice={ + "type": "function", + "function": {"name": "x", "future": True}, + }, + ), + "requires only a name", + ), + ( + ToolParams( + functions=({"type": "function", "name": "x", "parameters": {}},), + choice={"type": "function", "name": "x", "future": True}, + ), + "invalid function tool choice", + ), + ( + ToolParams( + functions=({"type": "function", "name": "x", "parameters": {}},), + choice={"type": "function", "name": ""}, + ), + "non-empty name", + ), + ( + ToolParams( + functions=({"type": "function", "name": "x", "parameters": {}},), + choice="none", + parallel=True, + ), + "meaningless with tool_choice='none'", + ), + ], + ) + async def test_invalid_tool_shape_is_rejected_before_http( + self, tools: ToolParams, message: str + ) -> None: + with pytest.raises(RequestAuditError, match=message): + await _run( + lambda _: pytest.fail("HTTP must not run"), + _request(tools=tools), + ) + + +class TestOpaqueContinuationValidation: + @staticmethod + def _payload() -> dict[str, object]: + return { + "version": 2, + "system": [], + "messages": [ + { + "role": "assistant", + "content": [ + { + "type": "thinking", + "thinking": "prior", + "signature": "sig", + } + ], + } + ], + "tools": None, + "open_turn": False, + "reasoning": {"thinking": None, "effort": None}, + } + + @pytest.mark.parametrize( + ("mutate", "message"), + [ + (lambda value: value.update(version=3), "version is unsupported"), + (lambda value: value.update(system={}), "history must be lists"), + (lambda value: value.update(messages={}), "history must be lists"), + (lambda value: value.update(open_turn=1), "open_turn must be a boolean"), + ( + lambda value: value.update(reasoning=None), + "reasoning state is malformed", + ), + ( + lambda value: value.update(reasoning={"thinking": None}), + "reasoning state is malformed", + ), + ( + lambda value: value.update( + reasoning={"thinking": None, "effort": "minimal"} + ), + "effort is invalid", + ), + ( + lambda value: value.update(reasoning={"thinking": [], "effort": None}), + "thinking state must be an object", + ), + ( + lambda value: value.update( + reasoning={ + "thinking": {"type": "enabled", "budget_tokens": 2048, "x": 1}, + "effort": None, + } + ), + "thinking state is malformed", + ), + ( + lambda value: value.update( + reasoning={ + "thinking": {"type": "enabled", "budget_tokens": 1}, + "effort": None, + } + ), + "thinking budget is invalid", + ), + ( + lambda value: value.update( + reasoning={ + "thinking": {"type": "adaptive", "future": True}, + "effort": None, + } + ), + "thinking state is malformed", + ), + ( + lambda value: value.update( + reasoning={"thinking": {"type": "off"}, "effort": None} + ), + "thinking type is invalid", + ), + ( + lambda value: value.update( + reasoning={ + "thinking": {"type": "adaptive", "display": "verbose"}, + "effort": None, + } + ), + "thinking display is invalid", + ), + ( + lambda value: value.update(messages=["wrong"]), + "continuation is malformed", + ), + ( + lambda value: value.update(open_turn=True), + "open_turn disagrees", + ), + ( + lambda value: value.update(messages=[]), + "contains no replayable", + ), + ], + ) + def test_invalid_envelope_is_rejected( + self, mutate: Callable[[dict[str, object]], None], message: str + ) -> None: + payload = self._payload() + mutate(payload) + + with pytest.raises(DialectError, match=message): + _decode_continuation(json.dumps(payload)) + + @pytest.mark.parametrize( + ("block", "role", "message"), + [ + ({"type": "text", "text": 1}, "user", "canonical text"), + ( + {"type": "text", "text": "x", "citations": [{}]}, + "assistant", + "cannot be replayed", + ), + ( + {"type": "image", "source": {"type": "url", "url": "x"}}, + "assistant", + "requires user role", + ), + ( + {"type": "image", "source": "wrong"}, + "user", + "canonical image", + ), + ( + { + "type": "image", + "source": {"type": "url", "url": "x", "future": True}, + }, + "user", + "canonical URL source", + ), + ( + {"type": "image", "source": {"type": "url", "url": ""}}, + "user", + "non-empty string", + ), + ( + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/bmp", + "data": "x", + }, + }, + "user", + "media_type is unsupported", + ), + ( + { + "type": "image", + "source": {"type": "base64", "media_type": "image/png", "data": ""}, + }, + "user", + "non-empty string", + ), + ( + {"type": "image", "source": {"type": "future"}}, + "user", + "source.type is unsupported", + ), + ( + {"type": "tool_use", "id": "id", "name": "x", "input": {}}, + "user", + "requires assistant role", + ), + ( + { + "type": "tool_use", + "id": "id", + "name": "x", + "input": {}, + "future": True, + }, + "assistant", + "canonical tool_use", + ), + ( + {"type": "tool_use", "id": "id", "name": "x", "input": []}, + "assistant", + "input must be an object", + ), + ( + { + "type": "tool_result", + "tool_use_id": "id", + "content": "x", + "is_error": False, + }, + "assistant", + "requires user role", + ), + ( + { + "type": "tool_result", + "tool_use_id": "id", + "content": 1, + "is_error": False, + }, + "user", + "content must be a string", + ), + ( + { + "type": "tool_result", + "tool_use_id": "id", + "content": "x", + "is_error": "no", + }, + "user", + "is_error must be a boolean", + ), + ( + {"type": "thinking", "thinking": "x", "signature": "sig"}, + "user", + "canonical thinking", + ), + ( + {"type": "thinking", "thinking": 1, "signature": "sig"}, + "assistant", + "thinking must be a string", + ), + ( + {"type": "redacted_thinking", "data": "opaque"}, + "user", + "canonical redacted-thinking", + ), + ( + {"type": "future"}, + "assistant", + "unsupported Anthropic block type", + ), + ], + ) + def test_lossy_history_block_is_rejected( + self, block: Mapping[str, object], role: str, message: str + ) -> None: + with pytest.raises(DialectError, match=message): + _validate_history_block(cast(Any, block), "history[0]", role=role) + + @pytest.mark.parametrize( + ("tool", "message"), + [ + ({"name": "x", "input_schema": {}, "future": True}, "unsupported"), + ({"name": "", "input_schema": {}}, "name must"), + ({"name": "x", "input_schema": []}, "input_schema must"), + ({"name": "x", "input_schema": {}, "description": 1}, "description"), + ({"name": "x", "input_schema": {}, "strict": "yes"}, "strict"), + ], + ) + def test_lossy_replay_tool_is_rejected( + self, tool: Mapping[str, object], message: str + ) -> None: + with pytest.raises(DialectError, match=message): + _validate_replay_tools([cast(Any, tool)]) + + @pytest.mark.parametrize( + "system", + [ + [{"type": "future", "text": "x"}], + [{"type": "text", "text": 1}], + ], + ) + def test_lossy_replay_system_is_rejected( + self, system: list[Mapping[str, object]] + ) -> None: + with pytest.raises(DialectError, match="opaque.system"): + _validate_replay_system(cast(Any, system)) + + @pytest.mark.parametrize( + ("messages", "message"), + [ + ([{"role": "user", "content": [], "future": True}], "unsupported fields"), + ([{"role": "tool", "content": []}], "role is invalid"), + ([{"role": "user", "content": []}], "content must be non-empty"), + ([{"role": "user", "content": ["wrong"]}], "must be an object"), + ], + ) + def test_lossy_replay_message_envelope_is_rejected( + self, messages: list[Mapping[str, object]], message: str + ) -> None: + payload = self._payload() + payload["messages"] = messages + + with pytest.raises(DialectError, match=message): + _decode_continuation(json.dumps(payload)) + + @pytest.mark.parametrize( + ("messages", "message"), + [ + ([{"role": "user", "content": "wrong"}], "content must be a list"), + ([{"role": "user", "content": ["wrong"]}], "must be an object"), + ( + [ + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "id"}], + }, + { + "role": "assistant", + "content": [{"type": "text", "text": "later"}], + }, + ], + "must immediately return", + ), + ( + [ + { + "role": "assistant", + "content": [{"type": "tool_result", "tool_use_id": "id"}], + } + ], + "require user role", + ), + ( + [ + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "id"}, + {"type": "tool_result", "tool_use_id": "id"}, + ], + } + ], + "duplicates a tool_result id", + ), + ( + [ + { + "role": "user", + "content": [{"type": "tool_use", "id": "id"}], + } + ], + "require assistant role", + ), + ], + ) + def test_tool_history_state_rejects_invalid_protocol( + self, messages: list[Mapping[str, object]], message: str + ) -> None: + rejection, pending = _tool_history_state(cast(Any, messages)) + + assert rejection is not None and message in rejection + assert pending == frozenset() + + +class TestWireVerifierNegativeEvidence: + def test_input_verifier_detects_system_presence_and_content_drift(self) -> None: + without_system = _AnthropicInputVerifier(_chat(), None) + assert ( + without_system.verify( + { + "messages": [ + {"role": "user", "content": [{"type": "text", "text": "Hello"}]} + ], + "system": [], + } + ) + == "body.system was added" + ) + + source = _chat( + ChatMessage("system", (TextPart("rules"),)), + ChatMessage("user", (TextPart("Hello"),)), + ) + verifier = _AnthropicInputVerifier(source, None) + assert verifier.verify({"messages": []}) == "body.system is missing" + assert ( + verifier.verify( + { + "system": [{"type": "text", "text": "changed"}], + "messages": [ + {"role": "user", "content": [{"type": "text", "text": "Hello"}]} + ], + } + ) + == "body.system changed" + ) + + def test_tool_and_reasoning_verifiers_reject_wrong_wire_shape(self) -> None: + source = ({"type": "function", "name": "lookup", "parameters": {}},) + tools = _AnthropicToolsVerifier(source) + assert tools.verify({"tools": []}) == "body.tools count changed" + + choice = _AnthropicToolChoiceVerifier(None, True) + assert choice.verify({"tool_choice": None}) == "body.tool_choice changed" + + summary = _ReasoningSummaryVerifier("auto", False) + assert summary.verify({}) == "body.thinking is missing" + assert ( + summary.verify({"thinking": {"type": "adaptive", "display": "omitted"}}) + == "body.thinking.display changed" + ) + assert ( + summary.verify({"thinking": {"type": "enabled", "display": "summarized"}}) + == "body.thinking.type changed" + ) + + +class TestAdditionalProviderResponseBoundaries: + @pytest.mark.anyio + @pytest.mark.parametrize( + ("response", "message"), + [ + ({**_response(), "id": ""}, "response.id"), + ({**_response(), "content": {}}, "response.content must be a list"), + ( + _response([{"type": "text", "text": "x", "future": True}]), + "unsupported text fields", + ), + (_response([{"type": "text", "text": 1}]), "text must be a string"), + ( + _response( + [{"type": "thinking", "thinking": "x", "signature": "sig", "x": 1}] + ), + "unsupported thinking fields", + ), + ( + _response([{"type": "thinking", "thinking": 1, "signature": "sig"}]), + "thinking must be a string", + ), + ( + _response([{"type": "redacted_thinking", "data": "x", "future": True}]), + "unsupported redacted-thinking fields", + ), + ( + _response( + [ + { + "type": "tool_use", + "id": "id", + "name": "x", + "input": {}, + "future": True, + } + ], + stop_reason="tool_use", + ), + "unsupported tool_use fields", + ), + ( + _response( + [{"type": "tool_use", "id": "id", "name": "x", "input": []}], + stop_reason="tool_use", + ), + "input must be an object", + ), + ( + {**_response(stop_reason="stop_sequence"), "stop_sequence": None}, + "response.stop_sequence", + ), + ( + _response(usage={"input_tokens": 1, "output_tokens": 1, "future": 1}), + "unsupported field", + ), + ( + _response( + usage={ + "input_tokens": 1, + "output_tokens": 1, + "cache_creation": {"ephemeral_1h_input_tokens": 1}, + } + ), + "cache_creation has unsupported fields", + ), + ( + _response( + usage={ + "input_tokens": 1, + "output_tokens": 1, + "server_tool_use": {"web_search_requests": 0}, + } + ), + "server_tool_use has unsupported fields", + ), + ( + _response( + usage={ + "input_tokens": 1, + "output_tokens": 1, + "service_tier": "free", + } + ), + "service_tier is invalid", + ), + ( + _response( + usage={"input_tokens": 1, "output_tokens": 1, "inference_geo": 1} + ), + "inference_geo must be a string", + ), + ( + _response( + usage={ + "input_tokens": 1, + "output_tokens": 1, + "output_tokens_details": {"thinking_tokens": 0, "future": 0}, + } + ), + "output_tokens_details has unsupported fields", + ), + ( + _response( + usage={ + "input_tokens": 1, + "output_tokens": 1, + "output_tokens_details": "not-an-object", + } + ), + "output_tokens_details must be an object", + ), + ( + _response( + usage={ + "input_tokens": 1, + "output_tokens": 1, + "output_tokens_details": [], + } + ), + "output_tokens_details must be an object", + ), + ], + ) + async def test_malformed_response_shape_fails_loudly( + self, response: Mapping[str, object], message: str + ) -> None: + with pytest.raises(OutputContractError, match=message): + await _run(lambda _: httpx.Response(200, json=response)) + + @pytest.mark.anyio + @pytest.mark.parametrize( + ("events", "message"), + [ + (({"type": "future"},), "unsupported Anthropic stream event"), + ( + (_stream_start(), _stream_start()), + "two message_start", + ), + ( + ( + { + **_stream_start(), + "message": _response( + content=[{"type": "text", "text": "x"}], stop_reason=None + ), + }, + ), + "content must be empty", + ), + ( + ( + { + **_stream_start(), + "message": _response(content=[], stop_reason="end_turn"), + }, + ), + "stop_reason must be null", + ), + ( + ( + _stream_start(), + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "text", "text": ""}, + }, + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "text", "text": ""}, + }, + ), + "duplicated content block", + ), + ( + ( + _stream_start(), + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "text_delta", "text": "x"}, + }, + ), + "inactive content block", + ), + ( + ( + _stream_start(), + { + "type": "content_block_start", + "index": 0, + "content_block": { + "type": "thinking", + "thinking": "", + "signature": "", + }, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "signature_delta", "signature": "a"}, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "signature_delta", "signature": "b"}, + }, + ), + "duplicated the signature delta", + ), + ( + ( + _stream_start(), + { + "type": "content_block_start", + "index": 0, + "content_block": { + "type": "thinking", + "thinking": "", + "signature": "", + }, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "signature_delta", "signature": "a"}, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "thinking_delta", "thinking": "late"}, + }, + ), + "followed the terminal signature delta", + ), + ( + ( + _stream_start(), + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "text", "text": ""}, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "text_delta", "text": 1}, + }, + ), + "text must be a string", + ), + ( + ( + _stream_start(), + { + "type": "content_block_start", + "index": 0, + "content_block": { + "type": "thinking", + "thinking": "", + "signature": "", + }, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "thinking_delta", "thinking": 1}, + }, + ), + "thinking must be a string", + ), + ( + ( + _stream_start(), + { + "type": "content_block_start", + "index": 0, + "content_block": { + "type": "thinking", + "thinking": "", + "signature": "", + }, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "signature_delta", "signature": 1}, + }, + ), + "signature must be a string", + ), + ( + ( + _stream_start(), + { + "type": "content_block_start", + "index": 0, + "content_block": { + "type": "tool_use", + "id": "id", + "name": "x", + "input": {}, + }, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "input_json_delta", "partial_json": 1}, + }, + ), + "partial_json must be a string", + ), + ( + ( + _stream_start(), + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "text", "text": ""}, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "thinking_delta", "thinking": "x"}, + }, + ), + "does not match block type", + ), + ( + ( + _stream_start(), + {"type": "content_block_stop", "index": 0}, + ), + "stop targets inactive", + ), + ( + ( + _stream_start(), + _stream_delta(), + _stream_delta(), + ), + "two message_delta", + ), + ( + ( + _stream_start(), + { + "type": "message_delta", + "delta": { + "stop_reason": "end_turn", + "stop_sequence": None, + "container": {}, + }, + "usage": {"output_tokens": 1}, + }, + ), + "container has no shared-IR mapping", + ), + ( + ( + _stream_start(), + _stream_delta(stop_reason="refusal"), + {"type": "message_stop"}, + ), + "response.stop_details must be an object", + ), + ( + ( + _stream_start(), + _stream_delta( + stop_reason="refusal", + stop_details={"type": "future"}, + ), + {"type": "message_stop"}, + ), + "stop_details.type must be 'refusal'", + ), + ( + ( + _stream_start(), + _stream_delta( + stop_details={"type": "refusal"}, + ), + {"type": "message_stop"}, + ), + "stop_details must be null unless stop_reason is 'refusal'", + ), + ( + ( + _stream_start(), + _stream_delta(), + {"type": "message_stop"}, + {"type": "ping"}, + ), + "data after message_stop", + ), + ( + ({"type": "ping"},), + "omitted message_start", + ), + ( + ( + _stream_start(), + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "text", "text": "x"}, + }, + {"type": "message_stop"}, + ), + "omitted message_delta", + ), + ( + ( + _stream_start(), + { + "type": "content_block_start", + "index": 1, + "content_block": {"type": "text", "text": "x"}, + }, + {"type": "content_block_stop", "index": 1}, + _stream_delta(), + {"type": "message_stop"}, + ), + "indexes are not dense", + ), + ], + ) + async def test_additional_malformed_streams_fail_loudly( + self, events: tuple[Mapping[str, object], ...], message: str + ) -> None: + with pytest.raises(OutputContractError, match=message): + await _run( + lambda _: httpx.Response( + 200, + content=_sse(*events), + headers={"content-type": "text/event-stream"}, + ), + _request(scheduling=SchedulingParams(stream=True)), + ) + + @pytest.mark.parametrize( + ("block", "message"), + [ + ({"type": "tool_use", "id": "id", "name": "x"}, "input is absent"), + ( + { + "type": "tool_use", + "id": "id", + "name": "x", + "input": {"x": 1}, + "_partial_json": '{"y":2}', + }, + "must start as an empty object", + ), + ( + { + "type": "tool_use", + "id": "id", + "name": "x", + "input": {}, + "_partial_json": "[]", + }, + "input must be an object", + ), + ], + ) + def test_stream_tool_block_finish_rejects_ambiguous_state( + self, block: dict[str, object], message: str + ) -> None: + with pytest.raises(OutputContractError, match=message): + _finish_stream_block(cast(Any, block), "content_block[0]") diff --git a/tests/unit/core/models/test_dialect.py b/tests/unit/core/models/test_dialect.py index 84287ca5..0a4baaf7 100644 --- a/tests/unit/core/models/test_dialect.py +++ b/tests/unit/core/models/test_dialect.py @@ -676,6 +676,43 @@ def test_active_present_or_error_channel_must_exist(self): Response(("x",)), ) + def test_refusal_skips_success_channel_and_continuation_requirements(self): + rules = _all_rules(Guarantee.BEST_EFFORT) + rules["reasoning"] = OutputRule(Guarantee.PRESENT_OR_ERROR) + rules["structured_output"] = OutputRule(Guarantee.PRESENT_OR_ERROR) + contract = OutputContract(rules) + req = Request( + input=CompletionInput("x"), + reasoning=ReasoningParams(summary="auto"), + structured_output=StructuredOutputParams(format="json_object"), + session=SessionParams( + opaque_continuation=OpaqueContinuation("future_dialect", "previous") + ), + ) + + contract.validate( + _Plan(), + req, + Response(("",), finish_reasons=("refusal",)), + ) + + def test_mixed_refusal_choices_still_require_success_channels(self): + rules = _all_rules(Guarantee.BEST_EFFORT) + rules["reasoning"] = OutputRule(Guarantee.PRESENT_OR_ERROR) + contract = OutputContract(rules) + req = Request( + input=CompletionInput("x"), + sampling=SamplingParams(n=2), + reasoning=ReasoningParams(summary="auto"), + ) + + with pytest.raises(OutputContractError, match="required.*absent"): + contract.validate( + _Plan(), + req, + Response(("", "answer"), finish_reasons=("refusal", "end_turn")), + ) + def test_required_best_effort_channel_is_not_a_guarantee(self): contract = OutputContract(_all_rules(Guarantee.BEST_EFFORT)) with pytest.raises(OutputContractError, match="best_effort guarantee"): diff --git a/tests/unit/core/models/test_dialect_registry.py b/tests/unit/core/models/test_dialect_registry.py index ded346e6..fd25a2dd 100644 --- a/tests/unit/core/models/test_dialect_registry.py +++ b/tests/unit/core/models/test_dialect_registry.py @@ -9,6 +9,7 @@ from types import SimpleNamespace from typing import Any, cast +import httpx import pytest import sieval.core.models.dialect_registry as registry @@ -18,6 +19,7 @@ DialectCapabilityStatus, Supported, ) +from sieval.core.models.connection_factory import AsyncHTTPJSONConnection from sieval.core.models.deployment import ( ConnectionIdentity, ConnectionPool, @@ -45,6 +47,7 @@ dialect_registry_to_json, get_dialect_spec, ) +from sieval.core.models.dialects.anthropic_messages import AnthropicMessagesDialect from sieval.core.models.dialects.openai_chat import OpenAIChatDialect from sieval.core.models.dialects.openai_completions import OpenAICompletionsDialect from sieval.core.models.dialects.openai_responses import OpenAIResponsesDialect @@ -68,6 +71,7 @@ def close(self) -> None: def _runtime_binding( dialect_id: str = "openai_chat", ) -> tuple[Deployment, ConnectionPool[Any], SimpleNamespace]: + family = DIALECT_SPECS[dialect_id].connection_family deployment = Deployment( deployment_id="deployment", plan=DeploymentPlanProjection("sha256:plan", "vllm"), @@ -79,7 +83,7 @@ def _runtime_binding( metrics_url=None, facts=ServingFacts(), ) - route = resolve_route(deployment, dialect_id, "openai_sdk") + route = resolve_route(deployment, dialect_id, family) identity = ConnectionIdentity( endpoint=route.endpoint, connection_family=route.connection_family, @@ -94,7 +98,10 @@ def _runtime_binding( resolved_route=route, connection_identity=identity, ) - return deployment, ConnectionPool(_Connection(), identity), plan + connection: object = _Connection() + if family == "async_http_json": + connection = AsyncHTTPJSONConnection(httpx.AsyncClient(), None) + return deployment, ConnectionPool(connection, identity), plan class TestDialectDescriptors: @@ -125,7 +132,7 @@ def test_request_seed_support_is_required_and_explicit(self) -> None: "openai_chat": RequestSeedSupport.SUPPORTED, "openai_completions": RequestSeedSupport.SUPPORTED, "openai_responses": RequestSeedSupport.UNSUPPORTED, - "anthropic_messages": RequestSeedSupport.RESERVED, + "anthropic_messages": RequestSeedSupport.UNSUPPORTED, "google_genai": RequestSeedSupport.RESERVED, "sglang_native": RequestSeedSupport.RESERVED, "vllm_native": RequestSeedSupport.RESERVED, @@ -141,6 +148,7 @@ def test_implemented_dialects_are_active_and_bindable(self) -> None: "openai_chat", "openai_completions", "openai_responses", + "anthropic_messages", } assert set(DIALECT_BINDERS) == active assert all(inspect.isfunction(binder) for binder in DIALECT_BINDERS.values()) @@ -253,8 +261,11 @@ def test_request_leaf_has_only_one_capability_owner(self) -> None: def test_capability_decisions_are_available_only_for_active_dialects(self) -> None: assert set(capability_decisions_for("openai_chat")) == set(CAPABILITY_KEYS) assert set(capability_decisions_for("openai_responses")) == set(CAPABILITY_KEYS) + assert set(capability_decisions_for("anthropic_messages")) == set( + CAPABILITY_KEYS + ) with pytest.raises(DialectNotImplemented, match="later #25 adapter"): - capability_decisions_for("anthropic_messages") + capability_decisions_for("google_genai") class TestDialectBinders: @@ -283,17 +294,21 @@ def test_reserved_dialects_fail_with_named_error( ("openai_chat", OpenAIChatDialect), ("openai_completions", OpenAICompletionsDialect), ("openai_responses", OpenAIResponsesDialect), + ("anthropic_messages", AnthropicMessagesDialect), ], ) def test_active_binders_construct_expected_dialect( self, dialect_id: str, expected_type: type ) -> None: binder = DIALECT_BINDERS[dialect_id] - dialect = binder(object(), "requested-model") + connection: object = object() + if dialect_id == "anthropic_messages": + connection = AsyncHTTPJSONConnection(httpx.AsyncClient(), None) + dialect = binder(connection, "requested-model") assert isinstance(dialect, expected_type) assert dialect.dialect_id == dialect_id - assert dialect.connection_family == "openai_sdk" + assert dialect.connection_family == DIALECT_SPECS[dialect_id].connection_family @pytest.mark.parametrize( ("dialect_id", "expected_type"), @@ -301,6 +316,7 @@ def test_active_binders_construct_expected_dialect( ("openai_chat", OpenAIChatDialect), ("openai_completions", OpenAICompletionsDialect), ("openai_responses", OpenAIResponsesDialect), + ("anthropic_messages", AnthropicMessagesDialect), ], ) def test_validated_runtime_plan_binds_active_dialect( diff --git a/tests/unit/core/tasks/test_anomaly.py b/tests/unit/core/tasks/test_anomaly.py index 2a1fa220..a73d2464 100644 --- a/tests/unit/core/tasks/test_anomaly.py +++ b/tests/unit/core/tasks/test_anomaly.py @@ -188,6 +188,11 @@ def test_single_sample(self, sample_model_meta): (["length"], {0}, "length"), (["max_output_tokens"], {0}, "max_output_tokens"), (["max_tokens"], {0}, "max_tokens"), + ( + ["model_context_window_exceeded"], + {0}, + "model_context_window_exceeded", + ), (["content_filter"], {0}, "content_filter"), (["stop"], set(), "stop"), (None, set(), "missing_finish_reasons"), diff --git a/tests/unit/core/utils/test_meta.py b/tests/unit/core/utils/test_meta.py index f8374e88..eb910cca 100644 --- a/tests/unit/core/utils/test_meta.py +++ b/tests/unit/core/utils/test_meta.py @@ -307,12 +307,14 @@ def test_counts_rollouts_not_samples(self): def test_every_provider_spelling_counts(self): # The IR keeps finish_reasons provider-verbatim: OpenAI-compatible # Chat/Completions and SGLang native say `length`, Responses says - # `max_output_tokens`, and Anthropic says `max_tokens`. A set holding only - # one spelling would silently read zero on another provider. + # `max_output_tokens`, and Anthropic says either `max_tokens` or + # `model_context_window_exceeded`. A set holding only one spelling would + # silently read zero on another provider. for reason in ( "length", "max_output_tokens", "max_tokens", + "model_context_window_exceeded", "content_filter", ): assert (