From f2230640f3259e59a6007b325864e4ff0d3fe2fc Mon Sep 17 00:00:00 2001 From: jack-scitix-ai Date: Sun, 13 Sep 2026 09:40:30 +0800 Subject: [PATCH 01/15] fix(models): audit prepared request control flow Allow dialect evidence to verify the final PreparedRequest rather than only JSON body fields. This lets operation-selected protocols prove stream and endpoint control flow without inventing a body parameter. --- sieval/core/models/dialect.py | 25 +++++++++++++++++++++---- tests/unit/core/models/test_dialect.py | 26 ++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/sieval/core/models/dialect.py b/sieval/core/models/dialect.py index b849445a..d939f0ee 100644 --- a/sieval/core/models/dialect.py +++ b/sieval/core/models/dialect.py @@ -74,7 +74,14 @@ class WireVerifier(Protocol): def verify(self, body: Mapping[str, JSONValue]) -> str | None: ... -type WireEvidence = WireObservation | WireVerifier +@runtime_checkable +class PreparedRequestVerifier(Protocol): + """Independently verify request evidence outside the JSON body.""" + + def verify_prepared(self, prepared: "PreparedRequest") -> str | None: ... + + +type WireEvidence = WireObservation | WireVerifier | PreparedRequestVerifier @dataclass(frozen=True) @@ -310,10 +317,16 @@ def consumed(self, path: str, *wire_evidence: WireEvidence) -> None: f"consumed request leaf {path!r} requires wire evidence" ) if not all( - isinstance(evidence, (WireObservation, WireVerifier)) + isinstance( + evidence, + (WireObservation, WireVerifier, PreparedRequestVerifier), + ) for evidence in wire_evidence ): - raise TypeError("wire evidence must be WireObservation or WireVerifier") + raise TypeError( + "wire evidence must be WireObservation, WireVerifier, or " + "PreparedRequestVerifier" + ) self._record(path, Consumed(path, tuple(wire_evidence))) def rejected(self, path: str, reason: str) -> None: @@ -382,7 +395,11 @@ def finish(self, prepared: PreparedRequest) -> None: if verifier_id in verified: continue verified.add(verifier_id) - mismatch = evidence.verify(prepared.body) + mismatch = ( + evidence.verify_prepared(prepared) + if isinstance(evidence, PreparedRequestVerifier) + else evidence.verify(prepared.body) + ) if mismatch is not None and ( not isinstance(mismatch, str) or not mismatch.strip() ): diff --git a/tests/unit/core/models/test_dialect.py b/tests/unit/core/models/test_dialect.py index 0a4baaf7..c30c8b18 100644 --- a/tests/unit/core/models/test_dialect.py +++ b/tests/unit/core/models/test_dialect.py @@ -96,6 +96,16 @@ def verify(self, body: Mapping[str, JSONValue]) -> str | None: raise RuntimeError("verifier bug") +@dataclass(frozen=True) +class _OperationVerifier: + expected: str + + def verify_prepared(self, prepared: PreparedRequest) -> str | None: + if prepared.operation != self.expected: + return f"operation {prepared.operation!r} does not match {self.expected!r}" + return None + + class TestLeafDerivation: def test_derives_nondefault_group_leaves_and_dynamic_none(self): req = Request( @@ -314,6 +324,22 @@ def test_prepared_request_thaws_one_detached_mutable_body(self): assert thawed == {"extra_body": {"stop_ids": [1, 2, 3]}} assert prepared.body == {"extra_body": {"stop_ids": [1, 2]}} + def test_prepared_request_verifier_can_prove_operation_control_flow(self): + audit = RequestAudit({"scheduling.stream": True}) + audit.consumed( + "scheduling.stream", + _OperationVerifier("models/model:streamGenerateContent"), + ) + + audit.finish( + PreparedRequest("models/model:streamGenerateContent", {"contents": []}) + ) + + with pytest.raises(RequestAuditError, match="operation.*does not match"): + audit.finish( + PreparedRequest("models/model:generateContent", {"contents": []}) + ) + def test_passthrough_destination_and_value_are_verified(self): audit = RequestAudit({"dialect_options.min_p": 0.1}) audit.passthrough("dialect_options.min_p", "extra_body") From 4abb7007faf89030faf31c213e805f965c896ea5 Mon Sep 17 00:00:00 2001 From: jack-scitix-ai Date: Sun, 13 Sep 2026 10:09:25 +0800 Subject: [PATCH 02/15] fix(meta): recognize Google token-limit truncation Keep Google generateContent finish reasons provider-verbatim while ensuring MAX_TOKENS contributes to both anomaly detection and truncated-rollout reporting. --- sieval/core/utils/meta.py | 6 ++++-- tests/unit/core/tasks/test_anomaly.py | 1 + tests/unit/core/utils/test_meta.py | 7 ++++--- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/sieval/core/utils/meta.py b/sieval/core/utils/meta.py index 01d962e3..e19c8dde 100644 --- a/sieval/core/utils/meta.py +++ b/sieval/core/utils/meta.py @@ -14,8 +14,9 @@ #: #: Every spelling, not one canonical name: the IR does not normalize these, so a #: set holding only ``length`` (OpenAI-compatible Chat/Completions and SGLang -#: native) would read zero against ``max_output_tokens`` (OpenAI Responses) or -#: ``max_tokens`` or ``model_context_window_exceeded`` (Anthropic). +#: native) would read zero against ``max_output_tokens`` (OpenAI Responses), +#: ``max_tokens`` or ``model_context_window_exceeded`` (Anthropic), or +#: ``MAX_TOKENS`` (Google generateContent). #: ``content_filter`` is in because the output is cut short the same way; #: separating causes is what the raw ``finish_reasons`` are for. Shared with #: :func:`sieval.core.tasks.anomaly.detect_truncated_output` so the rule and the @@ -26,6 +27,7 @@ "max_output_tokens", "max_tokens", "model_context_window_exceeded", + "MAX_TOKENS", "content_filter", } ) diff --git a/tests/unit/core/tasks/test_anomaly.py b/tests/unit/core/tasks/test_anomaly.py index a73d2464..1faa9955 100644 --- a/tests/unit/core/tasks/test_anomaly.py +++ b/tests/unit/core/tasks/test_anomaly.py @@ -193,6 +193,7 @@ def test_single_sample(self, sample_model_meta): {0}, "model_context_window_exceeded", ), + (["MAX_TOKENS"], {0}, "MAX_TOKENS"), (["content_filter"], {0}, "content_filter"), (["stop"], set(), "stop"), (None, set(), "missing_finish_reasons"), diff --git a/tests/unit/core/utils/test_meta.py b/tests/unit/core/utils/test_meta.py index eb910cca..bb042f30 100644 --- a/tests/unit/core/utils/test_meta.py +++ b/tests/unit/core/utils/test_meta.py @@ -307,14 +307,15 @@ def test_counts_rollouts_not_samples(self): def test_every_provider_spelling_counts(self): # The IR keeps finish_reasons provider-verbatim: OpenAI-compatible # Chat/Completions and SGLang native say `length`, Responses says - # `max_output_tokens`, and Anthropic says either `max_tokens` or - # `model_context_window_exceeded`. A set holding only one spelling would - # silently read zero on another provider. + # `max_output_tokens`, Anthropic says `max_tokens` or + # `model_context_window_exceeded`, and Google says `MAX_TOKENS`. A set + # holding only one spelling would silently read zero on another provider. for reason in ( "length", "max_output_tokens", "max_tokens", "model_context_window_exceeded", + "MAX_TOKENS", "content_filter", ): assert ( From 5a9bf3c491560724f30027db00942cbe0f5e1de2 Mon Sep 17 00:00:00 2001 From: jack-scitix-ai Date: Sun, 13 Sep 2026 10:20:38 +0800 Subject: [PATCH 03/15] feat(models): add Google GenAI dialect Bind the native generateContent protocol with audited request lowering, strict streaming and non-streaming lifting, capability declarations, deterministic seed support, and provider-native continuation evidence. --- sieval/core/models/dialect_registry.py | 18 +- sieval/core/models/dialects/google_genai.py | 1974 +++++++++++++++++ tests/unit/cli/leaderboard/test_session.py | 4 +- .../core/models/dialects/test_google_genai.py | 914 ++++++++ .../unit/core/models/test_dialect_registry.py | 11 +- 5 files changed, 2915 insertions(+), 6 deletions(-) create mode 100644 sieval/core/models/dialects/google_genai.py create mode 100644 tests/unit/core/models/dialects/test_google_genai.py diff --git a/sieval/core/models/dialect_registry.py b/sieval/core/models/dialect_registry.py index 9340eb34..b57ef5d6 100644 --- a/sieval/core/models/dialect_registry.py +++ b/sieval/core/models/dialect_registry.py @@ -34,6 +34,10 @@ CAPABILITY_DECISIONS as ANTHROPIC_MESSAGES_CAPABILITY_DECISIONS, ) from .dialects.anthropic_messages import AnthropicMessagesDialect +from .dialects.google_genai import ( + CAPABILITY_DECISIONS as GOOGLE_GENAI_CAPABILITY_DECISIONS, +) +from .dialects.google_genai import GoogleGenAIDialect from .dialects.openai_chat import ( CAPABILITY_DECISIONS as OPENAI_CHAT_CAPABILITY_DECISIONS, ) @@ -329,6 +333,10 @@ def _bind_anthropic_messages(connection: Any, requested_model_id: str) -> Dialec return AnthropicMessagesDialect(connection, requested_model_id) +def _bind_google_genai(connection: Any, requested_model_id: str) -> Dialect: + return GoogleGenAIDialect(connection, requested_model_id) + + def _binder( function: Callable[[Any, str], Dialect], decisions: Mapping[str, DialectCapabilityDecision], @@ -370,6 +378,13 @@ def _binder( ANTHROPIC_MESSAGES_CAPABILITY_DECISIONS, ), ), + "google_genai": _binder( + _bind_google_genai, + cast( + Mapping[str, DialectCapabilityDecision], + GOOGLE_GENAI_CAPABILITY_DECISIONS, + ), + ), } ) @@ -418,7 +433,8 @@ def _registered_decisions( "google_genai": _spec( "google_genai", "async_http_json", - request_seed_support=RequestSeedSupport.RESERVED, + decisions=_registered_decisions("google_genai"), + request_seed_support=RequestSeedSupport.SUPPORTED, input_kinds=("chat",), input_modalities=("text", "image", "tool_call", "tool_result"), ), diff --git a/sieval/core/models/dialects/google_genai.py b/sieval/core/models/dialects/google_genai.py new file mode 100644 index 00000000..db59487d --- /dev/null +++ b/sieval/core/models/dialects/google_genai.py @@ -0,0 +1,1974 @@ +"""Google Gemini native ``generateContent`` dialect for the model IR. + +The adapter speaks the public REST protocol through the provider-neutral +``async_http_json`` connection. Request leaves are translated only where the +shared IR has a lossless Google representation; provider-only controls remain +audited dialect options and ambiguous response shapes fail loudly. + +AI-Generated Code - GPT-5.6 (OpenAI) +""" + +import json +import math +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass, field +from types import MappingProxyType +from typing import cast +from urllib.parse import quote + +import httpx + +from sieval.core.models._shared import copy_json_value +from sieval.core.models.capabilities import ( + CAPABILITY_KEYS, + Capability, + CapabilityKey, + DialectCapabilityBinding, + DialectCapabilityDecision, + HostedToolsOptions, + MultimodalInputOptions, + ReasoningOptions, + Supported, + TopLogprobsOptions, + Unsupported, +) +from sieval.core.models.connection_factory import AsyncHTTPJSONConnection +from sieval.core.models.deployment import BINDING_RESOURCE_KEYS +from sieval.core.models.dialect import ( + DialectError, + Guarantee, + OutputContract, + OutputContractError, + OutputRule, + PreparedRequest, + RequestAudit, + RuntimePlanView, + WireEvidence, + WireObservation, + active_request_leaves, + validate_reasoning, + validate_request_invariants, + validate_runtime_binding_plan, + validate_structured_output, + validate_tool_calls, + validate_top_logprobs, +) +from sieval.core.models.ir import ( + ChatInput, + ChatMessage, + Citation, + FunctionToolCall, + GroundingChunk, + GroundingMetadata, + ImagePart, + ReasoningOutput, + Request, + Response, + ServerToolUse, + StructuredOutput, + TextPart, + TokenLogprob, + ToolCallPart, + ToolResultPart, + TopKEntry, + UsageStats, +) +from sieval.core.types import JSONValue + +_THINKING_LEVELS = frozenset({"minimal", "low", "medium", "high"}) +_HOSTED_TOOL_KINDS = frozenset({"google_search", "url_context"}) +_IMAGE_MEDIA_TYPES = frozenset( + { + "image/avif", + "image/gif", + "image/heic", + "image/heif", + "image/jpeg", + "image/jpg", + "image/png", + "image/webp", + } +) +_OPAQUE_VERSION = 1 +_IR_OWNED_KEYS = BINDING_RESOURCE_KEYS | frozenset( + { + "model", + "contents", + "systemInstruction", + "generationConfig", + "tools", + "toolConfig", + "cachedContent", + "serviceTier", + "store", + "stream", + } +) + + +def _reject_json_constant(value: str) -> None: + raise ValueError(f"non-finite JSON constant {value!r}") + + +def _reject_duplicate_keys( + pairs: list[tuple[str, JSONValue]], +) -> dict[str, JSONValue]: + result: dict[str, JSONValue] = {} + for key, value in pairs: + if key in result: + raise ValueError(f"duplicate JSON key {key!r}") + result[key] = value + return result + + +def _strict_json_loads(payload: str, path: str) -> JSONValue: + try: + return cast( + JSONValue, + json.loads( + payload, + object_pairs_hook=_reject_duplicate_keys, + parse_constant=_reject_json_constant, + ), + ) + except (json.JSONDecodeError, TypeError, ValueError) as exc: + raise OutputContractError(f"{path} is not strict JSON") from exc + + +def _tuple_validator(item_type: type, channel: str) -> Callable[[object], None]: + def validate(value: object) -> None: + if not isinstance(value, tuple) or not all( + isinstance(item, item_type) for item in value + ): + raise OutputContractError(f"{channel} channel has invalid shape") + + return validate + + +def _validate_top_logprobs_config(options: object) -> None: + assert isinstance(options, TopLogprobsOptions) + if options.minimum > 20: + raise ValueError("google_genai supports at most 20 top logprobs") + + +def _validate_reasoning_config(options: object) -> None: + assert isinstance(options, ReasoningOptions) + if options.summary not in {None, "none", "auto"}: + raise ValueError("google_genai supports only reasoning summary 'auto'") + if options.effort is not None and options.effort.lower() not in _THINKING_LEVELS: + raise ValueError(f"unsupported Google thinking level {options.effort!r}") + + +def _validate_hosted_tools_config(options: object) -> None: + assert isinstance(options, HostedToolsOptions) + unsupported = set(options.kinds) - _HOSTED_TOOL_KINDS + if unsupported: + raise ValueError( + "google_genai supports only google_search and url_context hosted tools; " + f"unsupported kinds: {sorted(unsupported)}" + ) + + +def _validate_multimodal_config(options: object) -> None: + assert isinstance(options, MultimodalInputOptions) + unsupported = set(options.modalities) - {"image"} + if unsupported: + raise ValueError(f"google_genai does not support modalities {unsupported!r}") + + +def _decisions() -> Mapping[CapabilityKey, DialectCapabilityDecision]: + decisions: dict[CapabilityKey, DialectCapabilityDecision] = { + "input_scoring": Unsupported( + "Gemini generateContent cannot score input tokens" + ), + "sampled_logprobs": Supported( + DialectCapabilityBinding( + "sampled_logprobs", + request_leaves=("scoring.sampled_logprobs",), + response_channels=("logprobs",), + ) + ), + "top_logprobs": Supported( + DialectCapabilityBinding( + "top_logprobs", + request_leaves=("scoring.top_logprobs",), + response_channels=("top_logprobs",), + _config_validator=_validate_top_logprobs_config, + ) + ), + "reasoning": Supported( + DialectCapabilityBinding( + "reasoning", + request_leaves=( + "reasoning.effort", + "reasoning.budget_tokens", + "reasoning.summary", + ), + response_channels=("reasoning",), + _config_validator=_validate_reasoning_config, + ) + ), + "function_tools": Supported( + DialectCapabilityBinding( + "function_tools", + request_leaves=("tools.functions", "tools.choice"), + response_channels=("tool_calls",), + ) + ), + "hosted_tools": Supported( + DialectCapabilityBinding( + "hosted_tools", + request_leaves=("tools.hosted",), + response_channels=("server_tool_uses",), + _config_validator=_validate_hosted_tools_config, + ) + ), + "structured_output": Supported( + DialectCapabilityBinding( + "structured_output", + request_leaves=( + "structured_output.format", + "structured_output.schema", + "structured_output.name", + "structured_output.strict", + ), + response_channels=("structured_output",), + ) + ), + "stateful_session": Unsupported( + "Gemini responseId is not a request continuation handle" + ), + "opaque_continuation": Supported( + DialectCapabilityBinding( + "opaque_continuation", + request_leaves=("session.opaque_continuation",), + response_channels=("reasoning",), + ) + ), + "multimodal_input": Supported( + DialectCapabilityBinding( + "multimodal_input", + request_leaves=( + "input.modality.image", + "input.modality.image.media_type", + ), + _config_validator=_validate_multimodal_config, + ) + ), + "prefill": Unsupported("Gemini has no independent assistant-prefill field"), + "fim": Unsupported("Gemini generateContent has no completion suffix field"), + } + if set(decisions) != set(CAPABILITY_KEYS): + raise AssertionError("Google GenAI capability row is incomplete") + return MappingProxyType(decisions) + + +CAPABILITY_DECISIONS = _decisions() + +OUTPUT_CONTRACT = OutputContract( + { + "reasoning": OutputRule(Guarantee.PRESENT_OR_ERROR, validate_reasoning), + "tool_calls": OutputRule(Guarantee.BEST_EFFORT, validate_tool_calls), + "server_tool_uses": OutputRule( + Guarantee.BEST_EFFORT, + _tuple_validator(ServerToolUse, "server_tool_uses"), + ), + "structured_output": OutputRule( + Guarantee.PRESENT_OR_ERROR, validate_structured_output + ), + "logprobs": OutputRule( + Guarantee.PRESENT_OR_ERROR, + _tuple_validator(TokenLogprob, "logprobs"), + ), + "top_logprobs": OutputRule(Guarantee.PRESENT_OR_ERROR, validate_top_logprobs), + "input_scoring": OutputRule(Guarantee.NEVER), + "citations": OutputRule( + Guarantee.BEST_EFFORT, _tuple_validator(Citation, "citations") + ), + "grounding": OutputRule(Guarantee.BEST_EFFORT), + "session_id": OutputRule(Guarantee.NEVER), + "usage": OutputRule(Guarantee.BEST_EFFORT), + } +) + + +@dataclass(frozen=True) +class _GoogleContext: + request: Request + + +@dataclass(frozen=True) +class _GoogleExecutionContext: + request: Request + request_params: Mapping[str, JSONValue] + input_contents: tuple[Mapping[str, JSONValue], ...] + + +@dataclass(frozen=True) +class _LegacyPlan: + dialect_id: str = "google_genai" + available_capabilities: frozenset[str] = frozenset( + key + for key, decision in CAPABILITY_DECISIONS.items() + if isinstance(decision, Supported) + ) + capability_minimums: Mapping[str, Mapping[str, JSONValue]] = field( + default_factory=dict + ) + required_output_channels: frozenset[str] = frozenset() + + +@dataclass(frozen=True) +class _OperationVerifier: + model_name: str + stream: bool + version_prefix: str + + def verify_prepared(self, prepared: PreparedRequest) -> str | None: + method = "streamGenerateContent" if self.stream else "generateContent" + expected = ( + f"{self.version_prefix}models/{quote(self.model_name, safe='')}:{method}" + ) + if self.stream: + expected += "?alt=sse" + if prepared.operation != expected: + return f"operation {prepared.operation!r} does not match {expected!r}" + return None + + +@dataclass(frozen=True) +class _InputVerifier: + expected_system: Mapping[str, JSONValue] | None + expected_contents: tuple[JSONValue, ...] + + @classmethod + def from_request( + cls, request_input: ChatInput, opaque: str | None + ) -> "_InputVerifier": + system, contents = _verification_input_to_wire(request_input, opaque) + return cls(system, tuple(contents)) + + def verify(self, body: Mapping[str, JSONValue]) -> str | None: + if body.get("contents") != list(self.expected_contents): + return "contents no longer match the provider-neutral input" + if body.get("systemInstruction") != self.expected_system: + return "systemInstruction no longer matches system/developer messages" + return None + + +def _get(raw: object, name: str, default: object = None) -> object: + if isinstance(raw, Mapping): + return cast(Mapping[object, object], raw).get(name, default) + return getattr(raw, name, default) + + +def _sequence(raw: object, path: str) -> Sequence[object]: + if not isinstance(raw, Sequence) or isinstance(raw, str | bytes): + raise OutputContractError(f"{path} must be a sequence") + return cast(Sequence[object], raw) + + +def _required_string(raw: object, path: str) -> str: + if not isinstance(raw, str) or not raw: + raise OutputContractError(f"{path} must be a non-empty string") + return raw + + +def _string(raw: object, path: str) -> str: + if not isinstance(raw, str): + raise OutputContractError(f"{path} must be a string") + return raw + + +def _optional_string(raw: object, path: str) -> str | None: + if raw is None: + return None + if not isinstance(raw, str): + raise OutputContractError(f"{path} must be a string or None") + return raw + + +def _finite_number(raw: object, path: str) -> float: + if isinstance(raw, bool) or not isinstance(raw, int | float): + raise OutputContractError(f"{path} must be numeric") + value = float(raw) + if not math.isfinite(value): + raise OutputContractError(f"{path} must be finite") + return value + + +def _nonnegative_int(raw: object, path: str, *, default: int = 0) -> int: + if raw is None: + return default + if isinstance(raw, bool) or not isinstance(raw, int) or raw < 0: + raise OutputContractError(f"{path} must be a non-negative integer") + return raw + + +def _optional_nonnegative_int(raw: object, path: str) -> int | None: + if raw is None: + return None + return _nonnegative_int(raw, path) + + +def _function_declaration(raw: Mapping[str, JSONValue]) -> dict[str, JSONValue]: + data = dict(raw) + if data.get("type") == "function": + nested = data.get("function") + if nested is not None: + if set(data) != {"type", "function"} or not isinstance(nested, Mapping): + raise DialectError("invalid OpenAI-shaped function declaration") + data = dict(nested) + else: + data.pop("type") + allowed = {"name", "description", "parameters"} + unknown = set(data) - allowed + if unknown: + raise DialectError( + "unsupported Google function declaration field(s): " + + ", ".join(sorted(unknown)) + ) + name = data.get("name") + if not isinstance(name, str) or not name: + raise DialectError("Google function declaration requires a name") + result: dict[str, JSONValue] = {"name": name} + if "description" in data: + result["description"] = data["description"] + if "parameters" in data: + result["parametersJsonSchema"] = data["parameters"] + return result + + +def _tool_choice(raw: JSONValue) -> dict[str, JSONValue]: + if isinstance(raw, str): + modes = {"auto": "AUTO", "none": "NONE", "required": "ANY"} + try: + return {"functionCallingConfig": {"mode": modes[raw]}} + except KeyError as exc: + raise DialectError(f"unsupported Google tool choice {raw!r}") from exc + if not isinstance(raw, Mapping): + raise DialectError("Google tool choice must be a string or mapping") + data = dict(raw) + nested = data.get("function") + if set(data) == {"type", "function"} and isinstance(nested, Mapping): + name = nested.get("name") + elif set(data) == {"type", "name"} and data.get("type") == "function": + name = data.get("name") + else: + raise DialectError("invalid Google function-specific tool choice") + if not isinstance(name, str) or not name: + raise DialectError("Google function-specific tool choice requires a name") + return { + "functionCallingConfig": { + "mode": "ANY", + "allowedFunctionNames": [name], + } + } + + +def _hosted_tool(spec: object) -> dict[str, JSONValue]: + kind = getattr(spec, "kind", None) + config = getattr(spec, "config", None) + if kind not in _HOSTED_TOOL_KINDS or not isinstance(config, Mapping): + raise DialectError(f"unsupported Google hosted tool {kind!r}") + key = "googleSearch" if kind == "google_search" else "urlContext" + return {key: dict(config)} + + +def _image_to_part(part: ImagePart) -> dict[str, JSONValue]: + if part.detail is not None: + raise DialectError("Google image parts have no per-image detail field") + if part.url is not None: + if not part.url: + raise DialectError("Google image URLs must not be empty") + data: dict[str, JSONValue] = {"fileUri": part.url} + if part.media_type is not None: + data["mimeType"] = part.media_type + return {"fileData": data} + if not part.data or not part.media_type: + raise DialectError( + "Google inline images require data and an explicit MIME type" + ) + if part.media_type not in _IMAGE_MEDIA_TYPES: + raise DialectError(f"unsupported Google image media type {part.media_type!r}") + return {"inlineData": {"data": part.data, "mimeType": part.media_type}} + + +def _tool_result_payload(part: ToolResultPart) -> dict[str, JSONValue]: + value = copy_json_value(part.result, "function response") + if part.is_error: + return {"error": value} + if isinstance(value, dict): + return dict(value) + return {"result": value} + + +def _content_parts( + message: ChatMessage, + call_names: dict[str, str], + answered_calls: set[str], +) -> list[dict[str, JSONValue]]: + parts: list[dict[str, JSONValue]] = [] + for part in message.content: + if isinstance(part, TextPart): + if message.role == "tool": + raise DialectError("Google tool messages may contain only tool results") + parts.append({"text": part.text}) + elif isinstance(part, ImagePart): + if message.role == "tool": + raise DialectError("Google tool messages may contain only tool results") + parts.append(_image_to_part(part)) + elif isinstance(part, ToolCallPart): + if message.role != "assistant": + raise DialectError( + "Google historical function calls require assistant role" + ) + if not part.call_id or not part.name: + raise DialectError( + "Google historical function calls require id and name" + ) + if part.call_id in call_names: + raise DialectError( + f"duplicate Google function call id {part.call_id!r}" + ) + call_names[part.call_id] = part.name + parts.append( + { + "functionCall": { + "id": part.call_id, + "name": part.name, + "args": copy_json_value(part.arguments, "function arguments"), + } + } + ) + elif isinstance(part, ToolResultPart): + if message.role != "tool": + raise DialectError("Google function results require tool role") + try: + name = call_names[part.call_id] + except KeyError as exc: + raise DialectError( + f"Google function result {part.call_id!r} has no prior call" + ) from exc + if part.call_id in answered_calls: + raise DialectError( + f"duplicate Google function result for {part.call_id!r}" + ) + answered_calls.add(part.call_id) + parts.append( + { + "functionResponse": { + "id": part.call_id, + "name": name, + "response": _tool_result_payload(part), + } + } + ) + else: + raise DialectError(f"unsupported Google content part {type(part).__name__}") + if not parts: + raise DialectError("Google contents must not contain an empty message") + return parts + + +_OPAQUE_PART_FIELDS = frozenset( + { + "text", + "inlineData", + "fileData", + "functionCall", + "functionResponse", + "toolCall", + "toolResponse", + } +) + + +def _validate_opaque_contents(raw: object) -> list[dict[str, JSONValue]]: + if not isinstance(raw, list) or not raw: + raise DialectError("Google opaque continuation has no contents") + copied = copy_json_value(raw, "Google opaque continuation contents") + if not isinstance(copied, list): + raise AssertionError("copied Google contents must remain a list") + result: list[dict[str, JSONValue]] = [] + saw_signature = False + for content_index, content in enumerate(copied): + if not isinstance(content, dict) or set(content) != {"role", "parts"}: + raise DialectError( + f"Google opaque content {content_index} has invalid shape" + ) + if content["role"] not in {"user", "model"}: + raise DialectError( + f"Google opaque content {content_index} has invalid role" + ) + parts = content["parts"] + if not isinstance(parts, list) or not parts: + raise DialectError(f"Google opaque content {content_index} has no parts") + for part_index, part in enumerate(parts): + if not isinstance(part, dict): + raise DialectError( + f"Google opaque part {content_index}.{part_index} is not an object" + ) + unknown = set(part) - ( + _OPAQUE_PART_FIELDS | {"thought", "thoughtSignature"} + ) + if unknown: + raise DialectError( + "Google opaque part has unsupported field(s): " + + ", ".join(sorted(unknown)) + ) + data_fields = set(part) & _OPAQUE_PART_FIELDS + if len(data_fields) != 1: + raise DialectError("Google opaque part must contain one data field") + if "text" in part and not isinstance(part["text"], str): + raise DialectError("Google opaque text part must contain a string") + for key in data_fields - {"text"}: + if not isinstance(part[key], dict): + raise DialectError(f"Google opaque {key} must be an object") + thought = part.get("thought") + if thought is not None and not isinstance(thought, bool): + raise DialectError("Google opaque thought marker must be boolean") + signature = part.get("thoughtSignature") + if signature is not None: + if not isinstance(signature, str) or not signature: + raise DialectError( + "Google opaque thoughtSignature must be a non-empty string" + ) + saw_signature = True + result.append(content) + if not saw_signature: + raise DialectError("Google opaque continuation has no thought signature") + return result + + +def _decode_opaque(value: str) -> list[dict[str, JSONValue]]: + try: + raw = json.loads( + value, + object_pairs_hook=_reject_duplicate_keys, + parse_constant=_reject_json_constant, + ) + except (json.JSONDecodeError, TypeError, ValueError) as exc: + raise DialectError("invalid Google opaque continuation JSON") from exc + if not isinstance(raw, dict) or set(raw) != {"version", "contents"}: + raise DialectError("invalid Google opaque continuation envelope") + if raw.get("version") != _OPAQUE_VERSION: + raise DialectError("unsupported Google opaque continuation version") + return _validate_opaque_contents(raw.get("contents")) + + +def _call_names_from_contents( + contents: Sequence[Mapping[str, JSONValue]], +) -> dict[str, str]: + names: dict[str, str] = {} + for content in contents: + parts = content.get("parts") + if not isinstance(parts, list): + continue + for part in parts: + if not isinstance(part, dict): + continue + call = part.get("functionCall") + if not isinstance(call, dict): + continue + call_id = call.get("id") + name = call.get("name") + if isinstance(call_id, str) and call_id and isinstance(name, str) and name: + if call_id in names: + raise DialectError(f"duplicate Google function call id {call_id!r}") + names[call_id] = name + return names + + +def _input_to_wire( + request_input: ChatInput, opaque: str | None +) -> tuple[dict[str, JSONValue] | None, list[JSONValue]]: + system_parts: list[JSONValue] = [] + opaque_contents = _decode_opaque(opaque) if opaque is not None else [] + contents: list[JSONValue] = list(opaque_contents) + call_names = _call_names_from_contents(opaque_contents) + answered_calls: set[str] = set() + saw_conversation = False + for message in request_input.messages: + if message.name is not None: + raise DialectError("Google contents have no ChatMessage.name field") + if message.role in {"system", "developer"}: + if saw_conversation: + raise DialectError( + "Google system/developer messages must precede conversation turns" + ) + if any(not isinstance(part, TextPart) for part in message.content): + raise DialectError("Google systemInstruction is text-only") + parts = _content_parts(message, call_names, answered_calls) + system_parts.extend(parts) + continue + saw_conversation = True + parts = _content_parts(message, call_names, answered_calls) + role = "model" if message.role == "assistant" else "user" + contents.append({"role": role, "parts": parts}) + system = {"parts": system_parts} if system_parts else None + return system, contents + + +def _verification_input_to_wire( + request_input: ChatInput, opaque: str | None +) -> tuple[dict[str, JSONValue] | None, list[JSONValue]]: + """Independently derive the expected input for request-audit verification.""" + + opaque_contents = _decode_opaque(opaque) if opaque is not None else [] + contents: list[JSONValue] = [ + copy_json_value(item, "verified Google opaque content") + for item in opaque_contents + ] + system_parts: list[JSONValue] = [] + call_names = _call_names_from_contents(opaque_contents) + answered: set[str] = set() + saw_conversation = False + + for message in request_input.messages: + if message.name is not None: + raise DialectError("Google contents have no ChatMessage.name field") + is_instruction = message.role in {"system", "developer"} + if is_instruction and saw_conversation: + raise DialectError( + "Google system/developer messages must precede conversation turns" + ) + if not is_instruction: + saw_conversation = True + + expected_parts: list[JSONValue] = [] + for part in message.content: + if isinstance(part, TextPart): + if message.role == "tool": + raise DialectError( + "Google tool messages may contain only tool results" + ) + expected_parts.append({"text": part.text}) + continue + if isinstance(part, ImagePart): + if message.role == "tool" or is_instruction: + raise DialectError("Google instruction/tool messages are text-only") + if part.detail is not None: + raise DialectError( + "Google image parts have no per-image detail field" + ) + if part.url is not None: + file_data: dict[str, JSONValue] = {"fileUri": part.url} + if part.media_type is not None: + file_data["mimeType"] = part.media_type + expected_parts.append({"fileData": file_data}) + else: + if not part.data or not part.media_type: + raise DialectError( + "Google inline images require data and MIME type" + ) + expected_parts.append( + { + "inlineData": { + "data": part.data, + "mimeType": part.media_type, + } + } + ) + continue + if isinstance(part, ToolCallPart): + if message.role != "assistant": + raise DialectError( + "Google historical function calls require assistant role" + ) + if part.call_id in call_names: + raise DialectError( + f"duplicate Google function call id {part.call_id!r}" + ) + call_names[part.call_id] = part.name + expected_parts.append( + { + "functionCall": { + "id": part.call_id, + "name": part.name, + "args": copy_json_value( + part.arguments, "verified function arguments" + ), + } + } + ) + continue + if isinstance(part, ToolResultPart): + if message.role != "tool": + raise DialectError("Google function results require tool role") + if part.call_id not in call_names: + raise DialectError( + f"Google function result {part.call_id!r} has no prior call" + ) + if part.call_id in answered: + raise DialectError( + f"duplicate Google function result for {part.call_id!r}" + ) + answered.add(part.call_id) + raw_result = copy_json_value( + part.result, "verified Google function response" + ) + response: JSONValue + if part.is_error: + response = {"error": raw_result} + elif isinstance(raw_result, dict): + response = raw_result + else: + response = {"result": raw_result} + expected_parts.append( + { + "functionResponse": { + "id": part.call_id, + "name": call_names[part.call_id], + "response": response, + } + } + ) + continue + raise DialectError(f"unsupported Google content part {type(part).__name__}") + + if not expected_parts: + raise DialectError("Google contents must not contain an empty message") + if is_instruction: + if any(not isinstance(part, TextPart) for part in message.content): + raise DialectError("Google systemInstruction is text-only") + system_parts.extend(expected_parts) + else: + role = "model" if message.role == "assistant" else "user" + contents.append({"role": role, "parts": expected_parts}) + + system = {"parts": system_parts} if system_parts else None + return system, contents + + +def _structured_output(req: Request, text: str) -> StructuredOutput | None: + if req.structured_output.format is None: + return None + try: + value = json.loads( + text, + object_pairs_hook=_reject_duplicate_keys, + parse_constant=_reject_json_constant, + ) + return StructuredOutput(cast(JSONValue, value)) + except (json.JSONDecodeError, TypeError, ValueError) as exc: + raise OutputContractError("Google structured output is not valid JSON") from exc + + +def _usage(raw: object) -> UsageStats | None: + if raw is None: + return None + if not isinstance(raw, Mapping): + raise OutputContractError("usageMetadata must be an object") + prompt = _nonnegative_int(_get(raw, "promptTokenCount"), "usage.promptTokenCount") + output = _nonnegative_int( + _get(raw, "candidatesTokenCount"), "usage.candidatesTokenCount" + ) + reported_total = _optional_nonnegative_int( + _get(raw, "totalTokenCount"), "usage.totalTokenCount" + ) + total = reported_total if reported_total is not None else prompt + output + return UsageStats( + input_tokens=prompt, + output_tokens=output, + total_tokens=total, + reasoning_tokens=( + _nonnegative_int( + _get(raw, "thoughtsTokenCount"), "usage.thoughtsTokenCount" + ) + if _get(raw, "thoughtsTokenCount") is not None + else None + ), + cached_tokens=( + _nonnegative_int( + _get(raw, "cachedContentTokenCount"), "usage.cachedContentTokenCount" + ) + if _get(raw, "cachedContentTokenCount") is not None + else None + ), + reported_total_tokens=reported_total, + ) + + +def _logprobs( + raw: object, path: str, requested_top_k: int +) -> tuple[tuple[TokenLogprob, ...], tuple[tuple[TopKEntry, ...], ...]]: + chosen = _sequence(_get(raw, "chosenCandidates"), f"{path}.chosenCandidates") + raw_tops = _get(raw, "topCandidates") + tops = _sequence(raw_tops, f"{path}.topCandidates") if raw_tops is not None else () + if requested_top_k > 0 and len(chosen) != len(tops): + raise OutputContractError("Google chosen/top logprob positions are misaligned") + if requested_top_k == 0 and tops and len(chosen) != len(tops): + raise OutputContractError("Google chosen/top logprob positions are misaligned") + sampled: list[TokenLogprob] = [] + alternatives: list[tuple[TopKEntry, ...]] = [] + for index, item in enumerate(chosen): + sampled.append( + TokenLogprob( + _required_string(_get(item, "token"), f"{path}.chosen[{index}].token"), + _finite_number( + _get(item, "logProbability"), + f"{path}.chosen[{index}].logProbability", + ), + _optional_nonnegative_int( + _get(item, "tokenId"), f"{path}.chosen[{index}].tokenId" + ), + ) + ) + if not tops: + continue + entries = _sequence(_get(tops[index], "candidates"), f"{path}.top[{index}]") + if len(entries) > requested_top_k and requested_top_k > 0: + raise OutputContractError( + "Google returned more top logprobs than requested" + ) + alternatives.append( + tuple( + TopKEntry( + _required_string(_get(entry, "token"), f"{path}.top.token"), + _finite_number( + _get(entry, "logProbability"), f"{path}.top.logprob" + ), + _optional_nonnegative_int( + _get(entry, "tokenId"), f"{path}.top.tokenId" + ), + ) + for entry in entries + ) + ) + return tuple(sampled), tuple(alternatives) + + +def _citations(raw: object, path: str) -> list[Citation]: + if raw is None: + return [] + if not isinstance(raw, Mapping): + raise OutputContractError(f"{path} must be an object") + result: list[Citation] = [] + for index, source in enumerate( + _sequence(_get(raw, "citationSources", []), f"{path}.citationSources") + ): + uri = _get(source, "uri") + if uri is None: + raise OutputContractError(f"{path}[{index}].uri is absent") + result.append(Citation(_required_string(uri, f"{path}[{index}].uri"))) + return result + + +def _url_context_citations(raw: object, path: str) -> list[Citation]: + if raw is None: + return [] + if not isinstance(raw, Mapping): + raise OutputContractError(f"{path} must be an object") + result: list[Citation] = [] + for index, item in enumerate( + _sequence(_get(raw, "urlMetadata", []), f"{path}.urlMetadata") + ): + url = _get(item, "retrievedUrl") + if url is None: + raise OutputContractError(f"{path}.urlMetadata[{index}] omitted its URL") + result.append(Citation(_required_string(url, f"{path}.urlMetadata[{index}]"))) + return result + + +def _grounding(raw: object, path: str) -> GroundingMetadata | None: + if raw is None: + return None + if not isinstance(raw, Mapping): + raise OutputContractError(f"{path} must be an object") + chunks: list[GroundingChunk] = [] + for index, chunk in enumerate( + _sequence(_get(raw, "groundingChunks", []), f"{path}.groundingChunks") + ): + sources = [ + _get(chunk, name) + for name in ("web", "retrievedContext", "maps", "image") + if _get(chunk, name) is not None + ] + if len(sources) != 1: + raise OutputContractError(f"{path}.groundingChunks[{index}] is ambiguous") + source = sources[0] + uri = ( + _get(source, "sourceUri", _get(source, "imageUri")) + if _get(chunk, "image") is not None + else _get(source, "uri") + ) + if uri is None: + raise OutputContractError( + f"{path}.groundingChunks[{index}] has no representable URI" + ) + chunks.append( + GroundingChunk( + _required_string(uri, f"{path}.groundingChunks[{index}].uri"), + _optional_string( + _get(source, "title"), f"{path}.groundingChunks[{index}].title" + ), + ) + ) + entry = _get(raw, "searchEntryPoint") + rendered = _optional_string( + _get(entry, "renderedContent") if entry is not None else None, + f"{path}.searchEntryPoint.renderedContent", + ) + return GroundingMetadata(tuple(chunks), rendered) + + +def _opaque_history( + input_contents: Sequence[Mapping[str, JSONValue]], + output_content: Mapping[str, JSONValue], +) -> str | None: + history: list[JSONValue] = [ + copy_json_value(item, "Google input history") for item in input_contents + ] + history.append(copy_json_value(output_content, "Google output history")) + saw_signature = False + for content in history: + if not isinstance(content, dict): + continue + parts = content.get("parts") + if not isinstance(parts, list): + continue + if any( + isinstance(part, dict) + and isinstance(part.get("thoughtSignature"), str) + and bool(part.get("thoughtSignature")) + for part in parts + ): + saw_signature = True + break + if not saw_signature: + return None + return json.dumps( + {"version": _OPAQUE_VERSION, "contents": history}, + ensure_ascii=False, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ) + + +class GoogleGenAIDialect: + """Executable adapter for Gemini native generateContent REST endpoints.""" + + dialect_id = "google_genai" + connection_family = "async_http_json" + capability_decisions = CAPABILITY_DECISIONS + output_contract = OUTPUT_CONTRACT + CAPABILITIES = frozenset( + { + Capability.Chat, + Capability.FunctionCalling, + Capability.ServerTools, + Capability.Reasoning, + Capability.ReasoningEffort, + Capability.SampledLogprobs, + Capability.StructuredOutput, + Capability.TopKLogprobs, + } + ) + + def __init__(self, connection: object, requested_model_id: str): + if not isinstance(connection, AsyncHTTPJSONConnection): + raise TypeError("google_genai requires AsyncHTTPJSONConnection") + if not requested_model_id: + raise ValueError("requested_model_id must not be empty") + model_name = requested_model_id.removeprefix("models/") + if not model_name or "/" in model_name: + raise ValueError( + "requested_model_id must be a Gemini model id or models/" + ) + self._connection = connection + self._requested_model_id = requested_model_id + self._model_name = model_name + base_path = connection.client.base_url.path.rstrip("/") + self._version_prefix = "" if base_path.endswith("/v1beta") else "v1beta/" + + @property + def capabilities(self) -> frozenset[Capability]: + return self.CAPABILITIES + + def validate_request( + self, req: Request, audit: RequestAudit, plan: RuntimePlanView + ) -> None: + audit.require_matches_request(req) + del plan + has_named_message = isinstance(req.input, ChatInput) and any( + message.name is not None for message in req.input.messages + ) + for path in audit.active_paths: + if path == "input.chat" and has_named_message: + audit.rejected(path, "Google contents have no ChatMessage.name field") + elif path.startswith("input.completion"): + audit.rejected(path, "google_genai requires ChatInput") + elif path == "input.modality.image.detail": + audit.rejected(path, "Google images have no per-image detail field") + elif path == "scoring.input_scoring": + audit.rejected(path, "Gemini cannot score input tokens") + elif path == "sampling.temperature" and not ( + 0 <= cast(float, req.sampling.temperature) <= 2 + ): + audit.rejected(path, "Google temperature must be between 0 and 2") + elif path == "sampling.top_p" and not ( + 0 <= cast(float, req.sampling.top_p) <= 1 + ): + audit.rejected(path, "Google top_p must be between 0 and 1") + elif path == "sampling.top_k" and cast(int, req.sampling.top_k) < 1: + audit.rejected(path, "Google top_k must be positive") + elif ( + path == "sampling.max_tokens" and cast(int, req.sampling.max_tokens) < 1 + ): + audit.rejected(path, "Google max_tokens must be positive") + elif ( + path == "sampling.stop" + and len(cast(tuple[str, ...], req.sampling.stop)) > 5 + ): + audit.rejected(path, "Google supports at most five stop sequences") + elif path == "scoring.top_logprobs" and req.scoring.top_logprobs > 20: + audit.rejected(path, "Gemini supports at most 20 top logprobs") + elif path == "reasoning.effort" and ( + not isinstance(req.reasoning.effort, str) + or req.reasoning.effort.lower() not in _THINKING_LEVELS + ): + audit.rejected(path, "unsupported Google thinking level") + elif path == "reasoning.summary" and req.reasoning.summary not in { + "none", + "auto", + }: + audit.rejected(path, "Google supports only summary='auto'") + elif path == "tools.parallel": + audit.rejected(path, "Google has no per-request parallel tool switch") + elif path == "tools.choice" and not req.tools.functions: + audit.rejected(path, "Google tool choice requires function tools") + elif path == "tools.hosted" and any( + tool.kind not in _HOSTED_TOOL_KINDS for tool in req.tools.hosted + ): + audit.rejected(path, "unsupported Google hosted tool kind") + elif path == "session.previous_response_id": + audit.rejected(path, "Gemini responseId is not a continuation handle") + elif ( + path == "structured_output.format" + and req.structured_output.format + not in { + "json_object", + "json_schema", + } + ): + audit.rejected(path, "unsupported Google structured output format") + elif path == "structured_output.schema" and ( + req.structured_output.format != "json_schema" + ): + audit.rejected( + path, "Google response schema requires format='json_schema'" + ) + elif path == "structured_output.name": + audit.rejected(path, "Google response JSON schemas have no name field") + elif path == "structured_output.strict": + audit.rejected(path, "Google exposes no JSON-schema strict switch") + elif path.startswith("dialect_options."): + key = path.removeprefix("dialect_options.") + if not key: + audit.rejected(path, "dialect option keys must be non-empty") + elif key in _IR_OWNED_KEYS: + audit.rejected(path, f"{key!r} must use its provider-neutral field") + + def prepare(self, req: Request, audit: RequestAudit) -> PreparedRequest: + audit.require_matches_request(req) + audit.raise_rejections() + if not isinstance(req.input, ChatInput): + raise DialectError("google_genai requires ChatInput") + active = frozenset(audit.active_paths) + + def consume(path: str, *evidence: WireEvidence) -> None: + if path in active and path not in audit.decisions: + audit.consumed(path, *evidence) + + opaque = ( + req.session.opaque_continuation.value + if req.session.opaque_continuation is not None + else None + ) + input_verifier = _InputVerifier.from_request(req.input, opaque) + system, contents = _input_to_wire(req.input, opaque) + body: dict[str, JSONValue] = {"contents": contents} + if system is not None: + body["systemInstruction"] = system + for path in ( + "input.chat", + "input.modality.text", + "input.modality.image", + "input.modality.image.media_type", + "input.modality.tool_call", + "input.modality.tool_result", + "input.modality.tool_result.is_error", + ): + consume(path, input_verifier) + if opaque is not None: + consume("session.opaque_continuation", input_verifier) + + generation: dict[str, JSONValue] = {} + sampling = req.sampling + fields = { + "temperature": "temperature", + "top_p": "topP", + "top_k": "topK", + "max_tokens": "maxOutputTokens", + "stop": "stopSequences", + "seed": "seed", + "frequency_penalty": "frequencyPenalty", + "presence_penalty": "presencePenalty", + "n": "candidateCount", + } + for source, destination in fields.items(): + value = getattr(sampling, source) + default = getattr(type(sampling)(), source) + if value != default: + generation[destination] = ( + list(value) if isinstance(value, tuple) else value + ) + consume( + f"sampling.{source}", + WireObservation(("generationConfig", destination)), + ) + + if req.scoring.sampled_logprobs: + generation["responseLogprobs"] = True + consume( + "scoring.sampled_logprobs", + WireObservation(("generationConfig", "responseLogprobs")), + ) + if req.scoring.top_logprobs > 0: + generation["logprobs"] = req.scoring.top_logprobs + consume( + "scoring.top_logprobs", + WireObservation(("generationConfig", "logprobs")), + ) + + thinking: dict[str, JSONValue] = {} + if req.reasoning.effort is not None: + thinking["thinkingLevel"] = req.reasoning.effort.upper() + consume("reasoning.effort", _ThinkingEffortVerifier(req.reasoning.effort)) + if req.reasoning.budget_tokens is not None: + thinking["thinkingBudget"] = req.reasoning.budget_tokens + consume( + "reasoning.budget_tokens", + WireObservation( + ("generationConfig", "thinkingConfig", "thinkingBudget") + ), + ) + if req.reasoning.summary == "none": + audit.noop( + "reasoning.summary", "omitting includeThoughts requests no thoughts" + ) + elif req.reasoning.summary == "auto": + thinking["includeThoughts"] = True + consume( + "reasoning.summary", + _ReasoningSummaryVerifier(), + ) + if thinking: + generation["thinkingConfig"] = thinking + + if req.structured_output.format is not None: + generation["responseMimeType"] = "application/json" + consume( + "structured_output.format", + _StructuredFormatVerifier(req.structured_output.format), + ) + if req.structured_output.schema is not None: + generation["responseJsonSchema"] = dict(req.structured_output.schema) + consume( + "structured_output.schema", + WireObservation(("generationConfig", "responseJsonSchema")), + ) + if generation: + body["generationConfig"] = generation + + tools: list[JSONValue] = [] + if req.tools.functions: + tools.append( + { + "functionDeclarations": [ + _function_declaration(item) for item in req.tools.functions + ] + } + ) + consume("tools.functions", _FunctionsVerifier(req.tools.functions)) + if req.tools.hosted: + tools.extend(_hosted_tool(tool) for tool in req.tools.hosted) + consume("tools.hosted", _HostedToolsVerifier(req.tools.hosted)) + if tools: + body["tools"] = tools + if req.tools.choice is not None: + tool_config = _tool_choice(req.tools.choice) + allowed = _get( + _get(tool_config, "functionCallingConfig"), + "allowedFunctionNames", + [], + ) + declared = { + _get(_function_declaration(item), "name") + for item in req.tools.functions + } + if any(name not in declared for name in _sequence(allowed, "tool choice")): + raise DialectError("Google tool choice names an undeclared function") + body["toolConfig"] = tool_config + consume("tools.choice", _ToolChoiceVerifier(req.tools.choice)) + if req.tools.hosted: + existing_tool_config = body.get("toolConfig") + if existing_tool_config is None: + tool_config: dict[str, JSONValue] = {} + elif isinstance(existing_tool_config, Mapping): + tool_config = dict(existing_tool_config) + else: + raise AssertionError("Google toolConfig must remain an object") + tool_config["includeServerSideToolInvocations"] = True + body["toolConfig"] = tool_config + + options = req.dialect_options + if options is not None: + if not options.values: + audit.noop( + "dialect_options", "empty matching options have no wire semantics" + ) + for key, value in options.values.items(): + path = f"dialect_options.{key}" + if path in audit.decisions: + continue + body[key] = value + audit.consumed(path, _RootOptionVerifier(key, value)) + + method = "streamGenerateContent" if req.scheduling.stream else "generateContent" + operation = ( + f"{self._version_prefix}models/{quote(self._model_name, safe='')}:{method}" + ) + if req.scheduling.stream: + operation += "?alt=sse" + consume( + "scheduling.stream", + _OperationVerifier( + self._model_name, + True, + self._version_prefix, + ), + ) + return PreparedRequest(operation, body, _GoogleContext(req)) + + def _lift(self, raw: object, context: _GoogleExecutionContext) -> Response: + req = context.request + candidates = _sequence(_get(raw, "candidates"), "response.candidates") + if len(candidates) != req.sampling.n: + raise OutputContractError( + f"expected {req.sampling.n} Google candidates, " + f"received {len(candidates)}" + ) + texts: list[str] = [] + finish_reasons: list[str] = [] + reasoning: list[ReasoningOutput | None] = [] + calls: list[FunctionToolCall] = [] + server_calls: dict[str, tuple[str, Mapping[str, JSONValue]]] = {} + server_results: dict[str, tuple[str, JSONValue]] = {} + citations: list[Citation] = [] + groundings: list[GroundingMetadata] = [] + sampled: list[TokenLogprob] = [] + top: list[tuple[TopKEntry, ...]] = [] + for index, candidate in enumerate(candidates): + candidate_index = _get(candidate, "index", index) + if candidate_index != index: + raise OutputContractError("Google candidate indexes are not ordered") + content = _get(candidate, "content") + if content is None: + content_mapping: Mapping[str, JSONValue] | None = None + parts: Sequence[object] = () + elif isinstance(content, Mapping): + content_mapping = cast(Mapping[str, JSONValue], content) + unknown_content = set(content_mapping) - {"role", "parts"} + if unknown_content: + raise OutputContractError( + "Google response content has unsupported field(s): " + + ", ".join(sorted(unknown_content)) + ) + role = content_mapping.get("role") + if role is not None and role != "model": + raise OutputContractError( + "Google response content role is not model" + ) + parts = _sequence( + content_mapping.get("parts"), f"candidates[{index}].parts" + ) + else: + raise OutputContractError("Google response content must be an object") + text_parts: list[str] = [] + thought_text: list[str] = [] + for part_index, raw_part in enumerate(parts): + if not isinstance(raw_part, Mapping): + raise OutputContractError("Google response part must be an object") + part = cast(Mapping[str, JSONValue], raw_part) + unknown_part = set(part) - ( + _OPAQUE_PART_FIELDS | {"thought", "thoughtSignature"} + ) + if unknown_part: + raise OutputContractError( + "Google response part has unsupported field(s): " + + ", ".join(sorted(unknown_part)) + ) + thought_marker = part.get("thought") + if thought_marker is not None and not isinstance(thought_marker, bool): + raise OutputContractError( + "Google response thought marker must be boolean" + ) + signature = part.get("thoughtSignature") + if signature is not None and ( + not isinstance(signature, str) or not signature + ): + raise OutputContractError( + "Google response thoughtSignature must be a non-empty string" + ) + known = [ + name + for name in ("text", "functionCall", "toolCall", "toolResponse") + if name in part + ] + if part.get("thought") is True: + if known != ["text"] or not isinstance(part.get("text"), str): + raise OutputContractError( + "Google thought part must contain exactly one text field" + ) + thought_text.append(cast(str, part["text"])) + continue + if len(known) != 1: + raise OutputContractError( + f"candidates[{index}].parts[{part_index}] has unsupported shape" + ) + if known[0] == "text": + text_parts.append(_string(part["text"], "response text")) + elif known[0] == "functionCall": + if not req.tools.functions: + raise OutputContractError( + "Google returned an unrequested function call" + ) + raw_call = part["functionCall"] + if not isinstance(raw_call, Mapping): + raise OutputContractError("functionCall must be an object") + call = cast(Mapping[str, object], raw_call) + arguments = copy_json_value( + _get(call, "args", {}), "functionCall.args" + ) + if not isinstance(arguments, dict): + raise OutputContractError("functionCall.args must be an object") + calls.append( + FunctionToolCall( + _required_string(_get(call, "id"), "functionCall.id"), + _required_string(_get(call, "name"), "functionCall.name"), + arguments, + ) + ) + elif known[0] == "toolCall": + if not req.tools.hosted: + raise OutputContractError( + "Google returned an unrequested server tool call" + ) + raw_item = part["toolCall"] + if not isinstance(raw_item, Mapping): + raise OutputContractError("toolCall must be an object") + item = cast(Mapping[str, object], raw_item) + tool_id = _required_string(_get(item, "id"), "toolCall.id") + if tool_id in server_calls: + raise OutputContractError( + f"duplicate Google server tool call id {tool_id!r}" + ) + raw_args = copy_json_value(_get(item, "args", {}), "toolCall.args") + if not isinstance(raw_args, dict): + raise OutputContractError("toolCall.args must be an object") + server_calls[tool_id] = ( + _required_string(_get(item, "toolType"), "toolCall.toolType"), + raw_args, + ) + else: + if not req.tools.hosted: + raise OutputContractError( + "Google returned an unrequested server tool response" + ) + raw_item = part["toolResponse"] + if not isinstance(raw_item, Mapping): + raise OutputContractError("toolResponse must be an object") + item = cast(Mapping[str, object], raw_item) + tool_id = _required_string(_get(item, "id"), "toolResponse.id") + if tool_id in server_results: + raise OutputContractError( + f"duplicate Google server tool response id {tool_id!r}" + ) + tool_result = copy_json_value( + _get(item, "response"), "toolResponse.response" + ) + if not isinstance(tool_result, dict): + raise OutputContractError( + "toolResponse.response must be an object" + ) + server_results[tool_id] = ( + _required_string( + _get(item, "toolType"), "toolResponse.toolType" + ), + tool_result, + ) + text = "".join(text_parts) + texts.append(text) + finish_reason = _required_string( + _get(candidate, "finishReason"), "candidate.finishReason" + ) + if finish_reason == "FINISH_REASON_UNSPECIFIED": + raise OutputContractError("Google candidate is not terminal") + finish_reasons.append(finish_reason) + opaque = ( + _opaque_history(context.input_contents, content_mapping) + if content_mapping is not None and parts + else None + ) + reasoning.append( + ReasoningOutput( + text="\n".join(thought_text) or None, opaque_roundtrip=opaque + ) + if thought_text or opaque is not None + else None + ) + citations.extend( + _citations(_get(candidate, "citationMetadata"), "citationMetadata") + ) + citations.extend( + _url_context_citations( + _get(candidate, "urlContextMetadata"), "urlContextMetadata" + ) + ) + grounding = _grounding( + _get(candidate, "groundingMetadata"), "groundingMetadata" + ) + if grounding is not None: + groundings.append(grounding) + raw_logprobs = _get(candidate, "logprobsResult") + if req.scoring.sampled_logprobs: + if raw_logprobs is None: + raise OutputContractError("Google reply omitted requested logprobs") + chosen, alternatives = _logprobs( + raw_logprobs, + "candidate.logprobsResult", + req.scoring.top_logprobs, + ) + sampled.extend(chosen) + top.extend(alternatives) + if req.reasoning.summary == "auto" and any( + item is None or not item.text for item in reasoning + ): + raise OutputContractError( + "Google reply omitted requested thoughts for one or more candidates" + ) + if req.sampling.n > 1 and (citations or groundings): + raise OutputContractError( + "Google singular citation/grounding metadata cannot represent choices" + ) + if len(groundings) > 1: + raise OutputContractError( + "multiple candidates returned singular grounding metadata" + ) + text = texts[0] if len(texts) == 1 else "" + server_uses: list[ServerToolUse] = [] + for tool_id, (tool_type, tool_input) in server_calls.items(): + result_entry = server_results.pop(tool_id, None) + if result_entry is not None and result_entry[0] != tool_type: + raise OutputContractError( + f"Google server tool type changed for {tool_id!r}" + ) + server_uses.append( + ServerToolUse( + tool_type=tool_type, + tool_use_id=tool_id, + input=tool_input, + result=None if result_entry is None else result_entry[1], + ) + ) + if server_results: + raise OutputContractError( + "Google returned a server tool response without its call" + ) + return Response( + texts=tuple(texts), + reasoning=tuple(reasoning) if any(reasoning) else None, + finish_reasons=tuple(finish_reasons), + tool_calls=tuple(calls) or None, + server_tool_uses=tuple(server_uses) or None, + structured_output=_structured_output(req, text), + logprobs=tuple(sampled) if req.scoring.sampled_logprobs else None, + top_logprobs=tuple(top) if req.scoring.top_logprobs > 0 else None, + citations=tuple(citations) or None, + grounding=groundings[0] if groundings else None, + usage=_usage(_get(raw, "usageMetadata")), + request_params=context.request_params, + response_model=_optional_string( + _get(raw, "modelVersion"), "response.modelVersion" + ), + ) + + async def _lift_stream( + self, response: httpx.Response, context: _GoogleExecutionContext + ) -> Response: + chunks: list[object] = [] + data_lines: list[str] = [] + + def flush_event() -> None: + if not data_lines: + return + payload = "\n".join(data_lines) + data_lines.clear() + chunks.append(_strict_json_loads(payload, "Google stream event")) + + async for line in response.aiter_lines(): + if not line: + flush_event() + continue + if line.startswith(":"): + continue + field, separator, value = line.partition(":") + if not separator: + raise OutputContractError("Google stream emitted a malformed SSE line") + if field == "data": + data_lines.append(value.removeprefix(" ")) + elif field not in {"event", "id", "retry"}: + raise OutputContractError( + f"Google stream emitted unsupported SSE field {field!r}" + ) + flush_event() + if not chunks: + raise OutputContractError("Google stream emitted no response chunks") + return self._merge_stream(chunks, context) + + def _merge_stream( + self, chunks: list[object], context: _GoogleExecutionContext + ) -> Response: + # Each stream event carries candidate deltas plus cumulative/terminal + # metadata. Merge only provider-defined additive content and require + # finish/usage metadata to agree when repeated. + by_index: dict[int, dict[str, object]] = {} + usage: object = None + model_version: object = None + for chunk_index, chunk in enumerate(chunks): + if _get(chunk, "usageMetadata") is not None: + usage = _get(chunk, "usageMetadata") + if _get(chunk, "modelVersion") is not None: + current_model = _get(chunk, "modelVersion") + if model_version is not None and current_model != model_version: + raise OutputContractError( + "Google stream changed modelVersion between events" + ) + model_version = current_model + for raw_candidate in _sequence( + _get(chunk, "candidates", []), f"stream[{chunk_index}].candidates" + ): + index = _get( + raw_candidate, + "index", + 0 if context.request.sampling.n == 1 else None, + ) + if isinstance(index, bool) or not isinstance(index, int) or index < 0: + raise OutputContractError( + "Google stream candidate index is invalid" + ) + state = by_index.setdefault(index, {"parts": []}) + content = _get(raw_candidate, "content") + if content is not None: + role = _get(content, "role") + if role is not None and role != "model": + raise OutputContractError( + "Google stream candidate role is not model" + ) + state["parts"] = cast(list[object], state["parts"]) + list( + _sequence(_get(content, "parts", []), "stream candidate parts") + ) + finish_reason = _get(raw_candidate, "finishReason") + if finish_reason is not None: + previous = state.get("finishReason") + if previous is not None and previous != finish_reason: + raise OutputContractError( + "Google stream changed a candidate finishReason" + ) + state["finishReason"] = finish_reason + for name in ("citationMetadata", "groundingMetadata"): + value = _get(raw_candidate, name) + if value is not None: + state[name] = value + raw_logprobs = _get(raw_candidate, "logprobsResult") + if raw_logprobs is not None: + if not isinstance(raw_logprobs, Mapping): + raise OutputContractError( + "Google stream logprobsResult must be an object" + ) + accumulated_raw = state.get("logprobsResult") + if accumulated_raw is None: + accumulated: dict[str, object] = { + "chosenCandidates": [], + "topCandidates": [], + } + state["logprobsResult"] = accumulated + elif isinstance(accumulated_raw, dict): + accumulated = cast(dict[str, object], accumulated_raw) + else: + raise AssertionError("Google logprobs accumulator changed type") + for field_name in ("chosenCandidates", "topCandidates"): + value = _get(raw_logprobs, field_name) + if value is not None: + target = accumulated.get(field_name) + if not isinstance(target, list): + raise AssertionError( + "Google logprobs accumulator changed shape" + ) + cast(list[object], target).extend( + _sequence( + value, + f"stream logprobsResult.{field_name}", + ) + ) + candidates: list[JSONValue] = [] + for index in sorted(by_index): + state = by_index[index] + candidate: dict[str, JSONValue] = { + "index": index, + "content": { + "role": "model", + "parts": cast(list[JSONValue], state.pop("parts")), + }, + } + for key, value in state.items(): + candidate[key] = copy_json_value(value, f"stream.{key}") + candidates.append(candidate) + merged: dict[str, JSONValue] = {"candidates": candidates} + if usage is not None: + merged["usageMetadata"] = copy_json_value(usage, "stream usage") + if model_version is not None: + merged["modelVersion"] = copy_json_value(model_version, "stream model") + return self._lift(merged, context) + + async def execute(self, prepared: PreparedRequest) -> Response: + context = prepared.context + if not isinstance(context, _GoogleContext): + raise DialectError("google_genai received invalid prepared context") + expected_suffix = ( + ":streamGenerateContent?alt=sse" + if context.request.scheduling.stream + else ":generateContent" + ) + if not prepared.operation.endswith(expected_suffix): + raise DialectError("google_genai received the wrong prepared operation") + body = prepared.thaw_body() + raw_contents = body.get("contents") + if not isinstance(raw_contents, list): + raise DialectError("prepared Google request has invalid contents") + copied_contents = copy_json_value(raw_contents, "prepared Google contents") + if not isinstance(copied_contents, list) or not all( + isinstance(item, dict) for item in copied_contents + ): + raise DialectError("prepared Google request has invalid content items") + headers = {"Content-Type": "application/json"} + credential = self._connection.credential + if credential is not None: + headers["x-goog-api-key"] = credential + request = self._connection.client.build_request( + "POST", + prepared.operation, + headers=headers, + json=body, + ) + actual_body = _strict_json_loads( + request.content.decode("utf-8"), "final Google request body" + ) + if not isinstance(actual_body, dict): + raise DialectError("final Google request body must be an object") + actual_contents = actual_body.get("contents") + if not isinstance(actual_contents, list) or not all( + isinstance(item, dict) for item in actual_contents + ): + raise DialectError("final Google request body has invalid contents") + request_params: dict[str, JSONValue] = { + "method": request.method, + "path": request.url.path, + "body": { + key: value + for key, value in actual_body.items() + if key not in {"contents", "systemInstruction"} + }, + } + actual_alt = request.url.params.get("alt") + if actual_alt is not None: + request_params["query"] = {"alt": actual_alt} + execution_context = _GoogleExecutionContext( + request=context.request, + request_params=request_params, + input_contents=tuple(cast(list[Mapping[str, JSONValue]], actual_contents)), + ) + response = await self._connection.client.send(request) + response.raise_for_status() + if context.request.scheduling.stream: + return await self._lift_stream(response, execution_context) + payload = _strict_json_loads(response.text, "Google response body") + return self._lift(payload, execution_context) + + async def arun(self, req: Request) -> Response: + plan = _LegacyPlan() + validate_request_invariants(req) + validate_runtime_binding_plan(plan, req) + audit = RequestAudit(active_request_leaves(req)) + self.validate_request(req, audit, plan) + audit.raise_rejections() + prepared = self.prepare(req, audit) + audit.finish(prepared) + response = await self.execute(prepared) + self.output_contract.validate(plan, req, response) + return response + + +@dataclass(frozen=True) +class _RootOptionVerifier: + key: str + value: JSONValue + + def __post_init__(self) -> None: + object.__setattr__( + self, + "value", + copy_json_value(self.value, f"Google dialect option {self.key}"), + ) + + def verify(self, body: Mapping[str, JSONValue]) -> str | None: + return ( + None + if self.key in body and body[self.key] == self.value + else f"Google dialect option {self.key!r} changed" + ) + + +@dataclass(frozen=True) +class _ThinkingEffortVerifier: + effort: str + + def verify(self, body: Mapping[str, JSONValue]) -> str | None: + actual = _get( + _get(_get(body, "generationConfig"), "thinkingConfig"), "thinkingLevel" + ) + return None if actual == self.effort.upper() else "thinking level changed" + + +@dataclass(frozen=True) +class _ReasoningSummaryVerifier: + def verify(self, body: Mapping[str, JSONValue]) -> str | None: + actual = _get( + _get(_get(body, "generationConfig"), "thinkingConfig"), + "includeThoughts", + ) + return None if actual is True else "visible-thought request changed" + + +@dataclass(frozen=True) +class _StructuredFormatVerifier: + format: str + + def verify(self, body: Mapping[str, JSONValue]) -> str | None: + generation = _get(body, "generationConfig") + if _get(generation, "responseMimeType") != "application/json": + return "structured output MIME type changed" + has_schema = _get(generation, "responseJsonSchema") is not None + if has_schema != (self.format == "json_schema"): + return "structured output schema presence changed" + return None + + +@dataclass(frozen=True) +class _FunctionsVerifier: + functions: tuple[Mapping[str, JSONValue], ...] + + def __post_init__(self) -> None: + copied: list[Mapping[str, JSONValue]] = [] + for index, function in enumerate(self.functions): + item = copy_json_value(function, f"Google function {index}") + if not isinstance(item, dict): + raise TypeError("Google function declarations must be objects") + copied.append(item) + object.__setattr__(self, "functions", tuple(copied)) + + def verify(self, body: Mapping[str, JSONValue]) -> str | None: + declarations: list[JSONValue] = [] + for function in self.functions: + source = dict(function) + nested = source.get("function") + if source.get("type") == "function" and isinstance(nested, dict): + source = dict(nested) + else: + source.pop("type", None) + declaration: dict[str, JSONValue] = {"name": source.get("name")} + if "description" in source: + declaration["description"] = source["description"] + if "parameters" in source: + declaration["parametersJsonSchema"] = source["parameters"] + declarations.append(declaration) + expected = {"functionDeclarations": declarations} + tools = _get(body, "tools") + return ( + None + if isinstance(tools, Sequence) and expected in tools + else "function tools changed" + ) + + +@dataclass(frozen=True) +class _HostedToolsVerifier: + hosted: tuple[object, ...] + + def verify(self, body: Mapping[str, JSONValue]) -> str | None: + tools = _get(body, "tools") + expected: list[dict[str, JSONValue]] = [] + for item in self.hosted: + kind = getattr(item, "kind", None) + config = getattr(item, "config", None) + if not isinstance(config, Mapping): + return "hosted tool config is invalid" + key = "googleSearch" if kind == "google_search" else "urlContext" + expected.append({key: dict(config)}) + if not isinstance(tools, Sequence) or not all( + item in tools for item in expected + ): + return "hosted tools changed" + if ( + _get(_get(body, "toolConfig"), "includeServerSideToolInvocations") + is not True + ): + return "hosted-tool invocation evidence was disabled" + return None + + +@dataclass(frozen=True) +class _ToolChoiceVerifier: + choice: JSONValue + + def verify(self, body: Mapping[str, JSONValue]) -> str | None: + if isinstance(self.choice, str): + mode = {"auto": "AUTO", "none": "NONE", "required": "ANY"}.get(self.choice) + expected: JSONValue = {"functionCallingConfig": {"mode": mode}} + elif isinstance(self.choice, dict): + nested = self.choice.get("function") + name = ( + nested.get("name") + if isinstance(nested, dict) + else self.choice.get("name") + ) + expected = { + "functionCallingConfig": { + "mode": "ANY", + "allowedFunctionNames": [name], + } + } + else: + return "source tool choice is invalid" + actual = _get(body, "toolConfig") + if not isinstance(actual, Mapping): + return "tool choice changed" + actual_choice = { + key: value + for key, value in actual.items() + if key != "includeServerSideToolInvocations" + } + return None if actual_choice == expected else "tool choice changed" diff --git a/tests/unit/cli/leaderboard/test_session.py b/tests/unit/cli/leaderboard/test_session.py index 4225b878..1b821a74 100644 --- a/tests/unit/cli/leaderboard/test_session.py +++ b/tests/unit/cli/leaderboard/test_session.py @@ -6847,7 +6847,9 @@ def test_infer_args_e2e_yaml(self, tmp_path): # Deterministic mode: dialect-declared seed injection + transparent sampling # =================================================================== class TestDeterministicRequestSeedPolicy: - @pytest.mark.parametrize("dialect_id", ["openai_chat", "openai_completions"]) + @pytest.mark.parametrize( + "dialect_id", ["openai_chat", "openai_completions", "google_genai"] + ) def test_supported_dialects_receive_the_automatic_seed(self, dialect_id): decision = _resolve_deterministic_request_seed( dialect_id=dialect_id, diff --git a/tests/unit/core/models/dialects/test_google_genai.py b/tests/unit/core/models/dialects/test_google_genai.py new file mode 100644 index 00000000..ff7a3a1b --- /dev/null +++ b/tests/unit/core/models/dialects/test_google_genai.py @@ -0,0 +1,914 @@ +"""Tests for the native Google Gemini ``generateContent`` dialect. + +AI-Generated Code - GPT-5.6 (OpenAI) +""" + +import json +from collections.abc import Callable, Mapping +from types import SimpleNamespace +from typing import Any, cast + +import httpx +import pytest + +from sieval.core.models.capabilities import ( + CAPABILITY_KEYS, + DialectCapabilityStatus, + HostedToolsOptions, + ReasoningOptions, + Supported, + TopLogprobsOptions, + Unsupported, +) +from sieval.core.models.connection_factory import AsyncHTTPJSONConnection +from sieval.core.models.dialect import ( + DialectError, + OutputContractError, + PreparedRequest, + RequestAudit, + RequestAuditError, + active_request_leaves, +) +from sieval.core.models.dialect_registry import DIALECT_SPECS +from sieval.core.models.dialects.google_genai import ( + CAPABILITY_DECISIONS, + GoogleGenAIDialect, +) +from sieval.core.models.ir import ( + ChatInput, + ChatMessage, + CompletionInput, + DialectOptions, + HostedToolSpec, + ImagePart, + OpaqueContinuation, + ReasoningParams, + Request, + SamplingParams, + SchedulingParams, + ScoringParams, + SessionParams, + StructuredOutputParams, + TextPart, + ToolCallPart, + ToolParams, + ToolResultPart, +) + + +def _chat(*messages: ChatMessage) -> ChatInput: + return ChatInput(messages or (ChatMessage("user", (TextPart("Hello"),)),)) + + +def _request(**changes: object) -> Request: + values: dict[str, object] = {"input": _chat()} + values.update(changes) + return Request(**cast(Any, values)) + + +def _response( + parts: list[dict[str, object]] | None = None, + *, + finish_reason: object = "STOP", + usage: object | None = None, + model_version: object = "gemini-2.5-flash", + candidate_extra: Mapping[str, object] | None = None, +) -> dict[str, object]: + candidate: dict[str, object] = { + "index": 0, + "content": { + "role": "model", + "parts": parts if parts is not None else [{"text": "Hi"}], + }, + "finishReason": finish_reason, + } + if candidate_extra: + candidate.update(candidate_extra) + return { + "candidates": [candidate], + "usageMetadata": usage + if usage is not None + else { + "promptTokenCount": 4, + "candidatesTokenCount": 2, + "totalTokenCount": 7, + "thoughtsTokenCount": 1, + "cachedContentTokenCount": 2, + }, + "modelVersion": model_version, + } + + +def _connection( + handler: Callable[[httpx.Request], httpx.Response], + *, + base_url: str = "https://generativelanguage.googleapis.com/", + credential: str | None = "google-secret", +) -> AsyncHTTPJSONConnection: + return AsyncHTTPJSONConnection( + httpx.AsyncClient(base_url=base_url, transport=httpx.MockTransport(handler)), + credential, + ) + + +async def _run( + handler: Callable[[httpx.Request], httpx.Response], + request: Request | None = None, + *, + model: str = "gemini-2.5-flash", + base_url: str = "https://generativelanguage.googleapis.com/", + credential: str | None = "google-secret", +): + connection = _connection(handler, base_url=base_url, credential=credential) + try: + return await GoogleGenAIDialect(connection, model).arun(request or _request()) + finally: + await connection.aclose() + + +def _sse(*chunks: Mapping[str, object]) -> bytes: + lines: list[str] = [] + for chunk in chunks: + payload = json.dumps(chunk, separators=(",", ":")) + split_at = payload.find(",", len(payload) // 2) + if split_at < 0: + lines.extend((f"data: {payload}", "")) + else: + lines.extend( + ( + f"data: {payload[: split_at + 1]}", + f"data:{payload[split_at + 1 :]}", + "", + ) + ) + return "\n".join(lines).encode() + + +def _no_io(_: httpx.Request) -> httpx.Response: + raise AssertionError("request rejection must happen before HTTP I/O") + + +class TestCapabilities: + def test_decision_row_and_registry_are_complete(self) -> None: + assert set(CAPABILITY_DECISIONS) == set(CAPABILITY_KEYS) + assert isinstance(CAPABILITY_DECISIONS["input_scoring"], Unsupported) + for key in ( + "sampled_logprobs", + "top_logprobs", + "reasoning", + "function_tools", + "hosted_tools", + "structured_output", + "opaque_continuation", + "multimodal_input", + ): + assert isinstance(CAPABILITY_DECISIONS[key], Supported) + + spec = DIALECT_SPECS["google_genai"] + assert spec.implementation_status.value == "active" + assert spec.request_seed_support.value == "supported" + assert spec.capability_outcomes["reasoning"] is ( + DialectCapabilityStatus.SUPPORTED + ) + + @pytest.mark.parametrize( + ("key", "options", "message"), + [ + ("top_logprobs", TopLogprobsOptions(21), "20"), + ("reasoning", ReasoningOptions(summary="detailed"), "summary"), + ("reasoning", ReasoningOptions(effort="xhigh"), "thinking level"), + ( + "hosted_tools", + HostedToolsOptions(("web_search",)), + "google_search", + ), + ], + ) + def test_capability_domains_are_provider_validated( + self, key: str, options: object, message: str + ) -> None: + decision = cast(Supported, CAPABILITY_DECISIONS[cast(Any, key)]) + with pytest.raises(ValueError, match=message): + decision.binding.validate_config(cast(Any, options)) + + +class TestRequestLowering: + @pytest.mark.anyio + @pytest.mark.parametrize( + ("base_url", "model", "expected_path"), + [ + ( + "https://generativelanguage.googleapis.com/", + "gemini-2.5-flash", + "/v1beta/models/gemini-2.5-flash:generateContent", + ), + ( + "https://generativelanguage.googleapis.com/v1beta/", + "models/gemini-2.5-flash", + "/v1beta/models/gemini-2.5-flash:generateContent", + ), + ], + ) + async def test_native_request_uses_final_url_header_and_wire_body( + self, base_url: str, model: str, expected_path: str + ) -> None: + seen: dict[str, object] = {} + + def handler(request: httpx.Request) -> httpx.Response: + seen["request"] = request + seen["body"] = json.loads(request.content) + return httpx.Response(200, json=_response()) + + req = _request( + input=_chat( + ChatMessage("system", (TextPart("Be concise"),)), + ChatMessage("developer", (TextPart("Use JSON"),)), + ChatMessage("user", (TextPart("Hello"),)), + ), + sampling=SamplingParams( + temperature=0.2, + top_p=0.9, + top_k=40, + max_tokens=64, + stop=("END",), + seed=7, + frequency_penalty=0.1, + presence_penalty=-0.2, + ), + ) + result = await _run(handler, req, model=model, base_url=base_url) + + request = cast(httpx.Request, seen["request"]) + assert request.url.path == expected_path + assert request.headers["x-goog-api-key"] == "google-secret" + assert seen["body"] == { + "contents": [{"role": "user", "parts": [{"text": "Hello"}]}], + "systemInstruction": { + "parts": [{"text": "Be concise"}, {"text": "Use JSON"}] + }, + "generationConfig": { + "temperature": 0.2, + "topP": 0.9, + "topK": 40, + "maxOutputTokens": 64, + "stopSequences": ["END"], + "seed": 7, + "frequencyPenalty": 0.1, + "presencePenalty": -0.2, + }, + } + assert result.texts == ("Hi",) + assert result.finish_reasons == ("STOP",) + assert result.request_params == { + "method": "POST", + "path": expected_path, + "body": { + "generationConfig": cast(dict[str, object], seen["body"])[ + "generationConfig" + ] + }, + } + assert "google-secret" not in json.dumps(result.request_params) + + @pytest.mark.anyio + async def test_missing_credential_does_not_emit_auth_header(self) -> None: + def handler(request: httpx.Request) -> httpx.Response: + assert "x-goog-api-key" not in request.headers + return httpx.Response(200, json=_response()) + + await _run(handler, credential=None) + + @pytest.mark.anyio + async def test_multimodal_parts_preserve_order(self) -> None: + seen: dict[str, object] = {} + + def handler(request: httpx.Request) -> httpx.Response: + seen["body"] = json.loads(request.content) + return httpx.Response(200, json=_response()) + + req = _request( + input=_chat( + ChatMessage( + "user", + ( + TextPart("before"), + ImagePart(data="aGVsbG8=", media_type="image/png"), + TextPart("after"), + ImagePart(url="gs://bucket/image.jpg", media_type="image/jpeg"), + ), + ) + ) + ) + await _run(handler, req) + + body = cast(dict[str, Any], seen["body"]) + assert body["contents"][0]["parts"] == [ + {"text": "before"}, + {"inlineData": {"data": "aGVsbG8=", "mimeType": "image/png"}}, + {"text": "after"}, + { + "fileData": { + "fileUri": "gs://bucket/image.jpg", + "mimeType": "image/jpeg", + } + }, + ] + + @pytest.mark.anyio + async def test_function_result_recovers_name_from_prior_call(self) -> None: + seen: dict[str, object] = {} + + def handler(request: httpx.Request) -> httpx.Response: + seen["body"] = json.loads(request.content) + return httpx.Response(200, json=_response()) + + req = _request( + input=_chat( + ChatMessage( + "assistant", + (ToolCallPart("call_1", "weather", {"city": "Paris"}),), + ), + ChatMessage( + "tool", + (ToolResultPart("call_1", {"temperature": 20}),), + ), + ChatMessage("user", (TextPart("Continue"),)), + ) + ) + await _run(handler, req) + + assert cast(dict[str, Any], seen["body"])["contents"] == [ + { + "role": "model", + "parts": [ + { + "functionCall": { + "id": "call_1", + "name": "weather", + "args": {"city": "Paris"}, + } + } + ], + }, + { + "role": "user", + "parts": [ + { + "functionResponse": { + "id": "call_1", + "name": "weather", + "response": {"temperature": 20}, + } + } + ], + }, + {"role": "user", "parts": [{"text": "Continue"}]}, + ] + + @pytest.mark.anyio + async def test_error_tool_result_is_explicit(self) -> None: + def handler(request: httpx.Request) -> httpx.Response: + body = json.loads(request.content) + assert body["contents"][1]["parts"][0]["functionResponse"]["response"] == { + "error": "failed" + } + return httpx.Response(200, json=_response()) + + await _run( + handler, + _request( + input=_chat( + ChatMessage("assistant", (ToolCallPart("c", "tool", {}),)), + ChatMessage("tool", (ToolResultPart("c", "failed", True),)), + ) + ), + ) + + @pytest.mark.anyio + async def test_function_and_hosted_tools_lower_with_observable_invocations( + self, + ) -> None: + def handler(request: httpx.Request) -> httpx.Response: + body = json.loads(request.content) + assert body["tools"] == [ + { + "functionDeclarations": [ + { + "name": "weather", + "description": "Get weather", + "parametersJsonSchema": { + "type": "object", + "properties": {"city": {"type": "string"}}, + }, + } + ] + }, + {"googleSearch": {}}, + ] + assert body["toolConfig"] == { + "functionCallingConfig": { + "mode": "ANY", + "allowedFunctionNames": ["weather"], + }, + "includeServerSideToolInvocations": True, + } + return httpx.Response(200, json=_response()) + + await _run( + handler, + _request( + tools=ToolParams( + functions=( + { + "type": "function", + "function": { + "name": "weather", + "description": "Get weather", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + }, + }, + }, + ), + choice={"type": "function", "name": "weather"}, + hosted=(HostedToolSpec("google_search"),), + ) + ), + ) + + @pytest.mark.anyio + async def test_safety_settings_are_audited_root_options(self) -> None: + settings = [ + { + "category": "HARM_CATEGORY_HATE_SPEECH", + "threshold": "BLOCK_ONLY_HIGH", + } + ] + + def handler(request: httpx.Request) -> httpx.Response: + assert json.loads(request.content)["safetySettings"] == settings + return httpx.Response(200, json=_response()) + + result = await _run( + handler, + _request( + dialect_options=DialectOptions( + "google_genai", {"safetySettings": settings} + ) + ), + ) + assert result.request_params is not None + assert result.request_params["body"] == {"safetySettings": settings} + + +class TestPreIORejections: + @pytest.mark.anyio + @pytest.mark.parametrize( + ("req", "message"), + [ + (_request(input=CompletionInput("prompt")), "ChatInput"), + ( + _request( + input=_chat( + ChatMessage("user", (TextPart("first"),)), + ChatMessage("system", (TextPart("late"),)), + ) + ), + "must precede", + ), + ( + _request( + input=_chat( + ChatMessage( + "user", + ( + ImagePart( + data="x", media_type="image/png", detail="high" + ), + ), + ) + ) + ), + "detail", + ), + ( + _request( + input=_chat(ChatMessage("tool", (ToolResultPart("missing", {}),))) + ), + "no prior call", + ), + ( + _request( + tools=ToolParams( + functions=({"name": "one"},), + choice={"type": "function", "name": "two"}, + ) + ), + "undeclared", + ), + ( + _request( + structured_output=StructuredOutputParams(schema={"type": "object"}) + ), + "requires format", + ), + ( + _request( + dialect_options=DialectOptions( + "google_genai", {"generationConfig": {"temperature": 2}} + ) + ), + "provider-neutral", + ), + ( + _request(sampling=SamplingParams(stop=("1", "2", "3", "4", "5", "6"))), + "five", + ), + ], + ) + async def test_invalid_shapes_fail_before_io( + self, req: Request, message: str + ) -> None: + with pytest.raises((DialectError, RequestAuditError), match=message): + await _run(_no_io, req) + + @pytest.mark.anyio + @pytest.mark.parametrize( + "sampling", + [ + SamplingParams(temperature=-0.1), + SamplingParams(temperature=2.1), + SamplingParams(top_p=1.1), + SamplingParams(top_k=0), + SamplingParams(max_tokens=0), + ], + ) + async def test_provider_sampling_domains_fail_before_io( + self, sampling: SamplingParams + ) -> None: + with pytest.raises(RequestAuditError): + await _run(_no_io, _request(sampling=sampling)) + + +class TestReasoningAndStructuredOutput: + @pytest.mark.anyio + async def test_visible_thoughts_and_full_history_round_trip(self) -> None: + first_body: dict[str, Any] = {} + + def first_handler(request: httpx.Request) -> httpx.Response: + first_body.update(json.loads(request.content)) + return httpx.Response( + 200, + json=_response( + [ + { + "text": "analysis", + "thought": True, + "thoughtSignature": "sig_1", + }, + {"text": "answer"}, + ] + ), + ) + + first = await _run( + first_handler, + _request(reasoning=ReasoningParams(effort="high", summary="auto")), + ) + assert first.reasoning is not None + assert first.reasoning[0] is not None + assert first.reasoning[0].text == "analysis" + opaque = cast(str, first.reasoning[0].opaque_roundtrip) + + def second_handler(request: httpx.Request) -> httpx.Response: + body = json.loads(request.content) + assert body["contents"][:2] == [ + {"role": "user", "parts": [{"text": "Hello"}]}, + { + "role": "model", + "parts": [ + { + "text": "analysis", + "thought": True, + "thoughtSignature": "sig_1", + }, + {"text": "answer"}, + ], + }, + ] + assert body["contents"][2] == { + "role": "user", + "parts": [{"text": "next"}], + } + return httpx.Response( + 200, + json=_response([{"text": "next answer", "thoughtSignature": "sig_2"}]), + ) + + second = await _run( + second_handler, + _request( + input=_chat(ChatMessage("user", (TextPart("next"),))), + session=SessionParams( + opaque_continuation=OpaqueContinuation("google_genai", opaque) + ), + ), + ) + assert second.reasoning is not None + assert second.reasoning[0] is not None + assert second.reasoning[0].opaque_roundtrip + + @pytest.mark.anyio + async def test_requested_summary_must_be_nonempty(self) -> None: + with pytest.raises(OutputContractError, match="omitted requested thoughts"): + await _run( + lambda _: httpx.Response(200, json=_response()), + _request(reasoning=ReasoningParams(summary="auto")), + ) + + @pytest.mark.anyio + @pytest.mark.parametrize("text", ['{"a":1,"a":2}', '{"a":NaN}', "not-json"]) + async def test_structured_output_requires_strict_json(self, text: str) -> None: + with pytest.raises(OutputContractError, match="valid JSON"): + await _run( + lambda _: httpx.Response(200, json=_response([{"text": text}])), + _request( + structured_output=StructuredOutputParams( + format="json_schema", schema={"type": "object"} + ) + ), + ) + + @pytest.mark.anyio + async def test_structured_output_uses_response_json_schema(self) -> None: + schema = {"type": "object", "properties": {"answer": {"type": "integer"}}} + + def handler(request: httpx.Request) -> httpx.Response: + assert json.loads(request.content)["generationConfig"] == { + "responseMimeType": "application/json", + "responseJsonSchema": schema, + } + return httpx.Response(200, json=_response([{"text": '{"answer":42}'}])) + + result = await _run( + handler, + _request( + structured_output=StructuredOutputParams( + format="json_schema", schema=schema + ) + ), + ) + assert result.structured_output is not None + assert result.structured_output.value == {"answer": 42} + + +class TestResponseLifting: + @pytest.mark.anyio + async def test_usage_logprobs_citations_grounding_and_server_tools(self) -> None: + logprobs = { + "chosenCandidates": [{"token": "Hi", "logProbability": -0.1, "tokenId": 7}], + "topCandidates": [ + { + "candidates": [ + {"token": "Hi", "logProbability": -0.1, "tokenId": 7}, + {"token": "Hey", "logProbability": -1.2, "tokenId": 8}, + ] + } + ], + } + candidate_extra = { + "logprobsResult": logprobs, + "citationMetadata": {"citationSources": [{"uri": "https://cite"}]}, + "groundingMetadata": { + "groundingChunks": [ + {"web": {"uri": "https://web", "title": "Web"}}, + { + "image": { + "sourceUri": "https://image-source", + "title": "Image", + } + }, + ], + "searchEntryPoint": {"renderedContent": "
sources
"}, + }, + } + parts = [ + { + "toolCall": { + "id": "server_1", + "toolType": "GOOGLE_SEARCH_WEB", + "args": {"q": "weather"}, + } + }, + { + "toolResponse": { + "id": "server_1", + "toolType": "GOOGLE_SEARCH_WEB", + "response": {"ok": True}, + } + }, + {"text": "Hi"}, + ] + result = await _run( + lambda _: httpx.Response( + 200, + json=_response(parts, candidate_extra=candidate_extra), + ), + _request( + scoring=ScoringParams(sampled_logprobs=True, top_logprobs=2), + tools=ToolParams(hosted=(HostedToolSpec("google_search"),)), + ), + ) + + assert result.logprobs is not None and result.logprobs[0].token_id == 7 + assert result.top_logprobs is not None + assert [item.token for item in result.top_logprobs[0]] == ["Hi", "Hey"] + assert result.citations is not None + assert result.citations[0].url == "https://cite" + assert result.grounding is not None + assert [chunk.uri for chunk in result.grounding.chunks] == [ + "https://web", + "https://image-source", + ] + assert result.server_tool_uses is not None + assert result.server_tool_uses[0].tool_type == "GOOGLE_SEARCH_WEB" + assert result.server_tool_uses[0].result == {"ok": True} + assert result.usage is not None + assert result.usage.total_tokens == 7 + assert result.usage.reported_total_tokens == 7 + assert result.usage.reasoning_tokens == 1 + assert result.usage.cached_tokens == 2 + + @pytest.mark.anyio + async def test_usage_computes_total_without_claiming_provider_report(self) -> None: + result = await _run( + lambda _: httpx.Response( + 200, + json=_response( + usage={"promptTokenCount": 3, "candidatesTokenCount": 4} + ), + ) + ) + assert result.usage is not None + assert result.usage.total_tokens == 7 + assert result.usage.reported_total_tokens is None + + @pytest.mark.anyio + @pytest.mark.parametrize( + ("payload", "message"), + [ + (_response(finish_reason="FINISH_REASON_UNSPECIFIED"), "not terminal"), + (_response(candidate_extra={"index": 1}), "not ordered"), + ( + _response( + parts=[{"executableCode": {"language": "PYTHON", "code": "1"}}] + ), + "unsupported field", + ), + ( + _response( + parts=[{"functionCall": {"id": "c", "name": "f", "args": {}}}] + ), + "unrequested function", + ), + ], + ) + async def test_unsupported_response_shapes_fail_loudly( + self, payload: dict[str, object], message: str + ) -> None: + with pytest.raises(OutputContractError, match=message): + await _run(lambda _: httpx.Response(200, json=payload)) + + @pytest.mark.anyio + async def test_missing_requested_logprobs_is_an_output_contract_error(self) -> None: + with pytest.raises(OutputContractError, match="omitted requested logprobs"): + await _run( + lambda _: httpx.Response(200, json=_response()), + _request(scoring=ScoringParams(sampled_logprobs=True)), + ) + + +class TestStreamingAndAuditEvidence: + @pytest.mark.anyio + async def test_streaming_uses_stream_operation_and_merges_deltas(self) -> None: + seen: dict[str, object] = {} + + def handler(request: httpx.Request) -> httpx.Response: + seen["url"] = str(request.url) + return httpx.Response( + 200, + content=_sse( + { + "candidates": [ + { + "index": 0, + "content": { + "role": "model", + "parts": [{"text": "Hel"}], + }, + } + ], + "modelVersion": "gemini-2.5-flash", + }, + { + "candidates": [ + { + "index": 0, + "content": {"role": "model", "parts": [{"text": "lo"}]}, + "finishReason": "MAX_TOKENS", + } + ], + "usageMetadata": { + "promptTokenCount": 2, + "candidatesTokenCount": 2, + "totalTokenCount": 4, + }, + "modelVersion": "gemini-2.5-flash", + }, + ), + headers={"content-type": "text/event-stream"}, + ) + + result = await _run( + handler, + _request(scheduling=SchedulingParams(stream=True)), + ) + assert seen["url"] == ( + "https://generativelanguage.googleapis.com/v1beta/models/" + "gemini-2.5-flash:streamGenerateContent?alt=sse" + ) + assert result.texts == ("Hello",) + assert result.finish_reasons == ("MAX_TOKENS",) + assert result.request_params is not None + assert result.request_params["query"] == {"alt": "sse"} + + @pytest.mark.anyio + async def test_streaming_logprobs_are_accumulated(self) -> None: + def logprob(token: str, token_id: int) -> dict[str, object]: + return {"token": token, "logProbability": -0.1, "tokenId": token_id} + + def handler(_: httpx.Request) -> httpx.Response: + chunks: list[dict[str, object]] = [] + for index, token in enumerate(("A", "B")): + candidate: dict[str, object] = { + "index": 0, + "content": {"role": "model", "parts": [{"text": token}]}, + "logprobsResult": { + "chosenCandidates": [logprob(token, index)], + "topCandidates": [{"candidates": [logprob(token, index)]}], + }, + } + if index == 1: + candidate["finishReason"] = "STOP" + chunks.append({"candidates": [candidate]}) + return httpx.Response(200, content=_sse(*chunks)) + + result = await _run( + handler, + _request( + scheduling=SchedulingParams(stream=True), + scoring=ScoringParams(sampled_logprobs=True, top_logprobs=1), + ), + ) + assert result.logprobs is not None + assert [item.token for item in result.logprobs] == ["A", "B"] + + def test_input_verifier_detects_lowering_mutation(self) -> None: + connection = _connection(_no_io) + dialect = GoogleGenAIDialect(connection, "gemini-2.5-flash") + req = _request( + input=_chat(ChatMessage("user", (TextPart("before"), TextPart("after")))) + ) + audit = RequestAudit(active_request_leaves(req)) + dialect.validate_request(req, audit, cast(Any, SimpleNamespace())) + prepared = dialect.prepare(req, audit) + body = prepared.thaw_body() + body["contents"] = [{"role": "user", "parts": [{"text": "afterbefore"}]}] + + with pytest.raises(RequestAuditError, match="contents"): + audit.finish(PreparedRequest(prepared.operation, body, prepared.context)) + + def test_operation_verifier_detects_stream_routing_mutation(self) -> None: + connection = _connection(_no_io) + dialect = GoogleGenAIDialect(connection, "gemini-2.5-flash") + req = _request(scheduling=SchedulingParams(stream=True)) + audit = RequestAudit(active_request_leaves(req)) + dialect.validate_request(req, audit, cast(Any, SimpleNamespace())) + prepared = dialect.prepare(req, audit) + wrong = PreparedRequest( + prepared.operation.replace("streamGenerateContent", "generateContent"), + prepared.body, + prepared.context, + ) + + with pytest.raises(RequestAuditError, match="operation"): + audit.finish(wrong) + + @pytest.mark.anyio + async def test_malformed_sse_fails_loudly(self) -> None: + with pytest.raises(OutputContractError, match="malformed SSE"): + await _run( + lambda _: httpx.Response(200, content=b"not-sse\n\n"), + _request(scheduling=SchedulingParams(stream=True)), + ) diff --git a/tests/unit/core/models/test_dialect_registry.py b/tests/unit/core/models/test_dialect_registry.py index 6711bdd4..c89d39b6 100644 --- a/tests/unit/core/models/test_dialect_registry.py +++ b/tests/unit/core/models/test_dialect_registry.py @@ -48,6 +48,7 @@ get_dialect_spec, ) from sieval.core.models.dialects.anthropic_messages import AnthropicMessagesDialect +from sieval.core.models.dialects.google_genai import GoogleGenAIDialect from sieval.core.models.dialects.openai_chat import OpenAIChatDialect from sieval.core.models.dialects.openai_completions import OpenAICompletionsDialect from sieval.core.models.dialects.openai_responses import OpenAIResponsesDialect @@ -133,7 +134,7 @@ def test_request_seed_support_is_required_and_explicit(self) -> None: "openai_completions": RequestSeedSupport.SUPPORTED, "openai_responses": RequestSeedSupport.UNSUPPORTED, "anthropic_messages": RequestSeedSupport.UNSUPPORTED, - "google_genai": RequestSeedSupport.RESERVED, + "google_genai": RequestSeedSupport.SUPPORTED, "sglang_native": RequestSeedSupport.RESERVED, "vllm_native": RequestSeedSupport.RESERVED, } @@ -149,6 +150,7 @@ def test_implemented_dialects_are_active_and_bindable(self) -> None: "openai_completions", "openai_responses", "anthropic_messages", + "google_genai", } assert set(DIALECT_BINDERS) == active assert all(inspect.isfunction(binder) for binder in DIALECT_BINDERS.values()) @@ -264,8 +266,7 @@ def test_capability_decisions_are_available_only_for_active_dialects(self) -> No assert set(capability_decisions_for("anthropic_messages")) == set( CAPABILITY_KEYS ) - with pytest.raises(DialectNotImplemented, match="no executable binder"): - capability_decisions_for("google_genai") + assert set(capability_decisions_for("google_genai")) == set(CAPABILITY_KEYS) class TestDialectBinders: @@ -295,6 +296,7 @@ def test_reserved_dialects_fail_with_named_error( ("openai_completions", OpenAICompletionsDialect), ("openai_responses", OpenAIResponsesDialect), ("anthropic_messages", AnthropicMessagesDialect), + ("google_genai", GoogleGenAIDialect), ], ) def test_active_binders_construct_expected_dialect( @@ -302,7 +304,7 @@ def test_active_binders_construct_expected_dialect( ) -> None: binder = DIALECT_BINDERS[dialect_id] connection: object = object() - if dialect_id == "anthropic_messages": + if dialect_id in {"anthropic_messages", "google_genai"}: connection = AsyncHTTPJSONConnection(httpx.AsyncClient(), None) dialect = binder(connection, "requested-model") @@ -317,6 +319,7 @@ def test_active_binders_construct_expected_dialect( ("openai_completions", OpenAICompletionsDialect), ("openai_responses", OpenAIResponsesDialect), ("anthropic_messages", AnthropicMessagesDialect), + ("google_genai", GoogleGenAIDialect), ], ) def test_validated_runtime_plan_binds_active_dialect( From edf7c3080e13b3053babcd2602bf5318897f7958 Mon Sep 17 00:00:00 2001 From: jack-scitix-ai Date: Sun, 13 Sep 2026 18:30:33 +0800 Subject: [PATCH 04/15] fix(models): harden Google GenAI dialect contracts Reject lossy chat and stream shapes before they can be reinterpreted, preserve terminal usage and citation evidence, and make multi-candidate and continuation behavior explicit. Snapshot the request used for response lifting and strengthen regression tests around final-wire provenance, stable call identities, replay, and required output channels. --- sieval/core/models/dialects/google_genai.py | 494 ++++++++++-- tests/unit/cli/test_validation.py | 68 +- .../core/models/dialects/test_google_genai.py | 730 +++++++++++++++++- 3 files changed, 1202 insertions(+), 90 deletions(-) diff --git a/sieval/core/models/dialects/google_genai.py b/sieval/core/models/dialects/google_genai.py index db59487d..4070305e 100644 --- a/sieval/core/models/dialects/google_genai.py +++ b/sieval/core/models/dialects/google_genai.py @@ -8,9 +8,11 @@ AI-Generated Code - GPT-5.6 (OpenAI) """ +import hashlib import json import math from collections.abc import Callable, Mapping, Sequence +from copy import deepcopy from dataclasses import dataclass, field from types import MappingProxyType from typing import cast @@ -301,7 +303,10 @@ class _GoogleContext: class _GoogleExecutionContext: request: Request request_params: Mapping[str, JSONValue] + system_instruction: Mapping[str, JSONValue] | None input_contents: tuple[Mapping[str, JSONValue], ...] + historical_function_call_ids: frozenset[str] + historical_server_call_ids: frozenset[str] @dataclass(frozen=True) @@ -324,13 +329,17 @@ class _OperationVerifier: stream: bool version_prefix: str - def verify_prepared(self, prepared: PreparedRequest) -> str | None: + def expected_operation(self) -> str: + """Return the one relative operation this bound dialect may execute.""" + method = "streamGenerateContent" if self.stream else "generateContent" expected = ( f"{self.version_prefix}models/{quote(self.model_name, safe='')}:{method}" ) - if self.stream: - expected += "?alt=sse" + return f"{expected}?alt=sse" if self.stream else expected + + def verify_prepared(self, prepared: PreparedRequest) -> str | None: + expected = self.expected_operation() if prepared.operation != expected: return f"operation {prepared.operation!r} does not match {expected!r}" return None @@ -478,6 +487,8 @@ def _hosted_tool(spec: object) -> dict[str, JSONValue]: def _image_to_part(part: ImagePart) -> dict[str, JSONValue]: if part.detail is not None: raise DialectError("Google image parts have no per-image detail field") + if part.media_type is not None and part.media_type not in _IMAGE_MEDIA_TYPES: + raise DialectError(f"unsupported Google image media type {part.media_type!r}") if part.url is not None: if not part.url: raise DialectError("Google image URLs must not be empty") @@ -489,8 +500,6 @@ def _image_to_part(part: ImagePart) -> dict[str, JSONValue]: raise DialectError( "Google inline images require data and an explicit MIME type" ) - if part.media_type not in _IMAGE_MEDIA_TYPES: - raise DialectError(f"unsupported Google image media type {part.media_type!r}") return {"inlineData": {"data": part.data, "mimeType": part.media_type}} @@ -641,7 +650,30 @@ def _validate_opaque_contents(raw: object) -> list[dict[str, JSONValue]]: return result -def _decode_opaque(value: str) -> list[dict[str, JSONValue]]: +def _validate_opaque_system( + raw: object, +) -> dict[str, JSONValue] | None: + if raw is None: + return None + copied = copy_json_value(raw, "Google opaque systemInstruction") + if not isinstance(copied, dict) or set(copied) != {"parts"}: + raise DialectError("Google opaque systemInstruction has invalid shape") + parts = copied["parts"] + if not isinstance(parts, list) or not parts: + raise DialectError("Google opaque systemInstruction has no parts") + if any( + not isinstance(part, dict) + or set(part) != {"text"} + or not isinstance(part["text"], str) + for part in parts + ): + raise DialectError("Google opaque systemInstruction must be text-only") + return cast(dict[str, JSONValue], copied) + + +def _decode_opaque( + value: str, +) -> tuple[dict[str, JSONValue] | None, list[dict[str, JSONValue]]]: try: raw = json.loads( value, @@ -650,49 +682,152 @@ def _decode_opaque(value: str) -> list[dict[str, JSONValue]]: ) except (json.JSONDecodeError, TypeError, ValueError) as exc: raise DialectError("invalid Google opaque continuation JSON") from exc - if not isinstance(raw, dict) or set(raw) != {"version", "contents"}: + if ( + not isinstance(raw, dict) + or not {"version", "contents"}.issubset(raw) + or set(raw) - {"version", "contents", "systemInstruction"} + ): raise DialectError("invalid Google opaque continuation envelope") if raw.get("version") != _OPAQUE_VERSION: raise DialectError("unsupported Google opaque continuation version") - return _validate_opaque_contents(raw.get("contents")) + return ( + _validate_opaque_system(raw.get("systemInstruction")), + _validate_opaque_contents(raw.get("contents")), + ) -def _call_names_from_contents( +def _call_state_from_contents( contents: Sequence[Mapping[str, JSONValue]], -) -> dict[str, str]: +) -> tuple[dict[str, str], set[str]]: names: dict[str, str] = {} - for content in contents: + answered: set[str] = set() + for content_index, content in enumerate(contents): parts = content.get("parts") if not isinstance(parts, list): continue - for part in parts: + for part_index, part in enumerate(parts): if not isinstance(part, dict): continue call = part.get("functionCall") - if not isinstance(call, dict): - continue - call_id = call.get("id") - name = call.get("name") - if isinstance(call_id, str) and call_id and isinstance(name, str) and name: + if isinstance(call, dict): + call_id = call.get("id") + name = call.get("name") + if not isinstance(call_id, str) or not call_id: + raise DialectError( + "Google opaque functionCall requires a non-empty id" + ) + if not isinstance(name, str) or not name: + raise DialectError( + "Google opaque functionCall requires a non-empty name" + ) if call_id in names: raise DialectError(f"duplicate Google function call id {call_id!r}") names[call_id] = name - return names + response = part.get("functionResponse") + if not isinstance(response, dict): + continue + call_id = response.get("id") + name = response.get("name") + if not isinstance(call_id, str) or not call_id: + raise DialectError( + "Google opaque functionResponse requires a non-empty id" + ) + try: + expected_name = names[call_id] + except KeyError as exc: + raise DialectError( + "Google opaque functionResponse at " + f"{content_index}.{part_index} has no prior functionCall" + ) from exc + if name != expected_name: + raise DialectError( + f"Google opaque functionResponse name changed for {call_id!r}" + ) + if call_id in answered: + raise DialectError(f"duplicate Google function result for {call_id!r}") + answered.add(call_id) + return names, answered + + +def _historical_output_call_ids( + contents: Sequence[Mapping[str, JSONValue]], +) -> tuple[frozenset[str], frozenset[str]]: + """Recover conversation-scoped call ids from the final request body.""" + + function_ids: set[str] = set() + server_ids: set[str] = set() + for content_index, content in enumerate(contents): + parts = content.get("parts") + if not isinstance(parts, list): + raise DialectError( + f"final Google contents[{content_index}].parts must be a list" + ) + for part_index, part in enumerate(parts): + if not isinstance(part, Mapping): + raise DialectError( + f"final Google contents[{content_index}].parts[{part_index}] " + "must be an object" + ) + for field_name, ids in ( + ("functionCall", function_ids), + ("toolCall", server_ids), + ): + if field_name not in part: + continue + call = part[field_name] + if not isinstance(call, Mapping): + raise DialectError( + f"final Google {field_name} history must be an object" + ) + call_id = call.get("id") + if not isinstance(call_id, str) or not call_id: + raise DialectError( + f"final Google {field_name} history requires a non-empty id" + ) + if call_id in ids: + raise DialectError( + f"final Google history duplicates {field_name} id {call_id!r}" + ) + ids.add(call_id) + return frozenset(function_ids), frozenset(server_ids) + + +def _chat_shape_error(request_input: ChatInput) -> str | None: + """Return a lossless-mapping error for Google chat history, if any.""" + + if any(message.role == "developer" for message in request_input.messages): + return ( + "Google systemInstruction cannot preserve the distinct developer " + "message authority" + ) + if request_input.messages[-1].role == "assistant": + return "google_genai does not support assistant-prefill input" + return None + + +def _require_generation_turn(contents: Sequence[JSONValue]) -> None: + """Reject a body that cannot ask Google for a new model turn.""" + + if not contents or _get(contents[-1], "role") != "user": + raise DialectError("Google contents must end with a user or tool-result turn") def _input_to_wire( request_input: ChatInput, opaque: str | None ) -> tuple[dict[str, JSONValue] | None, list[JSONValue]]: + if error := _chat_shape_error(request_input): + raise DialectError(error) system_parts: list[JSONValue] = [] - opaque_contents = _decode_opaque(opaque) if opaque is not None else [] + opaque_system, opaque_contents = ( + _decode_opaque(opaque) if opaque is not None else (None, []) + ) contents: list[JSONValue] = list(opaque_contents) - call_names = _call_names_from_contents(opaque_contents) - answered_calls: set[str] = set() + call_names, answered_calls = _call_state_from_contents(opaque_contents) saw_conversation = False for message in request_input.messages: if message.name is not None: raise DialectError("Google contents have no ChatMessage.name field") - if message.role in {"system", "developer"}: + if message.role == "system": if saw_conversation: raise DialectError( "Google system/developer messages must precede conversation turns" @@ -706,7 +841,18 @@ def _input_to_wire( parts = _content_parts(message, call_names, answered_calls) role = "model" if message.role == "assistant" else "user" contents.append({"role": role, "parts": parts}) - system = {"parts": system_parts} if system_parts else None + _require_generation_turn(contents) + current_system = {"parts": system_parts} if system_parts else None + if ( + opaque_system is not None + and current_system is not None + and opaque_system != current_system + ): + raise DialectError( + "Google opaque continuation systemInstruction conflicts with " + "the current request" + ) + system = current_system if current_system is not None else opaque_system return system, contents @@ -715,20 +861,23 @@ def _verification_input_to_wire( ) -> tuple[dict[str, JSONValue] | None, list[JSONValue]]: """Independently derive the expected input for request-audit verification.""" - opaque_contents = _decode_opaque(opaque) if opaque is not None else [] + if error := _chat_shape_error(request_input): + raise DialectError(error) + opaque_system, opaque_contents = ( + _decode_opaque(opaque) if opaque is not None else (None, []) + ) contents: list[JSONValue] = [ copy_json_value(item, "verified Google opaque content") for item in opaque_contents ] system_parts: list[JSONValue] = [] - call_names = _call_names_from_contents(opaque_contents) - answered: set[str] = set() + call_names, answered = _call_state_from_contents(opaque_contents) saw_conversation = False for message in request_input.messages: if message.name is not None: raise DialectError("Google contents have no ChatMessage.name field") - is_instruction = message.role in {"system", "developer"} + is_instruction = message.role == "system" if is_instruction and saw_conversation: raise DialectError( "Google system/developer messages must precede conversation turns" @@ -752,6 +901,13 @@ def _verification_input_to_wire( raise DialectError( "Google image parts have no per-image detail field" ) + if ( + part.media_type is not None + and part.media_type not in _IMAGE_MEDIA_TYPES + ): + raise DialectError( + f"unsupported Google image media type {part.media_type!r}" + ) if part.url is not None: file_data: dict[str, JSONValue] = {"fileUri": part.url} if part.media_type is not None: @@ -837,7 +993,18 @@ def _verification_input_to_wire( role = "model" if message.role == "assistant" else "user" contents.append({"role": role, "parts": expected_parts}) - system = {"parts": system_parts} if system_parts else None + _require_generation_turn(contents) + current_system = {"parts": system_parts} if system_parts else None + if ( + opaque_system is not None + and current_system is not None + and opaque_system != current_system + ): + raise DialectError( + "Google opaque continuation systemInstruction conflicts with " + "the current request" + ) + system = current_system if current_system is not None else opaque_system return system, contents @@ -867,7 +1034,7 @@ def _usage(raw: object) -> UsageStats | None: reported_total = _optional_nonnegative_int( _get(raw, "totalTokenCount"), "usage.totalTokenCount" ) - total = reported_total if reported_total is not None else prompt + output + total = prompt + output return UsageStats( input_tokens=prompt, output_tokens=output, @@ -886,7 +1053,11 @@ def _usage(raw: object) -> UsageStats | None: if _get(raw, "cachedContentTokenCount") is not None else None ), - reported_total_tokens=reported_total, + reported_total_tokens=( + reported_total + if reported_total is not None and reported_total != total + else None + ), ) @@ -918,9 +1089,9 @@ def _logprobs( if not tops: continue entries = _sequence(_get(tops[index], "candidates"), f"{path}.top[{index}]") - if len(entries) > requested_top_k and requested_top_k > 0: + if requested_top_k > 0 and len(entries) != requested_top_k: raise OutputContractError( - "Google returned more top logprobs than requested" + "Google returned a different number of top logprobs than requested" ) alternatives.append( tuple( @@ -1013,14 +1184,62 @@ def _grounding(raw: object, path: str) -> GroundingMetadata | None: return GroundingMetadata(tuple(chunks), rendered) +def _synthetic_call_id( + input_contents: Sequence[Mapping[str, JSONValue]], + candidate_index: int, + part_index: int, + kind: str, + name: str, + arguments: Mapping[str, JSONValue], +) -> str: + """Derive a stable IR id when Google omits an optional call id.""" + + payload = { + "input_contents": list(input_contents), + "candidate_index": candidate_index, + "part_index": part_index, + "kind": kind, + "name": name, + "arguments": dict(arguments), + } + encoded = json.dumps( + payload, + ensure_ascii=False, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + prefix = "google_call" if kind == "function" else "google_server" + return f"{prefix}_{hashlib.sha256(encoded).hexdigest()[:24]}" + + def _opaque_history( + system_instruction: Mapping[str, JSONValue] | None, input_contents: Sequence[Mapping[str, JSONValue]], output_content: Mapping[str, JSONValue], + opaque_call_ids: Mapping[int, tuple[str, str]], ) -> str | None: history: list[JSONValue] = [ copy_json_value(item, "Google input history") for item in input_contents ] - history.append(copy_json_value(output_content, "Google output history")) + copied_output = copy_json_value(output_content, "Google output history") + if not isinstance(copied_output, dict): + raise AssertionError("copied Google output content must remain an object") + copied_parts = copied_output.get("parts") + if not isinstance(copied_parts, list): + raise AssertionError("copied Google output content must retain parts") + for part_index, (field_name, call_id) in opaque_call_ids.items(): + try: + copied_part = copied_parts[part_index] + except IndexError as exc: + raise AssertionError("Google function-call part index changed") from exc + if not isinstance(copied_part, dict): + raise AssertionError("Google function-call part changed shape") + copied_call = copied_part.get(field_name) + if not isinstance(copied_call, dict): + raise AssertionError("Google call payload changed shape") + copied_call["id"] = call_id + history.append(copied_output) saw_signature = False for content in history: if not isinstance(content, dict): @@ -1038,8 +1257,16 @@ def _opaque_history( break if not saw_signature: return None + envelope: dict[str, JSONValue] = { + "version": _OPAQUE_VERSION, + "contents": history, + } + if system_instruction is not None: + envelope["systemInstruction"] = copy_json_value( + system_instruction, "Google system history" + ) return json.dumps( - {"version": _OPAQUE_VERSION, "contents": history}, + envelope, ensure_ascii=False, allow_nan=False, sort_keys=True, @@ -1095,9 +1322,17 @@ def validate_request( has_named_message = isinstance(req.input, ChatInput) and any( message.name is not None for message in req.input.messages ) + chat_shape_error = ( + _chat_shape_error(req.input) if isinstance(req.input, ChatInput) else None + ) for path in audit.active_paths: - if path == "input.chat" and has_named_message: - audit.rejected(path, "Google contents have no ChatMessage.name field") + if path == "input.chat": + if has_named_message: + audit.rejected( + path, "Google contents have no ChatMessage.name field" + ) + elif chat_shape_error is not None: + audit.rejected(path, chat_shape_error) elif path.startswith("input.completion"): audit.rejected(path, "google_genai requires ChatInput") elif path == "input.modality.image.detail": @@ -1337,21 +1572,18 @@ def consume(path: str, *evidence: WireEvidence) -> None: body[key] = value audit.consumed(path, _RootOptionVerifier(key, value)) - method = "streamGenerateContent" if req.scheduling.stream else "generateContent" - operation = ( - f"{self._version_prefix}models/{quote(self._model_name, safe='')}:{method}" + operation_verifier = _OperationVerifier( + self._model_name, + req.scheduling.stream, + self._version_prefix, ) + operation = operation_verifier.expected_operation() if req.scheduling.stream: - operation += "?alt=sse" consume( "scheduling.stream", - _OperationVerifier( - self._model_name, - True, - self._version_prefix, - ), + operation_verifier, ) - return PreparedRequest(operation, body, _GoogleContext(req)) + return PreparedRequest(operation, body, _GoogleContext(deepcopy(req))) def _lift(self, raw: object, context: _GoogleExecutionContext) -> Response: req = context.request @@ -1365,6 +1597,8 @@ def _lift(self, raw: object, context: _GoogleExecutionContext) -> Response: finish_reasons: list[str] = [] reasoning: list[ReasoningOutput | None] = [] calls: list[FunctionToolCall] = [] + call_ids = set(context.historical_function_call_ids) + server_call_ids = set(context.historical_server_call_ids) server_calls: dict[str, tuple[str, Mapping[str, JSONValue]]] = {} server_results: dict[str, tuple[str, JSONValue]] = {} citations: list[Citation] = [] @@ -1399,6 +1633,7 @@ def _lift(self, raw: object, context: _GoogleExecutionContext) -> Response: raise OutputContractError("Google response content must be an object") text_parts: list[str] = [] thought_text: list[str] = [] + opaque_call_ids: dict[int, tuple[str, str]] = {} for part_index, raw_part in enumerate(parts): if not isinstance(raw_part, Mapping): raise OutputContractError("Google response part must be an object") @@ -1455,10 +1690,33 @@ def _lift(self, raw: object, context: _GoogleExecutionContext) -> Response: ) if not isinstance(arguments, dict): raise OutputContractError("functionCall.args must be an object") + name = _required_string(_get(call, "name"), "functionCall.name") + raw_call_id = _get(call, "id") + if raw_call_id is None or raw_call_id == "": + call_id = _synthetic_call_id( + context.input_contents, + index, + part_index, + "function", + name, + arguments, + ) + elif isinstance(raw_call_id, str): + call_id = raw_call_id + else: + raise OutputContractError( + "functionCall.id must be a string when present" + ) + if call_id in call_ids: + raise OutputContractError( + f"duplicate Google function call id {call_id!r}" + ) + call_ids.add(call_id) + opaque_call_ids[part_index] = ("functionCall", call_id) calls.append( FunctionToolCall( - _required_string(_get(call, "id"), "functionCall.id"), - _required_string(_get(call, "name"), "functionCall.name"), + call_id, + name, arguments, ) ) @@ -1471,18 +1729,35 @@ def _lift(self, raw: object, context: _GoogleExecutionContext) -> Response: if not isinstance(raw_item, Mapping): raise OutputContractError("toolCall must be an object") item = cast(Mapping[str, object], raw_item) - tool_id = _required_string(_get(item, "id"), "toolCall.id") - if tool_id in server_calls: - raise OutputContractError( - f"duplicate Google server tool call id {tool_id!r}" - ) + tool_type = _required_string( + _get(item, "toolType"), "toolCall.toolType" + ) + raw_tool_id = _get(item, "id") raw_args = copy_json_value(_get(item, "args", {}), "toolCall.args") if not isinstance(raw_args, dict): raise OutputContractError("toolCall.args must be an object") - server_calls[tool_id] = ( - _required_string(_get(item, "toolType"), "toolCall.toolType"), - raw_args, - ) + if raw_tool_id is None or raw_tool_id == "": + tool_id = _synthetic_call_id( + context.input_contents, + index, + part_index, + "server", + tool_type, + raw_args, + ) + elif isinstance(raw_tool_id, str): + tool_id = raw_tool_id + else: + raise OutputContractError( + "toolCall.id must be a string when present" + ) + if tool_id in server_call_ids: + raise OutputContractError( + f"duplicate Google server tool call id {tool_id!r}" + ) + server_call_ids.add(tool_id) + server_calls[tool_id] = (tool_type, raw_args) + opaque_call_ids[part_index] = ("toolCall", tool_id) else: if not req.tools.hosted: raise OutputContractError( @@ -1492,7 +1767,27 @@ def _lift(self, raw: object, context: _GoogleExecutionContext) -> Response: if not isinstance(raw_item, Mapping): raise OutputContractError("toolResponse must be an object") item = cast(Mapping[str, object], raw_item) - tool_id = _required_string(_get(item, "id"), "toolResponse.id") + tool_type = _required_string( + _get(item, "toolType"), "toolResponse.toolType" + ) + raw_tool_id = _get(item, "id") + if raw_tool_id is None or raw_tool_id == "": + matching = [ + call_id + for call_id, (call_type, _) in server_calls.items() + if call_type == tool_type and call_id not in server_results + ] + if len(matching) != 1: + raise OutputContractError( + "Google toolResponse without id is ambiguous" + ) + tool_id = matching[0] + elif isinstance(raw_tool_id, str): + tool_id = raw_tool_id + else: + raise OutputContractError( + "toolResponse.id must be a string when present" + ) if tool_id in server_results: raise OutputContractError( f"duplicate Google server tool response id {tool_id!r}" @@ -1504,12 +1799,8 @@ def _lift(self, raw: object, context: _GoogleExecutionContext) -> Response: raise OutputContractError( "toolResponse.response must be an object" ) - server_results[tool_id] = ( - _required_string( - _get(item, "toolType"), "toolResponse.toolType" - ), - tool_result, - ) + server_results[tool_id] = (tool_type, tool_result) + opaque_call_ids[part_index] = ("toolResponse", tool_id) text = "".join(text_parts) texts.append(text) finish_reason = _required_string( @@ -1519,7 +1810,12 @@ def _lift(self, raw: object, context: _GoogleExecutionContext) -> Response: raise OutputContractError("Google candidate is not terminal") finish_reasons.append(finish_reason) opaque = ( - _opaque_history(context.input_contents, content_mapping) + _opaque_history( + context.system_instruction, + context.input_contents, + content_mapping, + opaque_call_ids, + ) if content_mapping is not None and parts else None ) @@ -1552,10 +1848,15 @@ def _lift(self, raw: object, context: _GoogleExecutionContext) -> Response: "candidate.logprobsResult", req.scoring.top_logprobs, ) + if text and not chosen: + raise OutputContractError( + "Google returned non-empty text without chosen logprobs" + ) sampled.extend(chosen) top.extend(alternatives) if req.reasoning.summary == "auto" and any( - item is None or not item.text for item in reasoning + item is None or item.text is None or not item.text.strip() + for item in reasoning ): raise OutputContractError( "Google reply omitted requested thoughts for one or more candidates" @@ -1617,7 +1918,12 @@ def flush_event() -> None: return payload = "\n".join(data_lines) data_lines.clear() - chunks.append(_strict_json_loads(payload, "Google stream event")) + event = _strict_json_loads(payload, "Google stream event") + if not isinstance(event, dict) or not event: + raise OutputContractError( + "Google stream event must be a non-empty response object" + ) + chunks.append(event) async for line in response.aiter_lines(): if not line: @@ -1643,14 +1949,11 @@ def _merge_stream( self, chunks: list[object], context: _GoogleExecutionContext ) -> Response: # Each stream event carries candidate deltas plus cumulative/terminal - # metadata. Merge only provider-defined additive content and require - # finish/usage metadata to agree when repeated. + # metadata. Merge only provider-defined additive content; usage is a + # terminal snapshot and must come from the final event. by_index: dict[int, dict[str, object]] = {} - usage: object = None model_version: object = None for chunk_index, chunk in enumerate(chunks): - if _get(chunk, "usageMetadata") is not None: - usage = _get(chunk, "usageMetadata") if _get(chunk, "modelVersion") is not None: current_model = _get(chunk, "modelVersion") if model_version is not None and current_model != model_version: @@ -1689,7 +1992,11 @@ def _merge_stream( "Google stream changed a candidate finishReason" ) state["finishReason"] = finish_reason - for name in ("citationMetadata", "groundingMetadata"): + for name in ( + "citationMetadata", + "urlContextMetadata", + "groundingMetadata", + ): value = _get(raw_candidate, name) if value is not None: state[name] = value @@ -1738,8 +2045,11 @@ def _merge_stream( candidate[key] = copy_json_value(value, f"stream.{key}") candidates.append(candidate) merged: dict[str, JSONValue] = {"candidates": candidates} - if usage is not None: - merged["usageMetadata"] = copy_json_value(usage, "stream usage") + terminal_usage = _get(chunks[-1], "usageMetadata") + if terminal_usage is not None: + merged["usageMetadata"] = copy_json_value( + terminal_usage, "terminal stream usage" + ) if model_version is not None: merged["modelVersion"] = copy_json_value(model_version, "stream model") return self._lift(merged, context) @@ -1748,13 +2058,16 @@ async def execute(self, prepared: PreparedRequest) -> Response: context = prepared.context if not isinstance(context, _GoogleContext): raise DialectError("google_genai received invalid prepared context") - expected_suffix = ( - ":streamGenerateContent?alt=sse" - if context.request.scheduling.stream - else ":generateContent" - ) - if not prepared.operation.endswith(expected_suffix): - raise DialectError("google_genai received the wrong prepared operation") + operation_mismatch = _OperationVerifier( + self._model_name, + context.request.scheduling.stream, + self._version_prefix, + ).verify_prepared(prepared) + if operation_mismatch is not None: + raise DialectError( + f"google_genai received the wrong prepared operation: " + f"{operation_mismatch}" + ) body = prepared.thaw_body() raw_contents = body.get("contents") if not isinstance(raw_contents, list): @@ -1784,6 +2097,20 @@ async def execute(self, prepared: PreparedRequest) -> Response: isinstance(item, dict) for item in actual_contents ): raise DialectError("final Google request body has invalid contents") + historical_function_call_ids, historical_server_call_ids = ( + _historical_output_call_ids( + tuple(cast(list[Mapping[str, JSONValue]], actual_contents)) + ) + ) + raw_system = actual_body.get("systemInstruction") + if raw_system is None: + actual_system: Mapping[str, JSONValue] | None = None + elif isinstance(raw_system, dict): + actual_system = cast(Mapping[str, JSONValue], raw_system) + else: + raise DialectError( + "final Google request body has invalid systemInstruction" + ) request_params: dict[str, JSONValue] = { "method": request.method, "path": request.url.path, @@ -1799,7 +2126,10 @@ async def execute(self, prepared: PreparedRequest) -> Response: execution_context = _GoogleExecutionContext( request=context.request, request_params=request_params, + system_instruction=actual_system, input_contents=tuple(cast(list[Mapping[str, JSONValue]], actual_contents)), + historical_function_call_ids=historical_function_call_ids, + historical_server_call_ids=historical_server_call_ids, ) response = await self._connection.client.send(request) response.raise_for_status() diff --git a/tests/unit/cli/test_validation.py b/tests/unit/cli/test_validation.py index 270d600e..c951734a 100644 --- a/tests/unit/cli/test_validation.py +++ b/tests/unit/cli/test_validation.py @@ -475,11 +475,23 @@ def test_active_canonical_dialects_and_typed_options_are_accepted(self): }, ) ) + google = validate_eval_config( + self._config( + dialect="google_genai", + capabilities={ + "reasoning": {"effort": "high", "summary": "auto"}, + "hosted_tools": {"kinds": ["google_search"]}, + "top_logprobs": {"minimum": 20}, + "input_scoring": False, + }, + ) + ) assert chat.ok, chat.errors assert completions.ok, completions.errors assert responses.ok, responses.errors assert anthropic.ok, anthropic.errors + assert google.ok, google.errors @pytest.mark.parametrize("dialect", [None, "", 7, ["openai_chat"]]) def test_explicit_malformed_dialect_is_rejected(self, dialect): @@ -591,6 +603,17 @@ def test_dialect_specific_option_validator_is_used(self): "does not support reasoning budget_tokens" in e for e in result.errors ) + def test_google_capability_domain_is_rejected_during_static_validation(self): + result = validate_eval_config( + self._config( + dialect="google_genai", + capabilities={"top_logprobs": {"minimum": 21}}, + ) + ) + + assert not result.ok + assert any("at most 20 top logprobs" in error for error in result.errors) + def test_legacy_type_is_not_a_second_default_dialect_resolver(self): result = validate_eval_config( self._config( @@ -1694,9 +1717,9 @@ def test_active_anthropic_dialect_reconciles_to_its_runtime_plan(self, tmp_path) " model: m\n", encoding="utf-8", ) - content = config.read_text(encoding="utf-8") - - direct = validate_eval_config(yaml.safe_load(content)) + direct = validate_eval_config( + yaml.safe_load(config.read_text(encoding="utf-8")) + ) dry_run = run_dry_run(config) checks = {check["name"]: check for check in dry_run["checks"]} @@ -1709,6 +1732,45 @@ def test_active_anthropic_dialect_reconciles_to_its_runtime_plan(self, tmp_path) "anthropic_messages" ) + def test_active_google_dialect_reconciles_to_its_runtime_plan(self, tmp_path): + """Static validation and dry-run carry Google through reconciliation.""" + from sieval.cli.validation import run_dry_run + + config = tmp_path / "google-genai.yaml" + config.write_text( + "models:\n" + " m:\n" + " name: gemini-2.5-flash\n" + " type: chat\n" + " dialect: google_genai\n" + " api_base: https://generativelanguage.googleapis.com\n" + " api_key: test-key\n" + "datasets:\n" + " d:\n" + " class: sieval.datasets.aime_2024.AIME2024Dataset\n" + "tasks:\n" + " t:\n" + " class: sieval.tasks.aime_2024_0shot_gen.AIME2024ZeroShotGenTask\n" + " dataset: d\n" + " model: m\n", + encoding="utf-8", + ) + + direct = validate_eval_config( + yaml.safe_load(config.read_text(encoding="utf-8")) + ) + dry_run = run_dry_run(config) + checks = {check["name"]: check for check in dry_run["checks"]} + + assert direct.ok, direct.errors + assert dry_run["n_errors"] == 0 + assert checks["schema"]["ok"] is True + assert checks["imports"]["ok"] is True + assert checks["capability_reconcile"]["ok"] is True + binding_plans = checks["capability_reconcile"]["plan"]["binding_plans"] + assert len(binding_plans) == 1 + assert next(iter(binding_plans.values()))["dialect_id"] == "google_genai" + def test_schema_validation_failure(self, tmp_path): """Config with bad schema returns schema check failure.""" from sieval.cli.validation import run_dry_run diff --git a/tests/unit/core/models/dialects/test_google_genai.py b/tests/unit/core/models/dialects/test_google_genai.py index ff7a3a1b..e508be46 100644 --- a/tests/unit/core/models/dialects/test_google_genai.py +++ b/tests/unit/core/models/dialects/test_google_genai.py @@ -222,7 +222,6 @@ def handler(request: httpx.Request) -> httpx.Response: req = _request( input=_chat( ChatMessage("system", (TextPart("Be concise"),)), - ChatMessage("developer", (TextPart("Use JSON"),)), ChatMessage("user", (TextPart("Hello"),)), ), sampling=SamplingParams( @@ -243,9 +242,7 @@ def handler(request: httpx.Request) -> httpx.Response: assert request.headers["x-goog-api-key"] == "google-secret" assert seen["body"] == { "contents": [{"role": "user", "parts": [{"text": "Hello"}]}], - "systemInstruction": { - "parts": [{"text": "Be concise"}, {"text": "Use JSON"}] - }, + "systemInstruction": {"parts": [{"text": "Be concise"}]}, "generationConfig": { "temperature": 0.2, "topP": 0.9, @@ -278,6 +275,51 @@ def handler(request: httpx.Request) -> httpx.Response: await _run(handler, credential=None) + @pytest.mark.anyio + async def test_prepared_context_is_detached_from_mutable_request_inputs( + self, + ) -> None: + function_specs: list[Mapping[str, object]] = [{"name": "lookup"}] + + def handler(request: httpx.Request) -> httpx.Response: + assert json.loads(request.content)["tools"] == [ + {"functionDeclarations": [{"name": "lookup"}]} + ] + return httpx.Response( + 200, + json=_response( + parts=[ + { + "functionCall": { + "id": "call_1", + "name": "lookup", + "args": {}, + } + } + ] + ), + ) + + connection = _connection(handler) + try: + dialect = GoogleGenAIDialect(connection, "gemini-2.5-flash") + req = _request( + tools=ToolParams(functions=cast(Any, function_specs)), + ) + audit = RequestAudit(active_request_leaves(req)) + dialect.validate_request(req, audit, cast(Any, SimpleNamespace())) + prepared = dialect.prepare(req, audit) + audit.finish(prepared) + + function_specs.clear() + + result = await dialect.execute(prepared) + finally: + await connection.aclose() + + assert result.tool_calls is not None + assert result.tool_calls[0].name == "lookup" + @pytest.mark.anyio async def test_multimodal_parts_preserve_order(self) -> None: seen: dict[str, object] = {} @@ -314,6 +356,24 @@ def handler(request: httpx.Request) -> httpx.Response: }, ] + @pytest.mark.anyio + async def test_url_image_cannot_bypass_image_only_media_types(self) -> None: + req = _request( + input=_chat( + ChatMessage( + "user", + ( + ImagePart( + url="gs://bucket/document.pdf", media_type="application/pdf" + ), + ), + ) + ) + ) + + with pytest.raises(DialectError, match="unsupported Google image media type"): + await _run(_no_io, req) + @pytest.mark.anyio async def test_function_result_recovers_name_from_prior_call(self) -> None: seen: dict[str, object] = {} @@ -477,6 +537,30 @@ class TestPreIORejections: ), "must precede", ), + ( + _request( + input=_chat( + ChatMessage("developer", (TextPart("higher priority"),)), + ChatMessage("user", (TextPart("question"),)), + ) + ), + "developer message authority", + ), + ( + _request( + input=_chat( + ChatMessage("user", (TextPart("complete this"),)), + ChatMessage("assistant", (TextPart("partial"),)), + ) + ), + "assistant-prefill", + ), + ( + _request( + input=_chat(ChatMessage("system", (TextPart("instructions"),))) + ), + "end with a user or tool-result turn", + ), ( _request( input=_chat( @@ -550,6 +634,34 @@ async def test_provider_sampling_domains_fail_before_io( with pytest.raises(RequestAuditError): await _run(_no_io, _request(sampling=sampling)) + @pytest.mark.anyio + @pytest.mark.parametrize( + "req", + [ + _request( + sampling=SamplingParams(n=2), + scoring=ScoringParams(sampled_logprobs=True), + ), + _request( + sampling=SamplingParams(n=2), + tools=ToolParams(functions=({"name": "f"},)), + ), + _request( + sampling=SamplingParams(n=2), + tools=ToolParams(hosted=(HostedToolSpec("google_search"),)), + ), + _request( + sampling=SamplingParams(n=2), + structured_output=StructuredOutputParams(format="json_object"), + ), + ], + ) + async def test_multiple_candidates_reject_singular_channels_before_io( + self, req: Request + ) -> None: + with pytest.raises(ValueError, match="singular response channels"): + await _run(_no_io, req) + class TestReasoningAndStructuredOutput: @pytest.mark.anyio @@ -619,6 +731,81 @@ def second_handler(request: httpx.Request) -> httpx.Response: assert second.reasoning[0] is not None assert second.reasoning[0].opaque_roundtrip + @pytest.mark.anyio + async def test_opaque_history_preserves_system_instruction(self) -> None: + first = await _run( + lambda _: httpx.Response( + 200, + json=_response( + [ + { + "text": "analysis", + "thought": True, + "thoughtSignature": "sig_1", + }, + {"text": "answer"}, + ] + ), + ), + _request( + input=_chat( + ChatMessage("system", (TextPart("Always answer as JSON"),)), + ChatMessage("user", (TextPart("first"),)), + ) + ), + ) + assert first.reasoning is not None and first.reasoning[0] is not None + opaque = cast(str, first.reasoning[0].opaque_roundtrip) + envelope = json.loads(opaque) + assert envelope["systemInstruction"] == { + "parts": [{"text": "Always answer as JSON"}] + } + + def handler(request: httpx.Request) -> httpx.Response: + body = json.loads(request.content) + assert body["systemInstruction"] == envelope["systemInstruction"] + return httpx.Response( + 200, + json=_response([{"text": "next", "thoughtSignature": "sig_2"}]), + ) + + await _run( + handler, + _request( + input=_chat(ChatMessage("user", (TextPart("next"),))), + session=SessionParams( + opaque_continuation=OpaqueContinuation("google_genai", opaque) + ), + ), + ) + + @pytest.mark.anyio + async def test_opaque_history_rejects_conflicting_system_instruction(self) -> None: + opaque = json.dumps( + { + "version": 1, + "systemInstruction": {"parts": [{"text": "original"}]}, + "contents": [ + { + "role": "model", + "parts": [{"text": "answer", "thoughtSignature": "signature"}], + } + ], + } + ) + req = _request( + input=_chat( + ChatMessage("system", (TextPart("replacement"),)), + ChatMessage("user", (TextPart("next"),)), + ), + session=SessionParams( + opaque_continuation=OpaqueContinuation("google_genai", opaque) + ), + ) + + with pytest.raises(DialectError, match="systemInstruction conflicts"): + await _run(_no_io, req) + @pytest.mark.anyio async def test_requested_summary_must_be_nonempty(self) -> None: with pytest.raises(OutputContractError, match="omitted requested thoughts"): @@ -627,6 +814,19 @@ async def test_requested_summary_must_be_nonempty(self) -> None: _request(reasoning=ReasoningParams(summary="auto")), ) + @pytest.mark.anyio + async def test_requested_summary_rejects_whitespace_only_thoughts(self) -> None: + with pytest.raises(OutputContractError, match="omitted requested thoughts"): + await _run( + lambda _: httpx.Response( + 200, + json=_response( + [{"text": " \n\t ", "thought": True}, {"text": "answer"}] + ), + ), + _request(reasoning=ReasoningParams(summary="auto")), + ) + @pytest.mark.anyio @pytest.mark.parametrize("text", ['{"a":1,"a":2}', '{"a":NaN}', "not-json"]) async def test_structured_output_requires_strict_json(self, text: str) -> None: @@ -664,6 +864,176 @@ def handler(request: httpx.Request) -> httpx.Response: class TestResponseLifting: + @pytest.mark.anyio + async def test_multiple_text_candidates_preserve_choice_order(self) -> None: + result = await _run( + lambda _: httpx.Response( + 200, + json={ + "candidates": [ + { + "index": 0, + "content": {"role": "model", "parts": [{"text": "A"}]}, + "finishReason": "STOP", + }, + { + "index": 1, + "content": {"role": "model", "parts": [{"text": "B"}]}, + "finishReason": "MAX_TOKENS", + }, + ] + }, + ), + _request(sampling=SamplingParams(n=2)), + ) + + assert result.texts == ("A", "B") + assert result.finish_reasons == ("STOP", "MAX_TOKENS") + + @pytest.mark.anyio + async def test_multiple_candidates_reject_singular_response_metadata(self) -> None: + with pytest.raises(OutputContractError, match="singular citation"): + await _run( + lambda _: httpx.Response( + 200, + json={ + "candidates": [ + { + "index": 0, + "content": { + "role": "model", + "parts": [{"text": "A"}], + }, + "finishReason": "STOP", + "citationMetadata": { + "citationSources": [{"uri": "https://a.test"}] + }, + }, + { + "index": 1, + "content": { + "role": "model", + "parts": [{"text": "B"}], + }, + "finishReason": "STOP", + }, + ] + }, + ), + _request(sampling=SamplingParams(n=2)), + ) + + @pytest.mark.anyio + async def test_missing_function_call_id_is_stable_and_replayable(self) -> None: + parts = [ + { + "functionCall": {"name": "weather", "args": {"city": "Paris"}}, + "thoughtSignature": "signature", + } + ] + tools = ToolParams(functions=({"name": "weather"},)) + request = _request(tools=tools) + first = await _run( + lambda _: httpx.Response(200, json=_response(parts)), + request, + ) + repeated = await _run( + lambda _: httpx.Response(200, json=_response(parts)), + request, + ) + + assert first.tool_calls is not None + call_id = first.tool_calls[0].call_id + assert call_id.startswith("google_call_") + assert repeated.tool_calls is not None + assert repeated.tool_calls[0].call_id == call_id + assert first.reasoning is not None and first.reasoning[0] is not None + opaque = cast(str, first.reasoning[0].opaque_roundtrip) + assert repeated.reasoning is not None and repeated.reasoning[0] is not None + assert repeated.reasoning[0].opaque_roundtrip == opaque + envelope = json.loads(opaque) + assert envelope["contents"][-1]["parts"][0]["functionCall"]["id"] == call_id + + changed = await _run( + lambda _: httpx.Response(200, json=_response(parts)), + _request( + input=_chat(ChatMessage("user", (TextPart("Different input"),))), + tools=tools, + ), + ) + assert changed.tool_calls is not None + assert changed.tool_calls[0].call_id != call_id + + def handler(http_request: httpx.Request) -> httpx.Response: + body = json.loads(http_request.content) + assert body["contents"][-1]["parts"] == [ + { + "functionResponse": { + "id": call_id, + "name": "weather", + "response": {"temperature": 20}, + } + } + ] + return httpx.Response(200, json=_response()) + + await _run( + handler, + _request( + input=_chat( + ChatMessage( + "tool", + (ToolResultPart(call_id, {"temperature": 20}),), + ) + ), + session=SessionParams( + opaque_continuation=OpaqueContinuation("google_genai", opaque) + ), + ), + ) + + @pytest.mark.anyio + async def test_opaque_history_restores_answered_function_calls(self) -> None: + opaque = json.dumps( + { + "version": 1, + "contents": [ + { + "role": "model", + "parts": [ + { + "functionCall": {"id": "call_1", "name": "f"}, + "thoughtSignature": "signature", + } + ], + }, + { + "role": "user", + "parts": [ + { + "functionResponse": { + "id": "call_1", + "name": "f", + "response": {"ok": True}, + } + } + ], + }, + ], + } + ) + req = _request( + input=_chat( + ChatMessage("tool", (ToolResultPart("call_1", {"again": True}),)) + ), + session=SessionParams( + opaque_continuation=OpaqueContinuation("google_genai", opaque) + ), + ) + + with pytest.raises(DialectError, match="duplicate Google function result"): + await _run(_no_io, req) + @pytest.mark.anyio async def test_usage_logprobs_citations_grounding_and_server_tools(self) -> None: logprobs = { @@ -735,11 +1105,44 @@ async def test_usage_logprobs_citations_grounding_and_server_tools(self) -> None assert result.server_tool_uses[0].tool_type == "GOOGLE_SEARCH_WEB" assert result.server_tool_uses[0].result == {"ok": True} assert result.usage is not None - assert result.usage.total_tokens == 7 + assert result.usage.total_tokens == 6 assert result.usage.reported_total_tokens == 7 assert result.usage.reasoning_tokens == 1 assert result.usage.cached_tokens == 2 + @pytest.mark.anyio + async def test_missing_server_tool_ids_are_synthesized_and_paired(self) -> None: + parts = [ + { + "toolCall": { + "toolType": "GOOGLE_SEARCH_WEB", + "args": {"q": "weather"}, + }, + "thoughtSignature": "signature", + }, + { + "toolResponse": { + "toolType": "GOOGLE_SEARCH_WEB", + "response": {"ok": True}, + } + }, + {"text": "sunny"}, + ] + result = await _run( + lambda _: httpx.Response(200, json=_response(parts)), + _request(tools=ToolParams(hosted=(HostedToolSpec("google_search"),))), + ) + + assert result.server_tool_uses is not None + tool_id = result.server_tool_uses[0].tool_use_id + assert tool_id.startswith("google_server_") + assert result.server_tool_uses[0].result == {"ok": True} + assert result.reasoning is not None and result.reasoning[0] is not None + opaque = json.loads(cast(str, result.reasoning[0].opaque_roundtrip)) + opaque_parts = opaque["contents"][-1]["parts"] + assert opaque_parts[0]["toolCall"]["id"] == tool_id + assert opaque_parts[1]["toolResponse"]["id"] == tool_id + @pytest.mark.anyio async def test_usage_computes_total_without_claiming_provider_report(self) -> None: result = await _run( @@ -754,6 +1157,178 @@ async def test_usage_computes_total_without_claiming_provider_report(self) -> No assert result.usage.total_tokens == 7 assert result.usage.reported_total_tokens is None + @pytest.mark.anyio + async def test_matching_provider_total_is_not_redundantly_persisted(self) -> None: + result = await _run( + lambda _: httpx.Response( + 200, + json=_response( + usage={ + "promptTokenCount": 3, + "candidatesTokenCount": 4, + "totalTokenCount": 7, + } + ), + ) + ) + assert result.usage is not None + assert result.usage.total_tokens == 7 + assert result.usage.reported_total_tokens is None + + @pytest.mark.anyio + async def test_requested_top_logprobs_require_exact_cardinality(self) -> None: + logprobs = { + "chosenCandidates": [{"token": "Hi", "logProbability": -0.1}], + "topCandidates": [ + {"candidates": [{"token": "Hi", "logProbability": -0.1}]} + ], + } + with pytest.raises(OutputContractError, match="different number"): + await _run( + lambda _: httpx.Response( + 200, + json=_response(candidate_extra={"logprobsResult": logprobs}), + ), + _request( + scoring=ScoringParams( + sampled_logprobs=True, + top_logprobs=2, + ) + ), + ) + + @pytest.mark.anyio + async def test_nonempty_text_requires_chosen_logprobs(self) -> None: + logprobs = {"chosenCandidates": [], "topCandidates": []} + with pytest.raises(OutputContractError, match="without chosen logprobs"): + await _run( + lambda _: httpx.Response( + 200, + json=_response(candidate_extra={"logprobsResult": logprobs}), + ), + _request(scoring=ScoringParams(sampled_logprobs=True)), + ) + + @pytest.mark.anyio + async def test_duplicate_provider_function_call_ids_fail_loudly(self) -> None: + with pytest.raises(OutputContractError, match="duplicate Google function call"): + await _run( + lambda _: httpx.Response( + 200, + json=_response( + [ + {"functionCall": {"id": "same", "name": "f", "args": {}}}, + {"functionCall": {"id": "same", "name": "g", "args": {}}}, + ] + ), + ), + _request(tools=ToolParams(functions=({"name": "f"}, {"name": "g"}))), + ) + + @pytest.mark.anyio + async def test_provider_cannot_reuse_function_call_id_across_rounds(self) -> None: + tools = ToolParams(functions=({"name": "f"},)) + first = await _run( + lambda _: httpx.Response( + 200, + json=_response( + [ + { + "functionCall": { + "id": "call_1", + "name": "f", + "args": {}, + }, + "thoughtSignature": "signature_1", + } + ] + ), + ), + _request(tools=tools), + ) + assert first.reasoning is not None and first.reasoning[0] is not None + opaque = cast(str, first.reasoning[0].opaque_roundtrip) + + with pytest.raises(OutputContractError, match="duplicate Google function call"): + await _run( + lambda _: httpx.Response( + 200, + json=_response( + [ + { + "functionCall": { + "id": "call_1", + "name": "f", + "args": {"again": True}, + }, + "thoughtSignature": "signature_2", + } + ] + ), + ), + _request( + input=_chat( + ChatMessage( + "tool", + (ToolResultPart("call_1", {"ok": True}),), + ) + ), + session=SessionParams( + opaque_continuation=OpaqueContinuation("google_genai", opaque) + ), + tools=tools, + ), + ) + + @pytest.mark.anyio + async def test_provider_cannot_reuse_server_tool_id_across_rounds(self) -> None: + tools = ToolParams(hosted=(HostedToolSpec("google_search"),)) + first = await _run( + lambda _: httpx.Response( + 200, + json=_response( + [ + { + "toolCall": { + "id": "server_1", + "toolType": "GOOGLE_SEARCH_WEB", + "args": {"q": "first"}, + }, + "thoughtSignature": "signature_1", + } + ] + ), + ), + _request(tools=tools), + ) + assert first.reasoning is not None and first.reasoning[0] is not None + opaque = cast(str, first.reasoning[0].opaque_roundtrip) + + with pytest.raises(OutputContractError, match="duplicate Google server tool"): + await _run( + lambda _: httpx.Response( + 200, + json=_response( + [ + { + "toolCall": { + "id": "server_1", + "toolType": "GOOGLE_SEARCH_WEB", + "args": {"q": "second"}, + }, + "thoughtSignature": "signature_2", + } + ] + ), + ), + _request( + session=SessionParams( + opaque_continuation=OpaqueContinuation("google_genai", opaque) + ), + tools=tools, + ), + ) + @pytest.mark.anyio @pytest.mark.parametrize( ("payload", "message"), @@ -790,6 +1365,34 @@ async def test_missing_requested_logprobs_is_an_output_contract_error(self) -> N class TestStreamingAndAuditEvidence: + @pytest.mark.anyio + async def test_execute_rejects_absolute_operation_before_credential_leak( + self, + ) -> None: + connection = _connection(_no_io) + try: + dialect = GoogleGenAIDialect(connection, "gemini-2.5-flash") + req = _request() + audit = RequestAudit(active_request_leaves(req)) + dialect.validate_request(req, audit, cast(Any, SimpleNamespace())) + prepared = dialect.prepare(req, audit) + audit.finish(prepared) + attacker_operation = ( + "https://attacker.invalid/v1beta/models/" + "gemini-2.5-flash:generateContent" + ) + + with pytest.raises(DialectError, match="wrong prepared operation"): + await dialect.execute( + PreparedRequest( + attacker_operation, + prepared.body, + prepared.context, + ) + ) + finally: + await connection.aclose() + @pytest.mark.anyio async def test_streaming_uses_stream_operation_and_merges_deltas(self) -> None: seen: dict[str, object] = {} @@ -840,9 +1443,126 @@ def handler(request: httpx.Request) -> httpx.Response: ) assert result.texts == ("Hello",) assert result.finish_reasons == ("MAX_TOKENS",) + assert result.usage is not None + assert result.usage.total_tokens == 4 assert result.request_params is not None assert result.request_params["query"] == {"alt": "sse"} + @pytest.mark.anyio + async def test_streaming_does_not_persist_nonterminal_usage(self) -> None: + result = await _run( + lambda _: httpx.Response( + 200, + content=_sse( + { + "candidates": [ + { + "index": 0, + "content": { + "role": "model", + "parts": [{"text": "Hel"}], + }, + } + ], + "usageMetadata": { + "promptTokenCount": 2, + "candidatesTokenCount": 1, + "totalTokenCount": 3, + }, + }, + { + "candidates": [ + { + "index": 0, + "content": { + "role": "model", + "parts": [{"text": "lo"}], + }, + "finishReason": "STOP", + } + ] + }, + ), + headers={"content-type": "text/event-stream"}, + ), + _request(scheduling=SchedulingParams(stream=True)), + ) + + assert result.texts == ("Hello",) + assert result.usage is None + + @pytest.mark.anyio + async def test_streaming_preserves_url_context_citations(self) -> None: + result = await _run( + lambda _: httpx.Response( + 200, + content=_sse( + { + "candidates": [ + { + "index": 0, + "content": { + "role": "model", + "parts": [{"text": "answer"}], + }, + } + ] + }, + { + "candidates": [ + { + "index": 0, + "finishReason": "STOP", + "urlContextMetadata": { + "urlMetadata": [ + {"retrievedUrl": "https://example.test/source"} + ] + }, + } + ] + }, + ), + headers={"content-type": "text/event-stream"}, + ), + _request( + scheduling=SchedulingParams(stream=True), + tools=ToolParams(hosted=(HostedToolSpec("url_context"),)), + ), + ) + + assert result.citations is not None + assert [item.url for item in result.citations] == [ + "https://example.test/source" + ] + + @pytest.mark.anyio + @pytest.mark.parametrize("invalid_event", ["junk", [], {}]) + async def test_streaming_rejects_non_response_data_events( + self, invalid_event: object + ) -> None: + terminal = { + "candidates": [ + { + "index": 0, + "content": {"role": "model", "parts": [{"text": "answer"}]}, + "finishReason": "STOP", + } + ] + } + content = ( + f"data: {json.dumps(invalid_event)}\n\ndata: {json.dumps(terminal)}\n\n" + ).encode() + + with pytest.raises(OutputContractError, match="non-empty response object"): + await _run( + lambda _: httpx.Response( + 200, + content=content, + headers={"content-type": "text/event-stream"}, + ), + _request(scheduling=SchedulingParams(stream=True)), + ) + @pytest.mark.anyio async def test_streaming_logprobs_are_accumulated(self) -> None: def logprob(token: str, token_id: int) -> dict[str, object]: From 7d68c5509cbe134cd1221aba60c8c4b324f5e725 Mon Sep 17 00:00:00 2001 From: jack-scitix-ai Date: Sun, 13 Sep 2026 19:10:12 +0800 Subject: [PATCH 05/15] fix(models): preserve streamed Google grounding metadata --- sieval/core/models/dialects/google_genai.py | 59 ++++- .../core/models/dialects/test_google_genai.py | 221 ++++++++++++++++++ 2 files changed, 275 insertions(+), 5 deletions(-) diff --git a/sieval/core/models/dialects/google_genai.py b/sieval/core/models/dialects/google_genai.py index 4070305e..adfdcc0d 100644 --- a/sieval/core/models/dialects/google_genai.py +++ b/sieval/core/models/dialects/google_genai.py @@ -1184,6 +1184,40 @@ def _grounding(raw: object, path: str) -> GroundingMetadata | None: return GroundingMetadata(tuple(chunks), rendered) +def _merge_stream_grounding_metadata( + accumulated: dict[str, object] | None, + incoming: object, + path: str, +) -> dict[str, object]: + """Merge Google streaming grounding metadata without losing prior evidence.""" + + if not isinstance(incoming, Mapping): + raise OutputContractError(f"{path} must be an object") + result: dict[str, object] = {} if accumulated is None else accumulated + for raw_name, value in incoming.items(): + if not isinstance(raw_name, str): + raise OutputContractError(f"{path} field names must be strings") + if value is None: + continue + if raw_name in {"groundingChunks", "groundingSupports"}: + target = result.get(raw_name) + if target is None: + new_target: list[object] = [] + result[raw_name] = new_target + target = new_target + if not isinstance(target, list): + raise AssertionError("Google grounding accumulator changed shape") + cast(list[object], target).extend(_sequence(value, f"{path}.{raw_name}")) + continue + previous = result.get(raw_name) + if previous is not None and previous != value: + raise OutputContractError( + f"Google stream changed {path}.{raw_name} between events" + ) + result[raw_name] = value + return result + + def _synthetic_call_id( input_contents: Sequence[Mapping[str, JSONValue]], candidate_index: int, @@ -1992,14 +2026,29 @@ def _merge_stream( "Google stream changed a candidate finishReason" ) state["finishReason"] = finish_reason - for name in ( - "citationMetadata", - "urlContextMetadata", - "groundingMetadata", - ): + for name in ("citationMetadata", "urlContextMetadata"): value = _get(raw_candidate, name) if value is not None: + previous = state.get(name) + if previous is not None and previous != value: + raise OutputContractError( + f"Google stream changed {name} between events" + ) state[name] = value + raw_grounding = _get(raw_candidate, "groundingMetadata") + if raw_grounding is not None: + accumulated_grounding = state.get("groundingMetadata") + if accumulated_grounding is not None and not isinstance( + accumulated_grounding, dict + ): + raise AssertionError( + "Google grounding accumulator changed type" + ) + state["groundingMetadata"] = _merge_stream_grounding_metadata( + cast(dict[str, object] | None, accumulated_grounding), + raw_grounding, + "groundingMetadata", + ) raw_logprobs = _get(raw_candidate, "logprobsResult") if raw_logprobs is not None: if not isinstance(raw_logprobs, Mapping): diff --git a/tests/unit/core/models/dialects/test_google_genai.py b/tests/unit/core/models/dialects/test_google_genai.py index e508be46..033f2035 100644 --- a/tests/unit/core/models/dialects/test_google_genai.py +++ b/tests/unit/core/models/dialects/test_google_genai.py @@ -30,6 +30,7 @@ active_request_leaves, ) from sieval.core.models.dialect_registry import DIALECT_SPECS +from sieval.core.models.dialects import google_genai from sieval.core.models.dialects.google_genai import ( CAPABILITY_DECISIONS, GoogleGenAIDialect, @@ -1535,6 +1536,226 @@ async def test_streaming_preserves_url_context_citations(self) -> None: "https://example.test/source" ] + @pytest.mark.anyio + async def test_streaming_accumulates_grounding_chunks_in_event_order(self) -> None: + result = await _run( + lambda _: httpx.Response( + 200, + content=_sse( + { + "candidates": [ + { + "index": 0, + "content": { + "role": "model", + "parts": [{"text": "grounded "}], + }, + "groundingMetadata": { + "groundingChunks": [ + { + "web": { + "uri": "https://first.test", + "title": "First", + } + } + ] + }, + } + ] + }, + { + "candidates": [ + { + "index": 0, + "content": { + "role": "model", + "parts": [{"text": "answer"}], + }, + "finishReason": "STOP", + "groundingMetadata": { + "groundingChunks": [ + { + "retrievedContext": { + "uri": "https://second.test", + "title": "Second", + } + } + ] + }, + } + ] + }, + ), + headers={"content-type": "text/event-stream"}, + ), + _request( + scheduling=SchedulingParams(stream=True), + tools=ToolParams(hosted=(HostedToolSpec("google_search"),)), + ), + ) + + assert result.grounding is not None + assert [chunk.uri for chunk in result.grounding.chunks] == [ + "https://first.test", + "https://second.test", + ] + + def test_streaming_grounding_metadata_preserves_all_incremental_evidence( + self, + ) -> None: + search_entry_point = {"renderedContent": "
sources
"} + retrieval_metadata = {"googleSearchDynamicRetrievalScore": 0.75} + merged = google_genai._merge_stream_grounding_metadata( + None, + { + "groundingChunks": [{"web": {"uri": "https://first.test"}}], + "groundingSupports": [{"groundingChunkIndices": [0]}], + "searchEntryPoint": search_entry_point, + "retrievalMetadata": retrieval_metadata, + "googleMapsWidgetContextToken": "maps-token", + }, + "groundingMetadata", + ) + merged = google_genai._merge_stream_grounding_metadata( + merged, + { + "groundingChunks": [{"web": {"uri": "https://second.test"}}], + "groundingSupports": [{"groundingChunkIndices": [1]}], + "searchEntryPoint": {"renderedContent": "
sources
"}, + "retrievalMetadata": {"googleSearchDynamicRetrievalScore": 0.75}, + "googleMapsWidgetContextToken": "maps-token", + }, + "groundingMetadata", + ) + + assert merged == { + "groundingChunks": [ + {"web": {"uri": "https://first.test"}}, + {"web": {"uri": "https://second.test"}}, + ], + "groundingSupports": [ + {"groundingChunkIndices": [0]}, + {"groundingChunkIndices": [1]}, + ], + "searchEntryPoint": search_entry_point, + "retrievalMetadata": retrieval_metadata, + "googleMapsWidgetContextToken": "maps-token", + } + + @pytest.mark.anyio + async def test_streaming_accepts_identical_repeated_singular_metadata( + self, + ) -> None: + citation_metadata = {"citationSources": [{"uri": "https://citation.test"}]} + url_context_metadata = { + "urlMetadata": [{"retrievedUrl": "https://context.test"}] + } + first_candidate_metadata = { + "citationMetadata": citation_metadata, + "urlContextMetadata": url_context_metadata, + } + second_candidate_metadata = { + "citationMetadata": {"citationSources": [{"uri": "https://citation.test"}]}, + "urlContextMetadata": { + "urlMetadata": [{"retrievedUrl": "https://context.test"}] + }, + } + result = await _run( + lambda _: httpx.Response( + 200, + content=_sse( + { + "candidates": [ + { + "index": 0, + "content": { + "role": "model", + "parts": [{"text": "answer"}], + }, + **first_candidate_metadata, + } + ] + }, + { + "candidates": [ + { + "index": 0, + "finishReason": "STOP", + **second_candidate_metadata, + } + ] + }, + ), + headers={"content-type": "text/event-stream"}, + ), + _request(scheduling=SchedulingParams(stream=True)), + ) + + assert result.citations is not None + assert [item.url for item in result.citations] == [ + "https://citation.test", + "https://context.test", + ] + + @pytest.mark.anyio + @pytest.mark.parametrize( + ("metadata_name", "first_value", "second_value"), + [ + ( + "citationMetadata", + {"citationSources": [{"uri": "https://first.test"}]}, + {"citationSources": [{"uri": "https://second.test"}]}, + ), + ( + "urlContextMetadata", + {"urlMetadata": [{"retrievedUrl": "https://first.test"}]}, + {"urlMetadata": [{"retrievedUrl": "https://second.test"}]}, + ), + ( + "groundingMetadata", + {"searchEntryPoint": {"renderedContent": "first"}}, + {"searchEntryPoint": {"renderedContent": "second"}}, + ), + ], + ) + async def test_streaming_rejects_conflicting_singular_metadata( + self, + metadata_name: str, + first_value: Mapping[str, object], + second_value: Mapping[str, object], + ) -> None: + with pytest.raises(OutputContractError, match="changed .* between events"): + await _run( + lambda _: httpx.Response( + 200, + content=_sse( + { + "candidates": [ + { + "index": 0, + "content": { + "role": "model", + "parts": [{"text": "answer"}], + }, + metadata_name: first_value, + } + ] + }, + { + "candidates": [ + { + "index": 0, + "finishReason": "STOP", + metadata_name: second_value, + } + ] + }, + ), + headers={"content-type": "text/event-stream"}, + ), + _request(scheduling=SchedulingParams(stream=True)), + ) + @pytest.mark.anyio @pytest.mark.parametrize("invalid_event", ["junk", [], {}]) async def test_streaming_rejects_non_response_data_events( From 21b23ba234a997233eb74b146711ab8d2c8f2045 Mon Sep 17 00:00:00 2001 From: jack-scitix-ai Date: Sun, 13 Sep 2026 19:26:10 +0800 Subject: [PATCH 06/15] fix(models): harden Google response state handling Surface prompt blocking reasons, preserve compatible streamed text deltas, and validate hosted-tool call state in opaque continuation history. Add fail-loud regression coverage for contradictory metadata, HTTP error status handling, and malformed or forged provider responses. --- sieval/core/models/dialects/google_genai.py | 169 +++++++- .../core/models/dialects/test_google_genai.py | 367 ++++++++++++++++++ 2 files changed, 521 insertions(+), 15 deletions(-) diff --git a/sieval/core/models/dialects/google_genai.py b/sieval/core/models/dialects/google_genai.py index adfdcc0d..65c14afc 100644 --- a/sieval/core/models/dialects/google_genai.py +++ b/sieval/core/models/dialects/google_genai.py @@ -701,6 +701,8 @@ def _call_state_from_contents( ) -> tuple[dict[str, str], set[str]]: names: dict[str, str] = {} answered: set[str] = set() + server_types: dict[str, str] = {} + answered_server_calls: set[str] = set() for content_index, content in enumerate(contents): parts = content.get("parts") if not isinstance(parts, list): @@ -724,28 +726,71 @@ def _call_state_from_contents( raise DialectError(f"duplicate Google function call id {call_id!r}") names[call_id] = name response = part.get("functionResponse") - if not isinstance(response, dict): + if isinstance(response, dict): + call_id = response.get("id") + name = response.get("name") + if not isinstance(call_id, str) or not call_id: + raise DialectError( + "Google opaque functionResponse requires a non-empty id" + ) + try: + expected_name = names[call_id] + except KeyError as exc: + raise DialectError( + "Google opaque functionResponse at " + f"{content_index}.{part_index} has no prior functionCall" + ) from exc + if name != expected_name: + raise DialectError( + f"Google opaque functionResponse name changed for {call_id!r}" + ) + if call_id in answered: + raise DialectError( + f"duplicate Google function result for {call_id!r}" + ) + answered.add(call_id) + server_call = part.get("toolCall") + if isinstance(server_call, dict): + call_id = server_call.get("id") + tool_type = server_call.get("toolType") + if not isinstance(call_id, str) or not call_id: + raise DialectError("Google opaque toolCall requires a non-empty id") + if not isinstance(tool_type, str) or not tool_type: + raise DialectError( + "Google opaque toolCall requires a non-empty toolType" + ) + if call_id in server_types: + raise DialectError( + f"duplicate Google server tool call id {call_id!r}" + ) + server_types[call_id] = tool_type + server_response = part.get("toolResponse") + if not isinstance(server_response, dict): continue - call_id = response.get("id") - name = response.get("name") + call_id = server_response.get("id") + tool_type = server_response.get("toolType") if not isinstance(call_id, str) or not call_id: + raise DialectError("Google opaque toolResponse requires a non-empty id") + if not isinstance(tool_type, str) or not tool_type: raise DialectError( - "Google opaque functionResponse requires a non-empty id" + "Google opaque toolResponse requires a non-empty toolType" ) try: - expected_name = names[call_id] + expected_type = server_types[call_id] except KeyError as exc: raise DialectError( - "Google opaque functionResponse at " - f"{content_index}.{part_index} has no prior functionCall" + "Google opaque toolResponse at " + f"{content_index}.{part_index} has no prior toolCall" ) from exc - if name != expected_name: + if tool_type != expected_type: raise DialectError( - f"Google opaque functionResponse name changed for {call_id!r}" + f"Google opaque toolResponse type changed for {call_id!r}" ) - if call_id in answered: - raise DialectError(f"duplicate Google function result for {call_id!r}") - answered.add(call_id) + if call_id in answered_server_calls: + raise DialectError( + f"duplicate Google server tool result for {call_id!r}" + ) + answered_server_calls.add(call_id) return names, answered @@ -1184,6 +1229,23 @@ def _grounding(raw: object, path: str) -> GroundingMetadata | None: return GroundingMetadata(tuple(chunks), rendered) +def _prompt_block_reason(raw: object) -> str | None: + feedback = _get(raw, "promptFeedback") + if feedback is None: + return None + if not isinstance(feedback, Mapping): + raise OutputContractError("response.promptFeedback must be an object") + feedback_mapping = cast(Mapping[str, object], feedback) + if "blockReason" not in feedback_mapping: + return None + reason = feedback_mapping["blockReason"] + if not isinstance(reason, str) or not reason.strip(): + raise OutputContractError( + "response.promptFeedback.blockReason must be a non-empty string" + ) + return None if reason == "BLOCK_REASON_UNSPECIFIED" else reason + + def _merge_stream_grounding_metadata( accumulated: dict[str, object] | None, incoming: object, @@ -1218,6 +1280,55 @@ def _merge_stream_grounding_metadata( return result +def _merge_stream_text_delta( + previous: object, current: object +) -> dict[str, object] | None: + """Join wire text deltas only while they remain one semantic part.""" + + if not isinstance(previous, Mapping) or not isinstance(current, Mapping): + return None + previous_mapping = cast(Mapping[str, object], previous) + current_mapping = cast(Mapping[str, object], current) + allowed = {"text", "thought", "thoughtSignature"} + if set(previous_mapping) - allowed or set(current_mapping) - allowed: + return None + previous_text = previous_mapping.get("text") + current_text = current_mapping.get("text") + if not isinstance(previous_text, str) or not isinstance(current_text, str): + return None + if previous_mapping.get("thought") != current_mapping.get("thought"): + return None + previous_signature = previous_mapping.get("thoughtSignature") + current_signature = current_mapping.get("thoughtSignature") + # A signature closes the prior semantic part. A signature delivered with + # the next delta belongs to the part formed by joining that delta backward. + if previous_signature is not None: + return None + if current_signature is not None and ( + not isinstance(current_signature, str) or not current_signature + ): + return None + merged = dict(previous_mapping) + merged["text"] = previous_text + current_text + if "thought" in current_mapping: + merged["thought"] = current_mapping["thought"] + if current_signature is not None: + merged["thoughtSignature"] = current_signature + return merged + + +def _append_stream_parts( + accumulated: list[object], incoming: object, path: str +) -> None: + parts = list(_sequence(incoming, path)) + if accumulated and parts: + merged = _merge_stream_text_delta(accumulated[-1], parts[0]) + if merged is not None: + accumulated[-1] = merged + parts.pop(0) + accumulated.extend(parts) + + def _synthetic_call_id( input_contents: Sequence[Mapping[str, JSONValue]], candidate_index: int, @@ -1621,7 +1732,20 @@ def consume(path: str, *evidence: WireEvidence) -> None: def _lift(self, raw: object, context: _GoogleExecutionContext) -> Response: req = context.request - candidates = _sequence(_get(raw, "candidates"), "response.candidates") + block_reason = _prompt_block_reason(raw) + raw_candidates = _get(raw, "candidates") + candidates = ( + () + if raw_candidates is None and block_reason is not None + else _sequence(raw_candidates, "response.candidates") + ) + if block_reason is not None: + if candidates: + raise OutputContractError( + "Google response is contradictory: prompt was blocked " + f"with reason {block_reason}, but candidates were returned" + ) + raise OutputContractError(f"Google prompt was blocked: {block_reason}") if len(candidates) != req.sampling.n: raise OutputContractError( f"expected {req.sampling.n} Google candidates, " @@ -1987,6 +2111,7 @@ def _merge_stream( # terminal snapshot and must come from the final event. by_index: dict[int, dict[str, object]] = {} model_version: object = None + prompt_feedback: object = None for chunk_index, chunk in enumerate(chunks): if _get(chunk, "modelVersion") is not None: current_model = _get(chunk, "modelVersion") @@ -1995,6 +2120,14 @@ def _merge_stream( "Google stream changed modelVersion between events" ) model_version = current_model + current_feedback = _get(chunk, "promptFeedback") + if current_feedback is not None: + _prompt_block_reason(chunk) + if prompt_feedback is not None and current_feedback != prompt_feedback: + raise OutputContractError( + "Google stream changed promptFeedback between events" + ) + prompt_feedback = current_feedback for raw_candidate in _sequence( _get(chunk, "candidates", []), f"stream[{chunk_index}].candidates" ): @@ -2015,8 +2148,10 @@ def _merge_stream( raise OutputContractError( "Google stream candidate role is not model" ) - state["parts"] = cast(list[object], state["parts"]) + list( - _sequence(_get(content, "parts", []), "stream candidate parts") + _append_stream_parts( + cast(list[object], state["parts"]), + _get(content, "parts", []), + "stream candidate parts", ) finish_reason = _get(raw_candidate, "finishReason") if finish_reason is not None: @@ -2101,6 +2236,10 @@ def _merge_stream( ) if model_version is not None: merged["modelVersion"] = copy_json_value(model_version, "stream model") + if prompt_feedback is not None: + merged["promptFeedback"] = copy_json_value( + prompt_feedback, "stream promptFeedback" + ) return self._lift(merged, context) async def execute(self, prepared: PreparedRequest) -> Response: diff --git a/tests/unit/core/models/dialects/test_google_genai.py b/tests/unit/core/models/dialects/test_google_genai.py index 033f2035..ca8b7fab 100644 --- a/tests/unit/core/models/dialects/test_google_genai.py +++ b/tests/unit/core/models/dialects/test_google_genai.py @@ -807,6 +807,177 @@ async def test_opaque_history_rejects_conflicting_system_instruction(self) -> No with pytest.raises(DialectError, match="systemInstruction conflicts"): await _run(_no_io, req) + @pytest.mark.anyio + @pytest.mark.parametrize( + ("server_parts", "message"), + [ + ( + [ + { + "toolResponse": { + "id": "server_1", + "toolType": "GOOGLE_SEARCH_WEB", + "response": {"ok": True}, + } + } + ], + "toolResponse.*has no prior toolCall", + ), + ( + [ + { + "toolCall": { + "id": "server_1", + "toolType": "GOOGLE_SEARCH_WEB", + "args": {}, + } + }, + { + "toolCall": { + "id": "server_1", + "toolType": "GOOGLE_SEARCH_WEB", + "args": {}, + } + }, + ], + "duplicate Google server tool call", + ), + ( + [ + { + "toolCall": { + "id": "server_1", + "toolType": "GOOGLE_SEARCH_WEB", + "args": {}, + } + }, + { + "toolResponse": { + "id": "server_1", + "toolType": "URL_CONTEXT", + "response": {"ok": True}, + } + }, + ], + "toolResponse type changed", + ), + ( + [ + { + "toolCall": { + "id": "server_1", + "toolType": "GOOGLE_SEARCH_WEB", + "args": {}, + } + }, + { + "toolResponse": { + "id": "server_1", + "toolType": "GOOGLE_SEARCH_WEB", + "response": {"ok": True}, + } + }, + { + "toolResponse": { + "id": "server_1", + "toolType": "GOOGLE_SEARCH_WEB", + "response": {"ok": True}, + } + }, + ], + "duplicate Google server tool result", + ), + ], + ) + async def test_opaque_server_tool_history_rejects_invalid_call_state_before_io( + self, server_parts: list[dict[str, object]], message: str + ) -> None: + opaque = json.dumps( + { + "version": 1, + "contents": [ + { + "role": "model", + "parts": [ + { + "text": "history", + "thoughtSignature": "signature", + }, + *server_parts, + ], + } + ], + } + ) + + with pytest.raises(DialectError, match=message): + await _run( + _no_io, + _request( + session=SessionParams( + opaque_continuation=OpaqueContinuation("google_genai", opaque) + ) + ), + ) + + @pytest.mark.anyio + @pytest.mark.parametrize( + "server_parts", + [ + [ + { + "toolCall": { + "id": "server_1", + "toolType": "GOOGLE_SEARCH_WEB", + "args": {"q": "weather"}, + }, + "thoughtSignature": "signature", + } + ], + [ + { + "toolCall": { + "id": "server_1", + "toolType": "GOOGLE_SEARCH_WEB", + "args": {"q": "weather"}, + }, + "thoughtSignature": "signature", + }, + { + "toolResponse": { + "id": "server_1", + "toolType": "GOOGLE_SEARCH_WEB", + "response": {"ok": True}, + } + }, + ], + ], + ) + async def test_opaque_server_tool_history_accepts_open_or_answered_call( + self, server_parts: list[dict[str, object]] + ) -> None: + opaque = json.dumps( + { + "version": 1, + "contents": [{"role": "model", "parts": server_parts}], + } + ) + + def handler(request: httpx.Request) -> httpx.Response: + assert json.loads(request.content)["contents"][0]["parts"] == server_parts + return httpx.Response(200, json=_response()) + + result = await _run( + handler, + _request( + session=SessionParams( + opaque_continuation=OpaqueContinuation("google_genai", opaque) + ) + ), + ) + + assert result.texts == ("Hi",) + @pytest.mark.anyio async def test_requested_summary_must_be_nonempty(self) -> None: with pytest.raises(OutputContractError, match="omitted requested thoughts"): @@ -865,6 +1036,55 @@ def handler(request: httpx.Request) -> httpx.Response: class TestResponseLifting: + @pytest.mark.anyio + async def test_blocked_prompt_surfaces_provider_reason(self) -> None: + with pytest.raises( + OutputContractError, match="Google prompt was blocked: SAFETY" + ): + await _run( + lambda _: httpx.Response( + 200, + json={"promptFeedback": {"blockReason": "SAFETY"}}, + ) + ) + + @pytest.mark.anyio + async def test_unspecified_prompt_block_reason_allows_candidates(self) -> None: + payload = _response() + payload["promptFeedback"] = {"blockReason": "BLOCK_REASON_UNSPECIFIED"} + + result = await _run(lambda _: httpx.Response(200, json=payload)) + + assert result.texts == ("Hi",) + + @pytest.mark.anyio + async def test_candidates_with_prompt_block_reason_are_contradictory(self) -> None: + payload = _response() + payload["promptFeedback"] = {"blockReason": "SAFETY"} + + with pytest.raises(OutputContractError, match="contradictory.*SAFETY"): + await _run(lambda _: httpx.Response(200, json=payload)) + + @pytest.mark.anyio + @pytest.mark.parametrize( + ("prompt_feedback", "message"), + [ + ([], "promptFeedback must be an object"), + ({"blockReason": None}, "blockReason must be a non-empty string"), + ({"blockReason": ""}, "blockReason must be a non-empty string"), + ({"blockReason": " "}, "blockReason must be a non-empty string"), + ({"blockReason": 1}, "blockReason must be a non-empty string"), + ], + ) + async def test_malformed_prompt_feedback_fails_loudly( + self, prompt_feedback: object, message: str + ) -> None: + payload = _response() + payload["promptFeedback"] = prompt_feedback + + with pytest.raises(OutputContractError, match=message): + await _run(lambda _: httpx.Response(200, json=payload)) + @pytest.mark.anyio async def test_multiple_text_candidates_preserve_choice_order(self) -> None: result = await _run( @@ -1366,6 +1586,27 @@ async def test_missing_requested_logprobs_is_an_output_contract_error(self) -> N class TestStreamingAndAuditEvidence: + @pytest.mark.anyio + @pytest.mark.parametrize("stream", [False, True]) + async def test_http_error_status_fails_before_response_lifting( + self, stream: bool + ) -> None: + def handler(_: httpx.Request) -> httpx.Response: + payload = _response() + if stream: + return httpx.Response( + 400, + content=_sse(payload), + headers={"content-type": "text/event-stream"}, + ) + return httpx.Response(400, json=payload) + + with pytest.raises(httpx.HTTPStatusError): + await _run( + handler, + _request(scheduling=SchedulingParams(stream=stream)), + ) + @pytest.mark.anyio async def test_execute_rejects_absolute_operation_before_credential_leak( self, @@ -1449,6 +1690,132 @@ def handler(request: httpx.Request) -> httpx.Response: assert result.request_params is not None assert result.request_params["query"] == {"alt": "sse"} + @pytest.mark.anyio + async def test_streaming_merges_reasoning_deltas_without_crossing_boundaries( + self, + ) -> None: + result = await _run( + lambda _: httpx.Response( + 200, + content=_sse( + { + "candidates": [ + { + "index": 0, + "content": { + "role": "model", + "parts": [{"text": "ana", "thought": True}], + }, + } + ] + }, + { + "candidates": [ + { + "index": 0, + "content": { + "role": "model", + "parts": [ + { + "text": "lysis", + "thought": True, + "thoughtSignature": "signature", + }, + {"text": "answer"}, + ], + }, + "finishReason": "STOP", + } + ] + }, + ), + headers={"content-type": "text/event-stream"}, + ), + _request( + reasoning=ReasoningParams(summary="auto"), + scheduling=SchedulingParams(stream=True), + ), + ) + + assert result.texts == ("answer",) + assert result.reasoning is not None + assert result.reasoning[0] is not None + assert result.reasoning[0].text == "analysis" + opaque = json.loads(cast(str, result.reasoning[0].opaque_roundtrip)) + assert opaque["contents"][-1]["parts"] == [ + { + "text": "analysis", + "thought": True, + "thoughtSignature": "signature", + }, + {"text": "answer"}, + ] + + @pytest.mark.parametrize( + ("previous", "current"), + [ + ( + {"text": "reasoning", "thought": True}, + {"text": "answer"}, + ), + ( + {"text": "before"}, + {"functionCall": {"id": "call_1", "name": "f", "args": {}}}, + ), + ( + { + "text": "signed", + "thought": True, + "thoughtSignature": "signature", + }, + {"text": "new thought", "thought": True}, + ), + ], + ) + def test_streaming_text_merge_respects_semantic_boundaries( + self, previous: object, current: object + ) -> None: + assert google_genai._merge_stream_text_delta(previous, current) is None + + @pytest.mark.anyio + async def test_streaming_blocked_prompt_surfaces_consistent_reason(self) -> None: + feedback = {"blockReason": "SAFETY", "blockReasonMessage": "unsafe"} + + with pytest.raises( + OutputContractError, match="Google prompt was blocked: SAFETY" + ): + await _run( + lambda _: httpx.Response( + 200, + content=_sse( + {"promptFeedback": feedback}, + { + "promptFeedback": { + "blockReason": "SAFETY", + "blockReasonMessage": "unsafe", + } + }, + ), + headers={"content-type": "text/event-stream"}, + ), + _request(scheduling=SchedulingParams(stream=True)), + ) + + @pytest.mark.anyio + async def test_streaming_rejects_conflicting_prompt_feedback(self) -> None: + with pytest.raises(OutputContractError, match="changed promptFeedback"): + await _run( + lambda _: httpx.Response( + 200, + content=_sse( + {"promptFeedback": {"blockReason": "BLOCK_REASON_UNSPECIFIED"}}, + {"promptFeedback": {"blockReason": "SAFETY"}}, + ), + headers={"content-type": "text/event-stream"}, + ), + _request(scheduling=SchedulingParams(stream=True)), + ) + @pytest.mark.anyio async def test_streaming_does_not_persist_nonterminal_usage(self) -> None: result = await _run( From 07fb1a8a103cc9f73e23c6b0f8b40471269e8891 Mon Sep 17 00:00:00 2001 From: jack-scitix-ai Date: Sun, 13 Sep 2026 19:41:32 +0800 Subject: [PATCH 07/15] fix(models): reject Google stream data after finish Treat each streamed candidate finishReason as terminal while continuing to allow response-level usage-only tail events. Add regression coverage and mutation verification for the terminal-state guard. --- sieval/core/models/dialects/google_genai.py | 5 ++ .../core/models/dialects/test_google_genai.py | 76 +++++++++++++++++++ 2 files changed, 81 insertions(+) diff --git a/sieval/core/models/dialects/google_genai.py b/sieval/core/models/dialects/google_genai.py index 65c14afc..f395eda7 100644 --- a/sieval/core/models/dialects/google_genai.py +++ b/sieval/core/models/dialects/google_genai.py @@ -2141,6 +2141,11 @@ def _merge_stream( "Google stream candidate index is invalid" ) state = by_index.setdefault(index, {"parts": []}) + if "finishReason" in state: + raise OutputContractError( + f"Google stream emitted candidate {index} after its " + "terminal finishReason" + ) content = _get(raw_candidate, "content") if content is not None: role = _get(content, "role") diff --git a/tests/unit/core/models/dialects/test_google_genai.py b/tests/unit/core/models/dialects/test_google_genai.py index ca8b7fab..aae0e538 100644 --- a/tests/unit/core/models/dialects/test_google_genai.py +++ b/tests/unit/core/models/dialects/test_google_genai.py @@ -1690,6 +1690,82 @@ def handler(request: httpx.Request) -> httpx.Response: assert result.request_params is not None assert result.request_params["query"] == {"alt": "sse"} + @pytest.mark.anyio + async def test_streaming_rejects_candidate_events_after_finish_reason(self) -> None: + with pytest.raises( + OutputContractError, match="candidate 0 after its terminal finishReason" + ): + await _run( + lambda _: httpx.Response( + 200, + content=_sse( + { + "candidates": [ + { + "index": 0, + "content": { + "role": "model", + "parts": [{"text": "done"}], + }, + "finishReason": "STOP", + } + ] + }, + { + "candidates": [ + { + "index": 0, + "content": { + "role": "model", + "parts": [{"text": " extra"}], + }, + } + ] + }, + ), + headers={"content-type": "text/event-stream"}, + ), + _request(scheduling=SchedulingParams(stream=True)), + ) + + @pytest.mark.anyio + async def test_streaming_allows_usage_only_event_after_candidate_finishes( + self, + ) -> None: + result = await _run( + lambda _: httpx.Response( + 200, + content=_sse( + { + "candidates": [ + { + "index": 0, + "content": { + "role": "model", + "parts": [{"text": "done"}], + }, + "finishReason": "STOP", + } + ] + }, + { + "usageMetadata": { + "promptTokenCount": 2, + "candidatesTokenCount": 1, + "totalTokenCount": 3, + } + }, + ), + headers={"content-type": "text/event-stream"}, + ), + _request(scheduling=SchedulingParams(stream=True)), + ) + + assert result.texts == ("done",) + assert result.finish_reasons == ("STOP",) + assert result.usage is not None + assert result.usage.total_tokens == 3 + @pytest.mark.anyio async def test_streaming_merges_reasoning_deltas_without_crossing_boundaries( self, From 9a411b9e5fe93fa753c0c4ded22b4576cd1338c5 Mon Sep 17 00:00:00 2001 From: jack-scitix-ai Date: Sun, 13 Sep 2026 19:57:28 +0800 Subject: [PATCH 08/15] refactor(models): remove stale Google stream check --- sieval/core/models/dialects/google_genai.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/sieval/core/models/dialects/google_genai.py b/sieval/core/models/dialects/google_genai.py index f395eda7..bf525117 100644 --- a/sieval/core/models/dialects/google_genai.py +++ b/sieval/core/models/dialects/google_genai.py @@ -2160,11 +2160,6 @@ def _merge_stream( ) finish_reason = _get(raw_candidate, "finishReason") if finish_reason is not None: - previous = state.get("finishReason") - if previous is not None and previous != finish_reason: - raise OutputContractError( - "Google stream changed a candidate finishReason" - ) state["finishReason"] = finish_reason for name in ("citationMetadata", "urlContextMetadata"): value = _get(raw_candidate, name) From c4ec1edb6b99e106522f0705ef9f381dcaf98883 Mon Sep 17 00:00:00 2001 From: jack-scitix-ai Date: Sun, 13 Sep 2026 20:22:36 +0800 Subject: [PATCH 09/15] test(models): harden Google dialect contracts Add discriminating coverage for request rejection, opaque replay, malformed provider responses, streaming state, wire verifiers, prepared-request integrity, and final request evidence.\n\nExercise both high-level I/O boundaries and lower-level verifier contracts so guard removal or mutation fails loudly. --- .../core/models/dialects/test_google_genai.py | 816 ++++++++++++++++++ 1 file changed, 816 insertions(+) diff --git a/tests/unit/core/models/dialects/test_google_genai.py b/tests/unit/core/models/dialects/test_google_genai.py index aae0e538..d6316191 100644 --- a/tests/unit/core/models/dialects/test_google_genai.py +++ b/tests/unit/core/models/dialects/test_google_genai.py @@ -2296,3 +2296,819 @@ async def test_malformed_sse_fails_loudly(self) -> None: lambda _: httpx.Response(200, content=b"not-sse\n\n"), _request(scheduling=SchedulingParams(stream=True)), ) + + +class TestAdditionalRequestContracts: + @pytest.mark.anyio + @pytest.mark.parametrize( + ("req", "message"), + [ + ( + _request( + input=_chat(ChatMessage("user", (TextPart("x"),), name="named")) + ), + "no ChatMessage.name", + ), + ( + _request(input=_chat(ChatMessage("user", ()))), + "empty message", + ), + ( + _request(input=_chat(ChatMessage("tool", (TextPart("x"),)))), + "only tool results", + ), + ( + _request(input=_chat(ChatMessage("tool", (ImagePart(url="x"),)))), + "instruction/tool messages are text-only", + ), + ( + _request( + input=_chat(ChatMessage("user", (ToolCallPart("call", "f", {}),))) + ), + "require assistant role", + ), + ( + _request( + input=_chat( + ChatMessage("assistant", (ToolCallPart("", "f", {}),)), + ChatMessage("user", (TextPart("continue"),)), + ), + ), + "require id and name", + ), + ( + _request( + input=_chat( + ChatMessage( + "assistant", + ( + ToolCallPart("call", "f", {}), + ToolCallPart("call", "f", {}), + ), + ), + ChatMessage("user", (TextPart("continue"),)), + ) + ), + "duplicate Google function call id", + ), + ( + _request( + input=_chat( + ChatMessage("assistant", (ToolCallPart("call", "f", {}),)), + ChatMessage("user", (ToolResultPart("call", {}),)), + ) + ), + "require tool role", + ), + ( + _request( + input=_chat( + ChatMessage("assistant", (ToolCallPart("call", "f", {}),)), + ChatMessage("tool", (ToolResultPart("call", {}),)), + ChatMessage("tool", (ToolResultPart("call", {}),)), + ) + ), + "duplicate Google function result", + ), + ( + _request(input=_chat(ChatMessage("user", (ImagePart(url=""),)))), + "URLs must not be empty", + ), + ( + _request( + input=_chat( + ChatMessage( + "user", (ImagePart(data="", media_type="image/png"),) + ) + ) + ), + "inline images require data", + ), + ( + _request( + tools=ToolParams( + functions=({"type": "function", "function": "wrong"},) + ) + ), + "invalid OpenAI-shaped function declaration", + ), + ( + _request( + tools=ToolParams( + functions=({"name": "f", "parameters": {}, "future": True},) + ) + ), + "unsupported Google function declaration field", + ), + ( + _request(tools=ToolParams(functions=({"parameters": {}},))), + "requires a name", + ), + ( + _request( + tools=ToolParams( + functions=({"name": "f"},), + choice="sometimes", + ) + ), + "unsupported Google tool choice", + ), + ( + _request(tools=ToolParams(functions=({"name": "f"},), choice=1)), + "string or mapping", + ), + ( + _request( + tools=ToolParams( + functions=({"name": "f"},), + choice={"type": "function", "name": ""}, + ) + ), + "requires a name", + ), + ( + _request( + tools=ToolParams( + functions=({"name": "f"},), + choice={"type": "function", "future": True}, + ) + ), + "invalid Google function-specific tool choice", + ), + ( + _request(tools=ToolParams(parallel=True)), + "parallel tool switch", + ), + ( + _request(tools=ToolParams(hosted=(HostedToolSpec("code_execution"),))), + "unsupported Google hosted tool kind", + ), + ( + _request(scoring=ScoringParams(sampled_logprobs=True, top_logprobs=21)), + "at most 20", + ), + ( + _request(reasoning=ReasoningParams(effort="xhigh")), + "unsupported Google thinking level", + ), + ( + _request(reasoning=ReasoningParams(summary="detailed")), + "supports only summary", + ), + ( + _request( + structured_output=StructuredOutputParams( + format="json_schema", schema={}, name="named" + ) + ), + "schemas have no name", + ), + ( + _request( + structured_output=StructuredOutputParams( + format="json_schema", schema={}, strict=True + ) + ), + "no JSON-schema strict switch", + ), + ( + _request(dialect_options=DialectOptions("google_genai", {"": True})), + "keys must be non-empty", + ), + ], + ) + async def test_invalid_request_is_rejected_before_io( + self, req: Request, message: str + ) -> None: + with pytest.raises((DialectError, RequestAuditError), match=message): + await _run(_no_io, req) + + @pytest.mark.parametrize( + ("req", "message"), + [ + (_request(input=CompletionInput("prompt")), "requires ChatInput"), + ( + _request(scoring=ScoringParams(input_scoring=True)), + "cannot score input tokens", + ), + ( + _request(session=SessionParams(previous_response_id="response")), + "not a continuation handle", + ), + ( + _request( + structured_output=cast( + Any, + StructuredOutputParams( + format="json_object", schema={"type": "object"} + ), + ) + ), + "requires format='json_schema'", + ), + ], + ) + def test_low_level_validation_rejects_unsupported_or_conflicting_leaves( + self, req: Request, message: str + ) -> None: + connection = _connection(_no_io) + dialect = GoogleGenAIDialect(connection, "gemini-2.5-flash") + audit = RequestAudit(active_request_leaves(req)) + + dialect.validate_request(req, audit, cast(Any, SimpleNamespace())) + + with pytest.raises(RequestAuditError, match=message): + audit.raise_rejections() + + +class TestOpaqueContinuationBoundaries: + @staticmethod + def _payload() -> dict[str, object]: + return { + "version": 1, + "contents": [ + { + "role": "model", + "parts": [ + { + "text": "thought", + "thought": True, + "thoughtSignature": "signature", + } + ], + } + ], + } + + @pytest.mark.parametrize( + ("mutate", "message"), + [ + (lambda value: value.update(version=2), "unsupported.*version"), + (lambda value: value.update(contents=[]), "has no contents"), + (lambda value: value.update(contents=["wrong"]), "invalid shape"), + ( + lambda value: value.update(contents=[{"role": "tool", "parts": [{}]}]), + "invalid role", + ), + ( + lambda value: value.update(contents=[{"role": "user", "parts": []}]), + "has no parts", + ), + ( + lambda value: value.update( + contents=[{"role": "user", "parts": ["wrong"]}] + ), + "is not an object", + ), + ( + lambda value: value.update( + contents=[ + { + "role": "user", + "parts": [{"text": "x", "future": True}], + } + ] + ), + "unsupported field", + ), + ( + lambda value: value.update( + contents=[ + { + "role": "user", + "parts": [{"text": "x", "fileData": {}}], + } + ] + ), + "one data field", + ), + ( + lambda value: value.update( + contents=[{"role": "user", "parts": [{"text": 1}]}] + ), + "text part must contain a string", + ), + ( + lambda value: value.update( + contents=[{"role": "user", "parts": [{"fileData": "x"}]}] + ), + "fileData must be an object", + ), + ( + lambda value: value.update( + contents=[ + {"role": "model", "parts": [{"text": "x", "thought": "yes"}]} + ] + ), + "thought marker must be boolean", + ), + ( + lambda value: value.update( + contents=[ + { + "role": "model", + "parts": [{"text": "x", "thoughtSignature": ""}], + } + ] + ), + "thoughtSignature must be a non-empty string", + ), + ( + lambda value: value.update( + contents=[{"role": "model", "parts": [{"text": "x"}]}] + ), + "has no thought signature", + ), + ( + lambda value: value.update(systemInstruction={}), + "systemInstruction has invalid shape", + ), + ( + lambda value: value.update(systemInstruction={"parts": []}), + "systemInstruction has no parts", + ), + ( + lambda value: value.update(systemInstruction={"parts": [{"text": 1}]}), + "systemInstruction must be text-only", + ), + (lambda value: value.update(future=True), "invalid.*envelope"), + ], + ) + def test_malformed_opaque_continuation_is_rejected( + self, mutate: Callable[[dict[str, object]], None], message: str + ) -> None: + payload = self._payload() + mutate(payload) + + with pytest.raises(DialectError, match=message): + google_genai._decode_opaque(json.dumps(payload)) + + @pytest.mark.parametrize( + ("contents", "message"), + [ + ( + [{"role": "model", "parts": [{"functionCall": {"name": "f"}}]}], + "functionCall requires a non-empty id", + ), + ( + [{"role": "model", "parts": [{"functionCall": {"id": "c"}}]}], + "functionCall requires a non-empty name", + ), + ( + [ + { + "role": "model", + "parts": [ + {"functionCall": {"id": "c", "name": "f"}}, + {"functionCall": {"id": "c", "name": "f"}}, + ], + } + ], + "duplicate Google function call id", + ), + ( + [{"role": "user", "parts": [{"functionResponse": {"name": "f"}}]}], + "functionResponse requires a non-empty id", + ), + ( + [ + { + "role": "user", + "parts": [ + { + "functionResponse": { + "id": "missing", + "name": "f", + } + } + ], + } + ], + "has no prior functionCall", + ), + ( + [ + { + "role": "model", + "parts": [{"functionCall": {"id": "c", "name": "f"}}], + }, + { + "role": "user", + "parts": [{"functionResponse": {"id": "c", "name": "g"}}], + }, + ], + "name changed", + ), + ( + [{"role": "model", "parts": [{"toolCall": {"toolType": "SEARCH"}}]}], + "toolCall requires a non-empty id", + ), + ( + [{"role": "model", "parts": [{"toolCall": {"id": "s"}}]}], + "toolCall requires a non-empty toolType", + ), + ( + [{"role": "user", "parts": [{"toolResponse": {"toolType": "SEARCH"}}]}], + "toolResponse requires a non-empty id", + ), + ( + [{"role": "user", "parts": [{"toolResponse": {"id": "s"}}]}], + "toolResponse requires a non-empty toolType", + ), + ], + ) + def test_opaque_call_state_rejects_ambiguous_history( + self, contents: list[Mapping[str, object]], message: str + ) -> None: + with pytest.raises(DialectError, match=message): + google_genai._call_state_from_contents(cast(Any, contents)) + + +class TestAdditionalResponseContracts: + @pytest.mark.anyio + @pytest.mark.parametrize( + ("payload", "message"), + [ + ({"candidates": {}}, "response.candidates must be a sequence"), + ({"candidates": []}, "expected 1 Google candidates"), + ( + _response(candidate_extra={"content": {"role": "user", "parts": []}}), + "content role is not model", + ), + ( + _response(candidate_extra={"content": "wrong"}), + "content must be an object", + ), + ( + _response(candidate_extra={"content": {"role": "model", "parts": [1]}}), + "response part must be an object", + ), + ( + _response(parts=[{"text": "x", "future": True}]), + "unsupported field", + ), + ( + _response(parts=[{"text": "x", "thought": "yes"}]), + "thought marker must be boolean", + ), + ( + _response(parts=[{"text": "x", "thoughtSignature": ""}]), + "thoughtSignature must be a non-empty string", + ), + ( + _response(parts=[{"text": "x", "functionCall": {}}]), + "unsupported shape", + ), + ( + _response(parts=[{"text": 1}]), + "response text must be a string", + ), + ( + _response( + parts=[{"functionCall": "wrong"}], + candidate_extra=None, + ), + "functionCall must be an object", + ), + ( + _response(parts=[{"functionCall": {"name": "f", "args": []}}]), + "functionCall.args must be an object", + ), + ( + _response(parts=[{"functionCall": {"name": "", "args": {}}}]), + "functionCall.name must be a non-empty string", + ), + ( + _response(parts=[{"functionCall": {"id": 1, "name": "f", "args": {}}}]), + "functionCall.id must be a string", + ), + ( + _response(parts=[{"toolCall": "wrong"}]), + "toolCall must be an object", + ), + ( + _response( + parts=[{"toolCall": {"id": "s", "toolType": "SEARCH", "args": []}}] + ), + "toolCall.args must be an object", + ), + ( + _response( + parts=[{"toolCall": {"id": 1, "toolType": "SEARCH", "args": {}}}] + ), + "toolCall.id must be a string", + ), + ( + _response(parts=[{"toolResponse": "wrong"}]), + "toolResponse must be an object", + ), + ( + _response( + parts=[ + { + "toolResponse": { + "id": 1, + "toolType": "SEARCH", + "response": {}, + } + } + ] + ), + "toolResponse.id must be a string", + ), + ( + _response( + parts=[ + { + "toolResponse": { + "id": "s", + "toolType": "SEARCH", + "response": [], + } + } + ] + ), + "toolResponse.response must be an object", + ), + ( + _response(model_version=1), + "response.modelVersion must be a string or None", + ), + ( + _response(usage=[]), + "usageMetadata must be an object", + ), + ], + ) + async def test_malformed_provider_response_fails_loudly( + self, payload: Mapping[str, object], message: str + ) -> None: + request = _request( + tools=ToolParams( + functions=({"name": "f"},), + hosted=(HostedToolSpec("google_search"),), + ) + ) + with pytest.raises(OutputContractError, match=message): + await _run(lambda _: httpx.Response(200, json=payload), request) + + @pytest.mark.anyio + @pytest.mark.parametrize( + ("candidate_extra", "message"), + [ + ({"citationMetadata": []}, "citationMetadata must be an object"), + ( + {"citationMetadata": {"citationSources": [{}]}}, + "uri is absent", + ), + ({"urlContextMetadata": []}, "urlContextMetadata must be an object"), + ( + {"urlContextMetadata": {"urlMetadata": [{}]}}, + "omitted its URL", + ), + ({"groundingMetadata": []}, "groundingMetadata must be an object"), + ( + {"groundingMetadata": {"groundingChunks": [{}]}}, + "is ambiguous", + ), + ( + {"groundingMetadata": {"groundingChunks": [{"web": {}}]}}, + "has no representable URI", + ), + ], + ) + async def test_malformed_evidence_metadata_fails_loudly( + self, candidate_extra: Mapping[str, object], message: str + ) -> None: + with pytest.raises(OutputContractError, match=message): + await _run( + lambda _: httpx.Response( + 200, json=_response(candidate_extra=candidate_extra) + ) + ) + + @pytest.mark.anyio + @pytest.mark.parametrize( + ("usage", "message"), + [ + ({"promptTokenCount": -1}, "non-negative integer"), + ( + {"promptTokenCount": 1, "candidatesTokenCount": True}, + "non-negative integer", + ), + ( + { + "promptTokenCount": 1, + "candidatesTokenCount": 1, + "totalTokenCount": -1, + }, + "non-negative integer", + ), + ( + { + "promptTokenCount": 1, + "candidatesTokenCount": 1, + "thoughtsTokenCount": -1, + }, + "non-negative integer", + ), + ( + { + "promptTokenCount": 1, + "candidatesTokenCount": 1, + "cachedContentTokenCount": -1, + }, + "non-negative integer", + ), + ], + ) + async def test_malformed_usage_fails_loudly( + self, usage: Mapping[str, object], message: str + ) -> None: + with pytest.raises(OutputContractError, match=message): + await _run(lambda _: httpx.Response(200, json=_response(usage=usage))) + + +class TestAdditionalStreamingContracts: + @pytest.mark.anyio + @pytest.mark.parametrize( + ("content", "message"), + [ + (b": keep-alive\n\n", "no response chunks"), + (b"data: not-json\n\n", "not strict JSON"), + (b"future: x\n\n", "unsupported SSE field"), + (b"data\n\n", "malformed SSE line"), + (b"data: []\n\n", "non-empty response object"), + (b"data: {}\n\n", "non-empty response object"), + ], + ) + async def test_stream_parser_rejects_malformed_events( + self, content: bytes, message: str + ) -> None: + with pytest.raises(OutputContractError, match=message): + await _run( + lambda _: httpx.Response(200, content=content), + _request(scheduling=SchedulingParams(stream=True)), + ) + + @pytest.mark.anyio + @pytest.mark.parametrize( + ("chunks", "message"), + [ + ( + ({"candidates": [{"index": -1}]},), + "candidate index is invalid", + ), + ( + ( + { + "candidates": [ + {"index": 0, "content": {"role": "user", "parts": []}} + ] + }, + ), + "candidate role is not model", + ), + ( + ({"candidates": [{"index": 0, "groundingMetadata": []}]},), + "groundingMetadata must be an object", + ), + ( + ({"candidates": [{"index": 0, "logprobsResult": []}]},), + "logprobsResult must be an object", + ), + ( + ( + {"candidates": [{"index": 0, "finishReason": "STOP"}]}, + {"modelVersion": "changed"}, + {"modelVersion": "again"}, + ), + "changed modelVersion", + ), + ], + ) + async def test_stream_merge_rejects_invalid_state( + self, chunks: tuple[Mapping[str, object], ...], message: str + ) -> None: + with pytest.raises(OutputContractError, match=message): + await _run( + lambda _: httpx.Response(200, content=_sse(*chunks)), + _request(scheduling=SchedulingParams(stream=True)), + ) + + +class TestPreparedRequestAndVerifierContracts: + def test_all_wire_verifiers_reject_mutated_output(self) -> None: + assert ( + google_genai._InputVerifier(None, ()).verify( + {"contents": [], "systemInstruction": {"parts": []}} + ) + == "systemInstruction no longer matches system/developer messages" + ) + assert ( + google_genai._RootOptionVerifier("serviceTier", "priority").verify({}) + == "Google dialect option 'serviceTier' changed" + ) + assert ( + google_genai._ThinkingEffortVerifier("high").verify({}) + == "thinking level changed" + ) + assert ( + google_genai._ReasoningSummaryVerifier().verify({}) + == "visible-thought request changed" + ) + structured = google_genai._StructuredFormatVerifier("json_schema") + assert structured.verify({}) == "structured output MIME type changed" + assert ( + structured.verify( + {"generationConfig": {"responseMimeType": "application/json"}} + ) + == "structured output schema presence changed" + ) + functions = google_genai._FunctionsVerifier(({"name": "f"},)) + assert functions.verify({}) == "function tools changed" + hosted = google_genai._HostedToolsVerifier((HostedToolSpec("google_search"),)) + assert hosted.verify({}) == "hosted tools changed" + assert ( + hosted.verify({"tools": [{"googleSearch": {}}]}) + == "hosted-tool invocation evidence was disabled" + ) + assert ( + google_genai._ToolChoiceVerifier(cast(Any, 1)).verify({}) + == "source tool choice is invalid" + ) + assert ( + google_genai._ToolChoiceVerifier("auto").verify({}) == "tool choice changed" + ) + + @pytest.mark.anyio + @pytest.mark.parametrize( + ("mutation", "message"), + [ + (lambda body: body.update(contents={}), "invalid contents"), + (lambda body: body.update(contents=["wrong"]), "invalid content items"), + ( + lambda body: body.update(systemInstruction=[]), + "invalid systemInstruction", + ), + ], + ) + async def test_execute_rejects_mutated_prepared_body_before_io( + self, mutation: Callable[[dict[str, object]], None], message: str + ) -> None: + connection = _connection(_no_io) + try: + dialect = GoogleGenAIDialect(connection, "gemini-2.5-flash") + req = _request() + audit = RequestAudit(active_request_leaves(req)) + dialect.validate_request(req, audit, cast(Any, SimpleNamespace())) + prepared = dialect.prepare(req, audit) + body = cast(dict[str, object], prepared.thaw_body()) + mutation(body) + + with pytest.raises(DialectError, match=message): + await dialect.execute( + PreparedRequest( + prepared.operation, cast(Any, body), prepared.context + ) + ) + finally: + await connection.aclose() + + @pytest.mark.anyio + async def test_execute_rejects_invalid_prepared_context_before_io(self) -> None: + connection = _connection(_no_io) + try: + dialect = GoogleGenAIDialect(connection, "gemini-2.5-flash") + with pytest.raises(DialectError, match="invalid prepared context"): + await dialect.execute( + PreparedRequest( + "v1beta/models/gemini-2.5-flash:generateContent", {} + ) + ) + finally: + await connection.aclose() + + @pytest.mark.anyio + async def test_request_params_come_from_final_serialized_request(self) -> None: + connection = _connection(lambda _: httpx.Response(200, json=_response())) + try: + dialect = GoogleGenAIDialect(connection, "gemini-2.5-flash") + req = _request(sampling=SamplingParams(temperature=0.2)) + audit = RequestAudit(active_request_leaves(req)) + dialect.validate_request(req, audit, cast(Any, SimpleNamespace())) + prepared = dialect.prepare(req, audit) + body = cast(dict[str, object], prepared.thaw_body()) + generation = cast(dict[str, object], body["generationConfig"]) + generation["temperature"] = 0.9 + + response = await dialect.execute( + PreparedRequest(prepared.operation, cast(Any, body), prepared.context) + ) + finally: + await connection.aclose() + + assert response.request_params is not None + assert response.request_params["body"] == { + "generationConfig": {"temperature": 0.9} + } From c7a9000b2ec19bc49898c8134a2e8ffdd2db5808 Mon Sep 17 00:00:00 2001 From: jack-scitix-ai Date: Fri, 9 Oct 2026 10:02:39 +0800 Subject: [PATCH 10/15] feat(models): complete Google GenAI dialect contracts --- docs/guide/infer.md | 161 + sieval/core/models/dialect.py | 106 +- .../models/dialects/_google_genai_wire.py | 383 ++ sieval/core/models/dialects/google_genai.py | 1652 ++++++- tests/unit/cli/leaderboard/test_session.py | 66 +- tests/unit/cli/test_validation.py | 51 +- .../fixtures/google_discovery_20260928.json | 321 ++ .../dialects/test__google_genai_wire.py | 451 ++ .../core/models/dialects/test_google_genai.py | 4039 ++++++++++++++--- tests/unit/core/models/test_dialect.py | 81 +- tests/unit/core/models/test_reconcile.py | 51 + 11 files changed, 6543 insertions(+), 819 deletions(-) create mode 100644 sieval/core/models/dialects/_google_genai_wire.py create mode 100644 tests/unit/core/models/dialects/fixtures/google_discovery_20260928.json create mode 100644 tests/unit/core/models/dialects/test__google_genai_wire.py diff --git a/docs/guide/infer.md b/docs/guide/infer.md index d1543d58..8218149c 100644 --- a/docs/guide/infer.md +++ b/docs/guide/infer.md @@ -117,6 +117,167 @@ Replace the placeholder above through your configuration/secret-management workflow; omit `api_key` only when the target endpoint does not require authentication. +### Google GenAI + +To call the native Gemini `generateContent` REST API, select the dialect +explicitly and provide the Google AI endpoint and API key: + +```yaml +models: + gemini: + name: gemini-3.8-flash + type: chat + dialect: google_genai + api_base: https://generativelanguage.googleapis.com + api_key: "" +``` + +The model name may be either `gemini-...` or `models/gemini-...`. The dialect +supports the Google AI Gemini API's `v1` and `v1beta` versions. An endpoint root +defaults to `v1beta`; an explicit `/v1` or `/v1beta` suffix selects that version: + +```yaml +# Stable API; use /v1beta explicitly for the beta version. +api_base: https://generativelanguage.googleapis.com/v1 +``` + +Requests use `POST //models/:generateContent`, or +`POST //models/:streamGenerateContent?alt=sse` for streaming. +Gateway path prefixes are preserved. Unknown API versions and full resource +URLs are rejected at dialect construction before I/O. The final HTTP URL is +checked against the selected version and recorded in request evidence; explicit +`/v1` and `/v1beta` endpoints also have distinct runtime/provenance fingerprints. +Vertex AI resource paths and OAuth authentication are not mapped by this dialect. + +Function parameter schemas have a version-specific wire contract, based on +Google's [v1 Discovery schema](https://generativelanguage.googleapis.com/$discovery/rest?version=v1) +and [v1beta Discovery schema](https://generativelanguage.googleapis.com/$discovery/rest?version=v1beta) +(revision `20260928`). `v1beta` permits both `parameters` and +`parametersJsonSchema`, mutually exclusively; SiEval selects +`parametersJsonSchema` to preserve the supplied JSON Schema. For either version, +function parameters must describe an object. SiEval requires an explicit root +`type: object`: scalar/array roots, type unions, and roots whose type is only +implied by `$ref` or composition are rejected before I/O. This is SiEval's +conservative supported subset, not a claim that Google rejects every referenced +object schema. A function with no parameters may omit `parameters`, or declare +`{"type": "object", "properties": {}}`. + +`v1` exposes only `parameters`, using Google's OpenAPI `Schema` subset. +SiEval converts explicit +types, nested properties/items/`anyOf`, string enums, and supported scalar +constraints without discarding keywords. Unsupported constructs such as +`additionalProperties`, `$ref`/`$defs`, type arrays, and numeric enums are rejected +before I/O; every converted schema node must declare an explicit `type`. +Use `/v1beta` for other schema constructs, retaining the explicit object root. +The `v1` conversion encodes Schema types as uppercase enums and size limits as +int64 strings. Structured output +continues to use `generationConfig.responseJsonSchema` on both API versions. +Lossless conversion here means preserving supported structural and validation +information, not promising that Google uses every annotation during generation. +`default` and `example` are preserved unchanged; Google explicitly ignores +`default` for validation. Nested `example` is not rejected solely on the basis +of Google's root-population description. Fixed Discovery contract tests run +offline; neither evaluation nor tests fetch a changing schema at runtime. + +The credential is sent only in the `x-goog-api-key` header. As with the +Anthropic example, YAML does not expand environment-variable expressions and +the provider-neutral HTTP connection does not infer a Google-specific +environment variable. Omit `api_key` only for a compatible endpoint that does +not require authentication. + +The wire adapter implements text and image input, function tools, +`google_search` and `url_context` hosted tools, JSON output, thinking controls, +logprobs, streaming, deterministic request seeds, and opaque continuation. +Function declarations require a non-empty description and a Google-compatible +name of at most 128 characters. When hosted tools are combined with an explicit +`tools.choice: auto`, the dialect sends Google's `VALIDATED` mode because +server-side tool invocation circulation does not support `AUTO`. +Google documents combining custom functions with hosted tools as a Gemini 3 +feature. Known Gemini 1.x/2.x model identifiers therefore reject this combination +before I/O, without disabling standalone function tools, Google Search, or URL +Context where supported. Unknown provider aliases do not acquire a model-family +constraint by name; model support still requires capability admission. +Historical function calls are validated turn by turn: every model turn that +emits calls must be followed immediately by one user/tool turn containing +exactly one matching `FunctionResponse` for each call. Matching the calls and +responses only across the conversation as a whole is rejected before I/O, as +required by Google's +[function-calling guidance](https://ai.google.dev/gemini-api/docs/generate-content/function-calling). +Gemini 3 responses must also provide Google's exact non-empty function-call ID, +which is replayed unchanged in the matching `FunctionResponse`; a missing ID is +an output-contract error. The first function call in each Gemini 3 step must +also preserve its provider-issued `thoughtSignature`; unsigned canonical tool +history is rejected for the current turn, while completed earlier turns remain +accepted. Older Gemini families retain a deterministic local fallback for +responses from endpoints that omit the formerly optional ID. + +Gemini 3 tool-context circulation has an additional state contract: when +`includeServerSideToolInvocations` is enabled, each returned `toolCall` and +`toolResponse` must carry its own non-empty `thoughtSignature`. Missing signatures +fail response validation rather than producing incomplete replay state. These +parts and signatures must also survive opaque replay and final HTTP serialization, +even if a later request no longer enables hosted tools. This does not require +signatures on every text part or every parallel custom function call. See Google's +[tool-combination contract](https://ai.google.dev/gemini-api/docs/generate-content/tool-combination). + +Hosted-tool configuration follows the Google Discovery schema. `url_context` +accepts only an empty object. `google_search` accepts `timeRangeFilter` and +`searchTypes`; time ranges require `startTime` and `endTime` together as +RFC3339 timestamps with explicit UTC offsets, and the start must not be after +the end. `webSearch` and `imageSearch` are empty marker objects. Unknown +configuration keys are rejected locally rather than being deferred to a +provider 400. The schema source is Google's public +[Generative Language Discovery document](https://generativelanguage.googleapis.com/$discovery/rest?version=v1beta). +Ordinary YAML bindings do not yet supply Google model-profile outcomes, so +model-dependent capabilities such as images, reasoning, tools, structured +output, logprobs, and opaque continuation fail reconciliation with +`model_outcome_missing` instead of being assumed to work on every Gemini model +or compatible endpoint. Plain text generation, streaming, seeds, and audited +provider options are currently reachable without those profile outcomes. + +Per-image `detail`, assistant prefill, input scoring, and +`previous_response_id` are rejected before I/O. Google has no distinct +developer-message role, so a `developer` message is also rejected instead of +being silently weakened to a system instruction. + +Gemini 3 rejects `candidateCount` greater than one, so the dialect rejects +`sampling.n > 1` for Gemini 3 model identifiers before network I/O. Earlier +Gemini families retain the provider's multiple-candidate behavior where no +singular response channel makes it ambiguous. + +The dialect also applies Google's documented model-family request domains. +Gemini 2.5 models reject `reasoning.effort` and validate positive +`reasoning.budget_tokens` against the published range for Pro, Flash, or +Flash-Lite. Gemini 3 models validate `reasoning.effort` against the model's +published thinking-level table; for example, 3.1 Pro and 3.8 Flash reject +`minimal`, 3 Pro Preview accepts only `low` and `high`, and 3.1 Flash-Lite +Image accepts only `minimal` and `high`. Gemini 3 continues to accept +`reasoning.budget_tokens` for provider backward compatibility, although Google +recommends `reasoning.effort` / `thinkingLevel` for more predictable behavior. +Google's current Gemini 3 migration guidance says to remove `temperature`, +`top_p`, and `top_k`, so explicit values are rejected for Gemini 3 rather than +being recorded as effective sampling controls. These policies follow the +[Google thinking guide](https://ai.google.dev/gemini-api/docs/generate-content/thinking) +and the +[current-model migration guide](https://ai.google.dev/gemini-api/docs/latest-model). + +`reasoning.summary: auto` requests visible thoughts with `includeThoughts`. +Google documents those summaries as optional when no thought is available, so +their absence is valid; returned `thoughtSignature` values and signed history +are preserved without rewriting provider parts in the opaque continuation +channel for later turns. +When `finishReason` is `MAX_TOKENS`, partial text is preserved, requested +channels may be absent, and incomplete JSON is not misreported as a malformed +structured response; downstream truncation accounting receives the provider's +finish reason unchanged. + +Provider-only root controls can be supplied through matching +`dialect_options`, for example `safetySettings`, `serviceTier`, or `store`. +`cachedContent` and `labels` are available only on `v1beta` and are rejected on +`v1`. Options are audited against the final JSON request. Canonical fields +such as `contents`, `generationConfig`, `tools`, `model`, and `stream` cannot be +overridden there, and there is no arbitrary HTTP-header passthrough. + ## Environment Variables Custom environment variables can be passed through the YAML config's `infer.env` section. Values are injected into the inference engine process. diff --git a/sieval/core/models/dialect.py b/sieval/core/models/dialect.py index d939f0ee..deee30c6 100644 --- a/sieval/core/models/dialect.py +++ b/sieval/core/models/dialect.py @@ -13,7 +13,7 @@ from copy import deepcopy from dataclasses import dataclass, fields from enum import StrEnum -from typing import Protocol, cast, runtime_checkable +from typing import Any, Protocol, cast, runtime_checkable from sieval.core.types import JSONValue @@ -74,14 +74,33 @@ class WireVerifier(Protocol): def verify(self, body: Mapping[str, JSONValue]) -> str | None: ... -@runtime_checkable class PreparedRequestVerifier(Protocol): """Independently verify request evidence outside the JSON body.""" def verify_prepared(self, prepared: "PreparedRequest") -> str | None: ... -type WireEvidence = WireObservation | WireVerifier | PreparedRequestVerifier +@dataclass(frozen=True) +class PreparedRequestEvidence: + """Explicitly select verification against the complete prepared request. + + A wrapper keeps structural ``WireVerifier`` objects on their historical + body-only path even when they happen to expose another similarly named + method. It also validates the callable at evidence-admission time rather + than failing later with an unhelpful ``TypeError``. + """ + + verifier: PreparedRequestVerifier + + def __post_init__(self) -> None: + if not callable(getattr(self.verifier, "verify_prepared", None)): + raise TypeError( + "PreparedRequestEvidence.verifier must provide callable " + "verify_prepared()" + ) + + +type WireEvidence = WireObservation | WireVerifier | PreparedRequestEvidence @dataclass(frozen=True) @@ -238,7 +257,7 @@ def active_request_leaves(req: Request) -> dict[str, object]: } for group_name, group in grouped.items(): default = type(group)() - for item in fields(group): + for item in fields(cast(Any, group)): value = getattr(group, item.name) if value != getattr(default, item.name): active[f"{group_name}.{item.name}"] = value @@ -316,17 +335,14 @@ def consumed(self, path: str, *wire_evidence: WireEvidence) -> None: raise RequestAuditError( f"consumed request leaf {path!r} requires wire evidence" ) - if not all( - isinstance( - evidence, - (WireObservation, WireVerifier, PreparedRequestVerifier), - ) - for evidence in wire_evidence - ): - raise TypeError( - "wire evidence must be WireObservation, WireVerifier, or " - "PreparedRequestVerifier" - ) + for evidence in wire_evidence: + if isinstance(evidence, WireObservation | PreparedRequestEvidence): + continue + if not callable(getattr(evidence, "verify", None)): + raise TypeError( + "wire evidence must be WireObservation, a verifier with " + "callable verify(), or PreparedRequestEvidence" + ) self._record(path, Consumed(path, tuple(wire_evidence))) def rejected(self, path: str, reason: str) -> None: @@ -357,7 +373,7 @@ def finish(self, prepared: PreparedRequest) -> None: "unaccounted request leaves: " + ", ".join(sorted(missing)) ) - verified: set[int] = set() + verified: set[tuple[str, int]] = set() for path, decision in self._decisions.items(): if isinstance(decision, Consumed): evidence_items = decision.wire_evidence @@ -391,24 +407,39 @@ def finish(self, prepared: PreparedRequest) -> None: ) continue - verifier_id = id(evidence) + verifier = ( + evidence.verifier + if isinstance(evidence, PreparedRequestEvidence) + else evidence + ) + verifier_id = ( + "prepared" + if isinstance(evidence, PreparedRequestEvidence) + else "body", + id(verifier), + ) if verifier_id in verified: continue verified.add(verifier_id) - mismatch = ( - evidence.verify_prepared(prepared) - if isinstance(evidence, PreparedRequestVerifier) - else evidence.verify(prepared.body) - ) + if isinstance(evidence, PreparedRequestEvidence): + mismatch = evidence.verifier.verify_prepared(prepared) + else: + verify = getattr(evidence, "verify", None) + if not callable(verify): + raise AssertionError( + "admitted wire verifier lost its verify() method" + ) + mismatch = verify(prepared.body) if mismatch is not None and ( not isinstance(mismatch, str) or not mismatch.strip() ): raise TypeError( - "WireVerifier.verify() must return None or a non-empty string" + "request evidence verifier must return None or a non-empty " + "string" ) if mismatch is not None: raise RequestAuditError( - f"prepared wire mismatch for {path!r}: {mismatch}" + f"prepared request evidence mismatch for {path!r}: {mismatch}" ) @@ -480,10 +511,11 @@ def validate_runtime_binding_plan(plan: RuntimePlanView, req: Request) -> None: ) minimum = plan.capability_minimums.get("top_logprobs", {}).get("minimum") + minimum_value = int(cast(int, minimum)) if minimum is not None else None if ( - minimum is not None + minimum_value is not None and req.scoring.top_logprobs > 0 - and req.scoring.top_logprobs < int(minimum) + and req.scoring.top_logprobs < minimum_value ): raise DialectError( f"top_logprobs={req.scoring.top_logprobs} weakens required " @@ -623,9 +655,21 @@ class OutputRule: @dataclass(frozen=True) class OutputContract: rules: Mapping[str, OutputRule] + empty_outcome_finish_reasons: frozenset[str] = frozenset({"refusal"}) def __post_init__(self) -> None: object.__setattr__(self, "rules", dict(self.rules)) + empty_outcome_finish_reasons = frozenset(self.empty_outcome_finish_reasons) + if not empty_outcome_finish_reasons or any( + not isinstance(reason, str) or not reason + for reason in empty_outcome_finish_reasons + ): + raise ValueError("empty-outcome finish reasons must be non-empty strings") + object.__setattr__( + self, + "empty_outcome_finish_reasons", + empty_outcome_finish_reasons, + ) channels = { name for name, (role, _) in response_field_contract().items() @@ -645,16 +689,18 @@ def validate(self, plan: RuntimePlanView, req: Request, response: Response) -> N # response channels this particular call must return. required = set(required_response_channels(req)) finish_reasons = response.finish_reasons - refused = ( + empty_outcome = ( finish_reasons is not None and bool(finish_reasons) - and all(reason == "refusal" for reason in finish_reasons) + and all( + reason in self.empty_outcome_finish_reasons for reason in finish_reasons + ) ) for name, rule in self.rules.items(): value = getattr(response, name) if rule.guarantee is Guarantee.NEVER and value is not None: raise OutputContractError(f"dialect promised {name!r} would be absent") - if name in required and not refused: + if name in required and not empty_outcome: if rule.guarantee is not Guarantee.PRESENT_OR_ERROR: raise OutputContractError( f"{name!r} is required but has {rule.guarantee.value} guarantee" @@ -666,7 +712,7 @@ def validate(self, plan: RuntimePlanView, req: Request, response: Response) -> N if value is not None: rule.validator(value) - if req.session.opaque_continuation is not None and not refused: + if req.session.opaque_continuation is not None and not empty_outcome: reasoning = response.reasoning if reasoning is None or any( item is None diff --git a/sieval/core/models/dialects/_google_genai_wire.py b/sieval/core/models/dialects/_google_genai_wire.py new file mode 100644 index 00000000..2c559171 --- /dev/null +++ b/sieval/core/models/dialects/_google_genai_wire.py @@ -0,0 +1,383 @@ +"""Versioned GenerateContent routes and function-schema representations. + +Contracts follow Google's v1/v1beta Discovery revision 20260928: +https://generativelanguage.googleapis.com/$discovery/rest?version=v1 +https://generativelanguage.googleapis.com/$discovery/rest?version=v1beta +No runtime discovery or schema-key dropping is used. v1beta accepts both +function schema forms; this adapter selects parametersJsonSchema to preserve +the shared IR's JSON Schema. v1 requires the narrower Google Schema form. +Lossless conversion preserves the supported schema's structure and constraints; +annotations are carried unchanged, not promised to affect model generation. +Google explicitly ignores default for validation. Its example documentation +describes root population, not an explicit rejection of nested input examples. + +AI-Generated Code - GPT-6 (OpenAI) +""" + +import math +import re +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from types import MappingProxyType +from typing import Literal, cast + +import httpx + +from sieval.core.models._shared import copy_json_value +from sieval.core.models.dialect import DialectError +from sieval.core.types import JSONValue + + +@dataclass(frozen=True) +class GoogleWireContract: + api_version: Literal["v1", "v1beta"] + function_schema_field: Literal["parameters", "parametersJsonSchema"] + unavailable_fields: Mapping[str, frozenset[str]] + + +_V1_UNAVAILABLE_FIELDS = MappingProxyType( + { + "request": frozenset({"cachedContent", "labels"}), + "generationConfig": frozenset( + { + "audioTranscriptionConfig", + "enableAffectiveDialog", + "speechConfig", + "translationConfig", + } + ), + "tool": frozenset({"computerUse", "mcpServers"}), + "functionDeclaration": frozenset( + {"behavior", "parametersJsonSchema", "responseJsonSchema"} + ), + "part": frozenset({"audioTranscription", "speechMetadata"}), + "functionResponse": frozenset({"parts", "scheduling", "willContinue"}), + "inlineData": frozenset({"displayName"}), + "fileData": frozenset({"displayName"}), + } +) +GOOGLE_WIRE_CONTRACTS: Mapping[str, GoogleWireContract] = MappingProxyType( + { + "v1": GoogleWireContract("v1", "parameters", _V1_UNAVAILABLE_FIELDS), + "v1beta": GoogleWireContract( + "v1beta", "parametersJsonSchema", MappingProxyType({}) + ), + } +) + + +def resolve_wire_endpoint(base_url: httpx.URL) -> tuple[GoogleWireContract, httpx.URL]: + """Resolve a root or versioned gateway URL without mutating its client.""" + + if base_url.query or base_url.fragment: + raise ValueError("Google api_base must not contain query data or fragments") + path = base_url.path.rstrip("/") + segments = path.split("/") + # Only the terminal segment selects the Google API version. Earlier + # version-like segments may belong to a gateway's own routing scheme. + if "models" in segments or ( + segments[-1] not in GOOGLE_WIRE_CONTRACTS + and re.match(r"v[0-9]", segments[-1], re.IGNORECASE) + ): + raise ValueError( + "Google api_base must be an endpoint root or end in /v1 or /v1beta" + ) + version = "v1beta" + if segments[-1] in GOOGLE_WIRE_CONTRACTS: + version = segments.pop() + path = "/".join(segments) + return GOOGLE_WIRE_CONTRACTS[version], base_url.copy_with(path=f"{path}/") + + +_SCHEMA_TYPES = frozenset( + {"string", "number", "integer", "boolean", "array", "object", "null"} +) +_SCHEMA_STRINGS = frozenset({"title", "description", "format", "pattern"}) +_SCHEMA_COUNTS = frozenset( + {"minItems", "maxItems", "minLength", "maxLength", "minProperties", "maxProperties"} +) +_SCHEMA_NUMBERS = frozenset({"minimum", "maximum"}) +_SCHEMA_STRING_LISTS = frozenset({"required", "propertyOrdering", "enum"}) +_SCHEMA_KEYS = ( + _SCHEMA_STRINGS + | _SCHEMA_COUNTS + | _SCHEMA_NUMBERS + | _SCHEMA_STRING_LISTS + | {"type", "properties", "items", "anyOf", "nullable", "default", "example"} +) +_INT64_MAX = 2**63 - 1 + + +def validate_function_parameters(raw: object) -> None: + """Require an explicit object root for function arguments, not results. + + This is SiEval's conservative admission rule for Google's object-schema + requirement. It does not infer the root type from refs or composition, nor + rewrite a schema to insert a type. Omitting parameters entirely is valid. + """ + + if ( + not isinstance(raw, Mapping) + or cast(Mapping[str, object], raw).get("type") != "object" + ): + raise DialectError( + "Google function parameters must describe an object with explicit " + "type: object" + ) + + +def _schema_object(raw: object, path: str) -> dict[str, JSONValue]: + if not isinstance(raw, Mapping): + raise DialectError(f"{path}: Google v1 Schema requires an object") + try: + copied = copy_json_value(raw, path) + except (TypeError, ValueError) as exc: + raise DialectError(f"{path}: invalid Google v1 Schema: {exc}") from exc + assert isinstance(copied, dict) + unsupported = set(copied) - _SCHEMA_KEYS + if unsupported: + raise DialectError( + f"{path}: Google v1 Schema cannot represent keyword(s) " + f"{', '.join(sorted(unsupported))}; use /v1beta for JSON Schema" + ) + return dict(copied) + + +def _check_scalar(schema: Mapping[str, JSONValue], key: str, path: str) -> None: + value = schema[key] + if key in _SCHEMA_STRINGS and not isinstance(value, str): + raise DialectError(f"{path}.{key}: Google v1 Schema requires a string") + if key == "nullable" and not isinstance(value, bool): + raise DialectError(f"{path}.{key}: Google v1 Schema requires a boolean") + if key in _SCHEMA_NUMBERS: + try: + exact = ( + isinstance(value, (int, float)) + and not isinstance(value, bool) + and math.isfinite(value) + and float(value) == value + ) + except OverflowError: + exact = False + if not exact: + raise DialectError( + f"{path}.{key}: Google v1 Schema requires an exactly representable " + "finite double" + ) + if key in _SCHEMA_STRING_LISTS: + if not isinstance(value, list) or any(not isinstance(x, str) for x in value): + raise DialectError(f"{path}.{key}: Google v1 Schema requires strings") + if len(value) != len(set(cast(list[str], value))): + raise DialectError( + f"{path}.{key}: Google v1 Schema requires unique strings" + ) + if key == "enum" and not value: + raise DialectError( + f"{path}.enum: Google v1 Schema requires a non-empty enum" + ) + + +def lower_function_schema( + raw: object, path: str = "tools.functions.parameters" +) -> dict[str, JSONValue]: + """Encode a lossless subset, requiring an explicit type on every node.""" + + source = _schema_object(raw, path) + type_name = source.get("type") + if not isinstance(type_name, str) or type_name not in _SCHEMA_TYPES: + raise DialectError( + f"{path}.type: Google v1 Schema requires a single explicit JSON Schema type" + ) + result: dict[str, JSONValue] = {"type": type_name.upper()} + for key, value in source.items(): + if key == "type": + continue + if key in _SCHEMA_COUNTS: + if ( + isinstance(value, bool) + or not isinstance(value, int) + or not 0 <= value <= _INT64_MAX + ): + raise DialectError( + f"{path}.{key}: Google v1 Schema requires a non-negative int64" + ) + result[key] = str(value) + elif key == "properties": + if not isinstance(value, Mapping): + raise DialectError( + f"{path}.properties: Google v1 Schema requires an object" + ) + property_schemas: dict[str, JSONValue] = {} + for name, child in value.items(): + assert isinstance(name, str) # _schema_object checked JSON keys. + property_schemas[name] = lower_function_schema( + child, f"{path}.properties.{name}" + ) + result[key] = property_schemas + elif key == "items": + result[key] = lower_function_schema(value, f"{path}.items") + elif key == "anyOf": + if not isinstance(value, list) or not value: + raise DialectError( + f"{path}.anyOf: Google v1 Schema requires a non-empty array" + ) + result[key] = [ + lower_function_schema(child, f"{path}.anyOf[{index}]") + for index, child in enumerate(value) + ] + else: + _check_scalar(source, key, path) + if key == "enum" and type_name != "string": + raise DialectError( + f"{path}.enum: Google v1 Schema only maps string enums" + ) + result[key] = value + return result + + +def inspect_function_schema( + raw: object, path: str = "final function parameters" +) -> dict[str, JSONValue]: + """Decode the actual v1 wire form independently of request lowering. + + The audit compares this result to the original JSON Schema. Calling the + encoder here would let an encoder mutation also rewrite its own oracle. + Only schema nodes are traversed; annotation values and property names are + opaque, even when they happen to contain keys such as ``type``. + """ + + wire = _schema_object(raw, path) + wire_type = wire.get("type") + if not isinstance(wire_type, str) or wire_type not in { + "STRING", + "NUMBER", + "INTEGER", + "BOOLEAN", + "ARRAY", + "OBJECT", + "NULL", + }: + raise DialectError( + f"{path}.type: Google v1 wire Schema requires an uppercase type" + ) + decoded: dict[str, JSONValue] = {} + for name, value in wire.items(): + if name == "type": + decoded[name] = wire_type.lower() + elif name in _SCHEMA_COUNTS: + if ( + not isinstance(value, str) + or re.fullmatch(r"0|[1-9][0-9]{0,18}", value) is None + ): + raise DialectError( + f"{path}.{name}: Google v1 wire Schema requires an int64 string" + ) + count = int(value) + if count > _INT64_MAX: + raise DialectError( + f"{path}.{name}: Google v1 wire Schema int64 is out of range" + ) + decoded[name] = count + elif name == "properties": + if not isinstance(value, Mapping): + raise DialectError( + f"{path}.properties: Google v1 wire Schema requires an object" + ) + properties: dict[str, JSONValue] = {} + for property_name, property_schema in value.items(): + assert isinstance( + property_name, str + ) # _schema_object checked JSON keys. + properties[property_name] = inspect_function_schema( + property_schema, f"{path}.properties.{property_name}" + ) + decoded[name] = properties + elif name == "items": + decoded[name] = inspect_function_schema(value, f"{path}.items") + elif name == "anyOf": + if not isinstance(value, list) or not value: + raise DialectError( + f"{path}.anyOf: Google v1 wire Schema requires a non-empty array" + ) + decoded[name] = [ + inspect_function_schema(child, f"{path}.anyOf[{index}]") + for index, child in enumerate(value) + ] + else: + _check_scalar(wire, name, path) + if name == "enum" and wire_type != "STRING": + raise DialectError( + f"{path}.enum: Google v1 wire Schema only maps string enums" + ) + decoded[name] = value + return decoded + + +def validate_wire_body( + body: Mapping[str, JSONValue], contract: GoogleWireContract +) -> None: + """Validate versioned fields and parameter roots at actual schema locations. + + Free-form JSON (function arguments, results, annotations, JSON Schema) + must not be searched by key name: a user may legitimately name a property + ``cachedContent`` or ``parametersJsonSchema``. + """ + + def check(raw: object, shape: str, path: str) -> None: + if isinstance(raw, Mapping): + forbidden = contract.unavailable_fields.get(shape, frozenset()) & raw.keys() + if forbidden: + raise DialectError( + f"{path}: field(s) {', '.join(sorted(forbidden))} are unavailable " + f"in Google {contract.api_version}" + ) + + def sequence(raw: object) -> Sequence[object]: + return ( + raw + if isinstance(raw, Sequence) and not isinstance(raw, str | bytes) + else () + ) + + check(body, "request", "request") + check(body.get("generationConfig"), "generationConfig", "generationConfig") + for index, tool in enumerate(sequence(body.get("tools"))): + check(tool, "tool", f"tools[{index}]") + if not isinstance(tool, Mapping): + continue + tool_mapping = cast(Mapping[str, JSONValue], tool) + for declaration in sequence(tool_mapping.get("functionDeclarations")): + check( + declaration, + "functionDeclaration", + f"tools[{index}].functionDeclarations", + ) + if not isinstance(declaration, Mapping): + continue + declaration_mapping = cast(Mapping[str, JSONValue], declaration) + for field in ("parameters", "response"): + if field in declaration and f"{field}JsonSchema" in declaration: + raise DialectError( + f"{field} and {field}JsonSchema are mutually exclusive" + ) + if field in declaration: + schema = inspect_function_schema( + declaration_mapping[field], f"function.{field}" + ) + if field == "parameters": + validate_function_parameters(schema) + if "parametersJsonSchema" in declaration: + validate_function_parameters( + declaration_mapping["parametersJsonSchema"] + ) + for content in (*sequence(body.get("contents")), body.get("systemInstruction")): + if not isinstance(content, Mapping): + continue + content_mapping = cast(Mapping[str, JSONValue], content) + for part in sequence(content_mapping.get("parts")): + if not isinstance(part, Mapping): + continue + check(part, "part", "contents.parts") + part_mapping = cast(Mapping[str, JSONValue], part) + for field in ("functionResponse", "inlineData", "fileData"): + check(part_mapping.get(field), field, f"contents.parts.{field}") diff --git a/sieval/core/models/dialects/google_genai.py b/sieval/core/models/dialects/google_genai.py index bf525117..beb87683 100644 --- a/sieval/core/models/dialects/google_genai.py +++ b/sieval/core/models/dialects/google_genai.py @@ -11,9 +11,11 @@ import hashlib import json import math +import re from collections.abc import Callable, Mapping, Sequence from copy import deepcopy from dataclasses import dataclass, field +from datetime import UTC, datetime from types import MappingProxyType from typing import cast from urllib.parse import quote @@ -43,6 +45,7 @@ OutputContractError, OutputRule, PreparedRequest, + PreparedRequestEvidence, RequestAudit, RuntimePlanView, WireEvidence, @@ -77,6 +80,15 @@ ) from sieval.core.types import JSONValue +from ._google_genai_wire import ( + GoogleWireContract, + inspect_function_schema, + lower_function_schema, + resolve_wire_endpoint, + validate_function_parameters, + validate_wire_body, +) + _THINKING_LEVELS = frozenset({"minimal", "low", "medium", "high"}) _HOSTED_TOOL_KINDS = frozenset({"google_search", "url_context"}) _IMAGE_MEDIA_TYPES = frozenset( @@ -92,6 +104,151 @@ } ) _OPAQUE_VERSION = 1 +_INT32_MIN = -(2**31) +_INT32_MAX = 2**31 - 1 +_CHAT_ROLES = frozenset({"system", "developer", "user", "assistant", "tool"}) +_FUNCTION_NAME_PATTERN = re.compile(r"[A-Za-z0-9_:.\-]{1,128}") +_RFC3339_TIMESTAMP_PATTERN = re.compile( + r"(?P\d{4}-\d{2}-\d{2})T" + r"(?P