From 4f6452c5d8ae5b4bfb4972f88b941cf779b413cf Mon Sep 17 00:00:00 2001 From: Osher-Elhadad Date: Thu, 30 Jul 2026 18:45:42 +0300 Subject: [PATCH] fix: make structured output work with reasoning models and empty responses Semantic SPARC calls failed on every request against watsonx gpt-oss-120b, and native structured output was unusable on Azure/OpenAI. Four independent defects, all in the shared validating-client path: 1. `json_schema_to_pydantic_model` was lossy. Nested object properties collapsed to a bare `dict`, array `items` to a bare `list`, and `minimum`/`maximum`/`enum`/`additionalProperties` were dropped entirely. Providers therefore received a much weaker schema than the one the response was later validated against, so models emitted out-of-vocabulary enum values and extra keys that then failed validation. `enum` is now a real `Literal` so it survives inside `items`, where a Field-level constraint cannot reach. 2. Azure/OpenAI native structured output always raised `TypeError: You tried to pass a BaseModel class to chat.completions.create()`. Both clients register `create`, not `parse`, so the schema is now rendered as the equivalent `{"type": "json_schema", ..., "strict": true}` dict via a `_render_native_schema` hook. 3. The retry loop never retried a contentless response (#115). The generate call sat outside the `try`, so the `ValueError: No content or tool calls found in response` raised by `_parse_llm_response` escaped immediately and the configured `retries` were never used. The call is now inside the `try` and `ValueError` is caught alongside `OutputValidationError`. Two related failure modes are fixed with it: an empty reply no longer appends an empty assistant turn (several backends answer a padded conversation with another empty response, burning every attempt), and a reply truncated by the token limit escalates `max_tokens` instead of re-asking with a budget already known to be too small. 4. Models that ignore `response_format` were still sent it. Chain-of- thought models such as gpt-oss on watsonx return empty content when it is present, so the schema is now injected into the system prompt for them, selected per model from litellm's own capability data rather than a hardcoded list. A negative answer is only trusted for a model litellm actually knows, so unknown models keep the previous behavior. `_parse_llm_response` also falls back to `reasoning_content` when `content` is empty, which is the long-term ALTK fix requested in rossoctl/cortex#676 and removes the need for the downstream monkey-patch. Verified against live providers with the real SPARC `function_selection_appropriateness` schema (nested object arrays, `["string","null"]` unions, `additionalProperties: false`): - watsonx `openai/gpt-oss-120b`: end-to-end SPARC reflection went from `decision: error` on every call to a correct APPROVE/REJECT verdict with zero LLM errors, stable across repeated runs. - Azure `gpt-4o-2024-08-06`: native structured output went from a hard `TypeError` to valid output; SPARC returns correct verdicts. - `anthropic/claude-haiku-4-5` via litellm: valid, all required fields. Fixes #115 Signed-off-by: Osher-Elhadad --- altk/core/llm/output_parser.py | 238 ++++++++++++++++--- altk/core/llm/providers/litellm/litellm.py | 42 ++++ altk/core/llm/providers/openai/openai.py | 25 ++ tests/core/test_litellm_structured_output.py | 76 ++++++ tests/core/test_validating_llm_client.py | 221 ++++++++++++++++- 5 files changed, 565 insertions(+), 37 deletions(-) create mode 100644 tests/core/test_litellm_structured_output.py diff --git a/altk/core/llm/output_parser.py b/altk/core/llm/output_parser.py index d5a9fc1b..307319b8 100644 --- a/altk/core/llm/output_parser.py +++ b/altk/core/llm/output_parser.py @@ -5,6 +5,7 @@ Any, Dict, List, + Literal, Optional, Type, TypeVar, @@ -59,25 +60,59 @@ def json_schema_to_pydantic_model( } def _map_object_for_prop(prop_schema: Dict[str, Any]) -> Type: - """Return dict/str for a property whose declared type is ``object``. + """Return a model/dict/str for a property whose declared type is ``object``. A property is "free-form" if it has no ``properties`` sub-schema; the - OpenAI workaround only applies to those. + OpenAI workaround only applies to those. An object *with* properties is + recursed into so nested constraints survive the conversion — a bare + ``dict`` would erase them and let providers emit output that fails the + original JSON Schema. """ - if free_form_object_as_str and "properties" not in prop_schema: + if "properties" in prop_schema: + return json_schema_to_pydantic_model( + prop_schema, + model_name=f"{model_name}_{_next_nested_id()}", + free_form_object_as_str=free_form_object_as_str, + ) + if free_form_object_as_str: return str return dict + _nested_count = [0] + + def _next_nested_id() -> int: + _nested_count[0] += 1 + return _nested_count[0] + def parse_type( type_def: Union[str, List[str], None], prop_schema: Dict[str, Any], ) -> Type[T]: def _lookup(t: str) -> Type: - return ( - _map_object_for_prop(prop_schema) - if t == "object" - else type_mapping.get(t, Any) - ) + if t == "object": + return _map_object_for_prop(prop_schema) + if t == "array": + return _map_array_for_prop(prop_schema) + return type_mapping.get(t, Any) + + def _map_array_for_prop(prop_schema: Dict[str, Any]) -> Type: + """Preserve ``items`` so array element constraints are not lost.""" + items = prop_schema.get("items") + if not isinstance(items, dict): + return list + item_type = parse_type(items.get("type"), items) + return List[item_type] # type: ignore[valid-type] + + # ``enum`` becomes a real ``Literal`` type so the choices survive even + # inside ``items``, where a Field-level constraint could not reach. + enum_values = prop_schema.get("enum") + if enum_values and all( + isinstance(v, (str, int, bool)) or v is None for v in enum_values + ): + literal = Literal[tuple(enum_values)] # type: ignore[valid-type] + if isinstance(type_def, list) and "null" in type_def: + return Optional[literal] # type: ignore[return-value] + return literal # type: ignore[return-value] if isinstance(type_def, list): python_types = [_lookup(t) for t in type_def] @@ -93,14 +128,38 @@ def _lookup(t: str) -> Type: return _lookup(type_def) return Any # type: ignore[return-value] + # JSON Schema keyword -> Pydantic ``Field`` argument. Carrying these over + # keeps provider-native structured output faithful to the source schema + # (a dropped ``minimum``/``enum`` shows up later as a validation failure). + _CONSTRAINT_ARGS = { + "minimum": "ge", + "maximum": "le", + "exclusiveMinimum": "gt", + "exclusiveMaximum": "lt", + "minLength": "min_length", + "maxLength": "max_length", + "minItems": "min_length", + "maxItems": "max_length", + "pattern": "pattern", + } + for prop_name, prop_schema in schema.get("properties", {}).items(): field_type: Any = parse_type(prop_schema.get("type"), prop_schema) default = ... if prop_name in required_fields else None description = prop_schema.get("description", None) - field_args = {"description": description} if description else {} + field_args: Dict[str, Any] = {"description": description} if description else {} + for json_kw, field_kw in _CONSTRAINT_ARGS.items(): + if json_kw in prop_schema and field_kw not in field_args: + field_args[field_kw] = prop_schema[json_kw] fields[prop_name] = (field_type, Field(default, **field_args)) - return create_model(model_name, **fields) # type: ignore + model = create_model(model_name, **fields) # type: ignore + # Mirror ``additionalProperties: false`` — providers with strict structured + # output need it, and without it the model may invent extra keys that the + # original schema then rejects. + if schema.get("additionalProperties") is False: + model.model_config["extra"] = "forbid" + return model def relax_freeform_object_schema(schema: Dict[str, Any]) -> Dict[str, Any]: @@ -129,6 +188,26 @@ class OutputValidationError(Exception): """Raised when LLM output cannot be validated against the provided schema.""" +def _is_truncated(raw: Any) -> bool: + """Return ``True`` when *raw* was cut off by the token limit. + + A ``finish_reason`` of ``"length"`` means the model never finished writing, + so whatever came back cannot be valid JSON. Reasoning ("thinking") models + hit this routinely: the reasoning tokens consume the whole budget and the + content field arrives empty. + """ + choices = getattr(raw, "choices", None) or ( + raw.get("choices", []) if isinstance(raw, dict) else [] + ) + if not choices: + return False + first = choices[0] + finish = getattr(first, "finish_reason", None) or ( + first.get("finish_reason") if isinstance(first, dict) else None + ) + return finish == "length" + + class ValidatingLLMClient(BaseLLMClient, ABC): """ An LLMClient wrapper enforcing output structure via: @@ -178,6 +257,10 @@ def __init__( self.default_generation_kwargs: Dict[str, Any] = dict( default_generation_kwargs or {} ) + # Set by the wrapped parser: was the most recent reply cut off by the + # token limit? Retries use it to grow ``max_tokens`` instead of + # re-asking with a budget already known to be too small. + self._last_response_truncated: bool = False super().__init__(**base_kwargs) # Wrap the subclass's _parse_llm_response so empty / malformed LLM # outputs retry gracefully (the retry loop treats "" as invalid) @@ -204,8 +287,18 @@ def configure_validation( self.default_generation_kwargs = dict(default_generation_kwargs) return self - @staticmethod - def _build_safe_parse(orig): # noqa: ANN001, ANN205 + #: Budget assumed to be in play when the provider applied its own default + #: (``max_tokens`` was never passed) and the reply came back truncated. + ASSUMED_PROVIDER_MAX_TOKENS: int = 1024 + #: Ceiling for the retry escalation, so a stuck model cannot grow forever. + MAX_TOKENS_ESCALATION_LIMIT: int = 16384 + + def _escalate_max_tokens(self, current: Optional[int]) -> int: + """Return a larger ``max_tokens`` for the next attempt after truncation.""" + base = current or self.ASSUMED_PROVIDER_MAX_TOKENS + return min(base * 4, self.MAX_TOKENS_ESCALATION_LIMIT) + + def _build_safe_parse(self, orig): # noqa: ANN001, ANN205 """Wrap ``_parse_llm_response`` so parse failures become retry-worthy empty strings instead of raising. Also surfaces a targeted warning when a reasoning-only response exhausted the token budget.""" @@ -214,6 +307,7 @@ def _build_safe_parse(orig): # noqa: ANN001, ANN205 _logger = _logging.getLogger("altk.core.llm.output_parser") def _safe_parse(raw): # noqa: ANN001, ANN202 + self._last_response_truncated = _is_truncated(raw) try: return orig(raw) except (ValueError, KeyError): @@ -251,6 +345,34 @@ def _safe_parse(raw): # noqa: ANN001, ANN202 def provider_class(cls) -> Type[Any]: """Return the underlying SDK client class, e.g. openai.OpenAI.""" + def supports_native_structured_output(self) -> bool: + """Whether the target model honors a native structured-output kwarg. + + Defaults to ``True`` (previous behavior). Providers that can tell which + models support it override this; when it returns ``False`` the schema is + injected into the system prompt instead, because a model that ignores + ``response_format`` cannot be constrained by it. + """ + return True + + def _render_native_schema( + self, schema: Union[Dict[str, Any], Type[BaseModel], Type[Any]] + ) -> Any: + """Render *schema* into the value this provider expects for its native + structured-output kwarg (``schema_field``). + + The default converts a JSON Schema dict into a Pydantic model, which is + what litellm accepts. Providers whose SDK rejects a model class + override this — see the OpenAI/Azure clients, which need a + ``{"type": "json_schema", ...}`` dict for ``chat.completions.create``. + """ + if isinstance(schema, dict): + return json_schema_to_pydantic_model( + schema, + free_form_object_as_str=self.free_form_object_as_str, + ) + return schema + @abstractmethod def _register_methods(self) -> None: """ @@ -406,19 +528,19 @@ def generate( if self.prompt_based_validation: include_schema_in_system_prompt = True schema_field = None + # Models that ignore a native schema kwarg must be steered by the + # prompt instead; sending ``response_format`` to them is at best a + # no-op and at worst returns empty content. + elif schema_field and not self.supports_native_structured_output(): + include_schema_in_system_prompt = True + schema_field = None current = prompt instr = None if include_schema_in_system_prompt: instr = self._make_instruction(schema) current = self._inject_system(prompt, instr) if schema_field: - kwargs[schema_field] = schema - if isinstance(schema, dict): - new_schema = json_schema_to_pydantic_model( - schema, - free_form_object_as_str=self.free_form_object_as_str, - ) - kwargs[schema_field] = new_schema + kwargs[schema_field] = self._render_native_schema(schema) last_error: Optional[str] = None for _ in range(1, retries + 1): @@ -434,13 +556,39 @@ def generate( "include_schema_in_system_prompt", ] } - raw = super()._generate(**{"prompt": current, **filtered_kwargs}) + raw = "" try: + # Inside the try: ``_parse_llm_response`` raises ValueError for a + # contentless reply, and that must be retried like any other + # invalid output rather than aborting the whole call. + raw = super()._generate(**{"prompt": current, **filtered_kwargs}) if isinstance(raw, str): return self._validate(raw, schema) return raw - except OutputValidationError as e: + except (OutputValidationError, ValueError) as e: + # ValueError covers providers whose ``_parse_llm_response`` + # rejects an empty/contentless response ("No content or tool + # calls found in response"). Without it, a single blank reply + # from the backend aborts the whole call and the configured + # ``retries`` are never used. last_error = str(e) + # An empty response carries no mistake to correct: it is a + # transient backend failure (empty content under load). Retry + # the original prompt untouched — appending correction turns + # would grow a conversation that several backends answer with + # another empty response, burning every remaining attempt. + if not (isinstance(raw, str) and raw.strip()): + # Truncated by the token limit? Re-asking with the same + # budget yields the identical truncation, so grow it. This + # is the common failure for reasoning models, whose + # "thinking" tokens can consume a small default budget + # (watsonx defaults to 1024) before any content is emitted. + if self._last_response_truncated: + kwargs["max_tokens"] = self._escalate_max_tokens( + kwargs.get("max_tokens") + ) + current = self._inject_system(prompt, instr) if instr else prompt + continue correction = ( f"The previous response did not conform: {last_error}\nPlease correct it." " And remember to output ONLY the requested schema, without any additional text." @@ -484,19 +632,19 @@ async def generate_async( if self.prompt_based_validation: include_schema_in_system_prompt = True schema_field = None + # Models that ignore a native schema kwarg must be steered by the + # prompt instead; sending ``response_format`` to them is at best a + # no-op and at worst returns empty content. + elif schema_field and not self.supports_native_structured_output(): + include_schema_in_system_prompt = True + schema_field = None current = prompt instr = None if include_schema_in_system_prompt: instr = self._make_instruction(schema) current = self._inject_system(prompt, instr) if schema_field: - kwargs[schema_field] = schema - if isinstance(schema, dict): - new_schema = json_schema_to_pydantic_model( - schema, - free_form_object_as_str=self.free_form_object_as_str, - ) - kwargs[schema_field] = new_schema + kwargs[schema_field] = self._render_native_schema(schema) last_error: Optional[str] = None for _ in range(1, retries + 1): @@ -512,15 +660,41 @@ async def generate_async( "include_schema_in_system_prompt", ] } - raw = await super()._generate_async( - **{"prompt": current, **filtered_kwargs} - ) + raw = "" try: + # Inside the try: ``_parse_llm_response`` raises ValueError for a + # contentless reply, and that must be retried like any other + # invalid output rather than aborting the whole call. + raw = await super()._generate_async( + **{"prompt": current, **filtered_kwargs} + ) if isinstance(raw, str): return self._validate(raw, schema) return raw - except OutputValidationError as e: + except (OutputValidationError, ValueError) as e: + # ValueError covers providers whose ``_parse_llm_response`` + # rejects an empty/contentless response ("No content or tool + # calls found in response"). Without it, a single blank reply + # from the backend aborts the whole call and the configured + # ``retries`` are never used. last_error = str(e) + # An empty response carries no mistake to correct: it is a + # transient backend failure (empty content under load). Retry + # the original prompt untouched — appending correction turns + # would grow a conversation that several backends answer with + # another empty response, burning every remaining attempt. + if not (isinstance(raw, str) and raw.strip()): + # Truncated by the token limit? Re-asking with the same + # budget yields the identical truncation, so grow it. This + # is the common failure for reasoning models, whose + # "thinking" tokens can consume a small default budget + # (watsonx defaults to 1024) before any content is emitted. + if self._last_response_truncated: + kwargs["max_tokens"] = self._escalate_max_tokens( + kwargs.get("max_tokens") + ) + current = self._inject_system(prompt, instr) if instr else prompt + continue correction = ( f"The previous response did not conform: {last_error}\nPlease correct it." " And remember to output ONLY the requested schema, without any additional text." diff --git a/altk/core/llm/providers/litellm/litellm.py b/altk/core/llm/providers/litellm/litellm.py index 7ec917fc..9d7cd060 100644 --- a/altk/core/llm/providers/litellm/litellm.py +++ b/altk/core/llm/providers/litellm/litellm.py @@ -158,6 +158,16 @@ def _parse_llm_response(self, raw: Any) -> Union[str, LLMResponse]: if not content: content = first.get("delta", {}).get("content", first.get("text", "")) + # Chain-of-thought models (e.g. gpt-oss on watsonx) can put the answer in + # ``reasoning_content`` and leave ``content`` empty. Prefer it over + # raising, so a usable response isn't discarded as "no content". + if not content and not tool_calls and msg: + reasoning = getattr(msg, "reasoning_content", None) or ( + msg.get("reasoning_content") if isinstance(msg, dict) else None + ) + if reasoning: + content = reasoning + if not content and not tool_calls: raise ValueError("No content or tool calls found in response") @@ -303,6 +313,28 @@ def provider_class(cls) -> Type[Any]: """ return litellm # type: ignore + def supports_native_structured_output(self) -> bool: + """Whether this model honors a native ``response_format`` schema. + + Uses litellm's per-model capability data, so newly supported models are + picked up without changes here. Models that lack it (e.g. gpt-oss on + watsonx, ollama, gemini) silently ignore ``response_format`` — some + return empty content when it is sent — so the caller falls back to + injecting the schema into the system prompt instead. + """ + try: + if litellm.supports_response_schema(model=self.model_path): + return True + # ``False`` is also what litellm returns for a model it has no + # metadata for, which would silently downgrade every unknown model. + # Only trust a negative answer when the model is actually known. + litellm.get_model_info(model=self.model_path) + return False + except Exception: + # Unknown model: assume native support and let validation + retries + # catch it, preserving the previous behavior. + return True + def _register_methods(self) -> None: """ Register how to call litellm methods - only chat modes are supported: @@ -421,6 +453,16 @@ def _parse_llm_response(self, raw: Any) -> Union[str, LLMResponse]: if not content: content = first.get("delta", {}).get("content", first.get("text", "")) + # Chain-of-thought models (e.g. gpt-oss on watsonx) can put the answer in + # ``reasoning_content`` and leave ``content`` empty. Prefer it over + # raising, so a usable response isn't discarded as "no content". + if not content and not tool_calls and msg: + reasoning = getattr(msg, "reasoning_content", None) or ( + msg.get("reasoning_content") if isinstance(msg, dict) else None + ) + if reasoning: + content = reasoning + if not content and not tool_calls: raise ValueError("No content or tool calls found in response") diff --git a/altk/core/llm/providers/openai/openai.py b/altk/core/llm/providers/openai/openai.py index 765e5c18..34bb8eed 100644 --- a/altk/core/llm/providers/openai/openai.py +++ b/altk/core/llm/providers/openai/openai.py @@ -81,6 +81,31 @@ def transform_min_tokens(value: Any, mode: Any) -> dict[str, Any]: class BaseValidatingOpenAIClient(ValidatingLLMClient): """Base class for validating OpenAI and Azure OpenAI clients with shared parameter mapping""" + def _render_native_schema(self, schema: Any) -> Any: + """Render *schema* as an OpenAI ``response_format`` payload. + + ``chat.completions.create`` rejects a Pydantic model class outright + ("You tried to pass a `BaseModel` class ... use chat.completions.parse() + instead"), so the base implementation cannot be used as-is. Emitting the + equivalent ``{"type": "json_schema", ..., "strict": True}`` dict keeps + native structured output working on the ``create`` method that both the + OpenAI and Azure clients already register. + """ + from openai.lib._pydantic import to_strict_json_schema + from pydantic import BaseModel + + model = super()._render_native_schema(schema) + if not (isinstance(model, type) and issubclass(model, BaseModel)): + return model + return { + "type": "json_schema", + "json_schema": { + "name": model.__name__, + "strict": True, + "schema": to_strict_json_schema(model), + }, + } + def _setup_parameter_mapper(self) -> None: """Set up parameter mapper for OpenAI-compatible APIs""" self._parameter_mapper = ParameterMapper() diff --git a/tests/core/test_litellm_structured_output.py b/tests/core/test_litellm_structured_output.py new file mode 100644 index 00000000..dfc9c38e --- /dev/null +++ b/tests/core/test_litellm_structured_output.py @@ -0,0 +1,76 @@ +"""Offline tests for the LiteLLM validating client's response handling. + +Covers two production failures seen with reasoning ("thinking") models such as +``gpt-oss`` on watsonx: + +- the answer arrives in ``reasoning_content`` while ``content`` is empty, and +- the model ignores a native ``response_format`` schema entirely, so the schema + has to be injected into the system prompt instead. +""" + +from __future__ import annotations + +import pytest + +from altk.core.llm.providers.litellm.litellm import LiteLLMClientOutputVal +from altk.core.llm.providers.litellm.watsonx import WatsonxLiteLLMClientOutputVal + + +def _response(content=None, reasoning=None, finish="stop"): + """Build a litellm-shaped response object.""" + from litellm.types.utils import Choices, Message, ModelResponse + + kwargs = {"content": content, "role": "assistant"} + if reasoning is not None: + kwargs["reasoning_content"] = reasoning + return ModelResponse( + choices=[Choices(message=Message(**kwargs), finish_reason=finish, index=0)] + ) + + +class TestReasoningContentFallback: + """``_parse_llm_response`` must not discard a usable reasoning-only reply.""" + + def test_reasoning_content_used_when_content_empty(self): + client = LiteLLMClientOutputVal.__new__(LiteLLMClientOutputVal) + raw = _response(content="", reasoning='{"a": 1}') + # Call the unwrapped parser directly: the instance-level wrapper is + # installed in __init__, which we skip here on purpose. + assert LiteLLMClientOutputVal._parse_llm_response(client, raw) == '{"a": 1}' + + def test_content_still_preferred_over_reasoning(self): + client = LiteLLMClientOutputVal.__new__(LiteLLMClientOutputVal) + raw = _response(content='{"real": true}', reasoning="thinking out loud") + assert ( + LiteLLMClientOutputVal._parse_llm_response(client, raw) == '{"real": true}' + ) + + def test_still_raises_when_nothing_usable(self): + client = LiteLLMClientOutputVal.__new__(LiteLLMClientOutputVal) + with pytest.raises(ValueError, match="No content or tool calls"): + LiteLLMClientOutputVal._parse_llm_response(client, _response(content="")) + + +class TestNativeStructuredOutputCapability: + """Native ``response_format`` is only used where the model honors it.""" + + @pytest.mark.parametrize( + "model_name, expected", + [ + # Reasoning model with no response-schema support: must fall back. + ("openai/gpt-oss-120b", False), + # Model litellm reports as supporting response schemas. + ("mistralai/mistral-large", True), + ], + ) + def test_watsonx_capability_is_per_model(self, model_name, expected): + client = WatsonxLiteLLMClientOutputVal.__new__(WatsonxLiteLLMClientOutputVal) + client.model_path = f"watsonx/{model_name}" + assert client.supports_native_structured_output() is expected + + def test_unknown_model_assumes_native_support(self): + """Unknown models keep the previous behavior rather than silently + switching every call to prompt-based validation.""" + client = LiteLLMClientOutputVal.__new__(LiteLLMClientOutputVal) + client.model_path = "some-provider/not-a-real-model-xyz" + assert client.supports_native_structured_output() is True diff --git a/tests/core/test_validating_llm_client.py b/tests/core/test_validating_llm_client.py index a0c7f1dd..9e0a665f 100644 --- a/tests/core/test_validating_llm_client.py +++ b/tests/core/test_validating_llm_client.py @@ -16,11 +16,13 @@ from __future__ import annotations +import json import logging from types import SimpleNamespace from typing import Any, Type import pytest +from pydantic import BaseModel from altk.core.llm.output_parser import ( OutputValidationError, @@ -92,10 +94,11 @@ def test_freeform_object_flag_switches_to_str(self): ) assert m.model_fields["a"].annotation is str - def test_freeform_flag_keeps_nested_objects_as_dict(self): - # only free-form (no properties) converts; an object with properties - # keeps its dict shape (OpenAI can still satisfy additionalProperties - # when the sub-schema is fully specified). + def test_freeform_flag_recurses_into_nested_objects(self): + # Only a free-form object (no properties) becomes a str. An object that + # *has* properties is recursed into as a nested model, so its fields + # survive into the schema handed to the provider — a bare ``dict`` would + # erase them and let the model emit output the real schema rejects. m = json_schema_to_pydantic_model( { "type": "object", @@ -110,7 +113,9 @@ def test_freeform_flag_keeps_nested_objects_as_dict(self): free_form_object_as_str=True, ) assert m.model_fields["flat"].annotation is str - assert m.model_fields["structured"].annotation is dict + nested = m.model_fields["structured"].annotation + assert issubclass(nested, BaseModel) + assert set(nested.model_fields) == {"x"} # --------------------------------------------------------------------------- @@ -304,3 +309,209 @@ def _parse_llm_response(self, raw): # will be wrapped out = c2._parse_llm_response(raw) assert out == "" assert any("reasoning" in r.message.lower() for r in caplog.records) + + +# --------------------------------------------------------------------------- +# Schema fidelity: constraints must survive the Pydantic round-trip, otherwise +# provider-native structured output is handed a weaker schema than the one the +# response is later validated against. +# --------------------------------------------------------------------------- + + +class TestSchemaFidelity: + def test_numeric_bounds_survive(self): + m = json_schema_to_pydantic_model( + { + "type": "object", + "properties": { + "score": {"type": "integer", "minimum": 1, "maximum": 5} + }, + } + ) + prop = m.model_json_schema()["properties"]["score"] + assert prop["minimum"] == 1 + assert prop["maximum"] == 5 + + def test_enum_survives_inside_array_items(self): + # A Field-level constraint cannot reach into ``items``; only a real + # Literal type does. This is what made reasoning models emit + # out-of-vocabulary values that failed the original schema. + m = json_schema_to_pydantic_model( + { + "type": "object", + "properties": { + "kinds": { + "type": "array", + "items": {"type": "string", "enum": ["A", "B"]}, + } + }, + } + ) + emitted = json.dumps(m.model_json_schema()) + assert '"A"' in emitted and '"B"' in emitted + + def test_nested_array_object_properties_survive(self): + m = json_schema_to_pydantic_model( + { + "type": "object", + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "properties": {"rationale": {"type": "string"}}, + "required": ["rationale"], + }, + } + }, + } + ) + assert "rationale" in json.dumps(m.model_json_schema()) + + def test_additional_properties_false_forbids_extras(self): + m = json_schema_to_pydantic_model( + { + "type": "object", + "properties": {"a": {"type": "string"}}, + "additionalProperties": False, + } + ) + assert m.model_config.get("extra") == "forbid" + + def test_enum_with_null_type_is_optional(self): + m = json_schema_to_pydantic_model( + { + "type": "object", + "properties": { + "t": {"type": ["string", "null"], "enum": ["x", "y", None]} + }, + } + ) + # Must accept None without raising. + assert m(t=None).t is None + + +# --------------------------------------------------------------------------- +# Retry behavior for empty / truncated responses (altk-boost#115). +# --------------------------------------------------------------------------- + + +class TestEmptyResponseRetries: + def test_value_error_is_retried_not_propagated(self, monkeypatch): + """A provider raising ValueError('No content...') must consume retries.""" + observed: list = [] + calls = {"n": 0} + from altk.core.llm.base import BaseLLMClient + + def fake_generate(self, **kwargs): + observed.append(kwargs) + calls["n"] += 1 + if calls["n"] == 1: + raise ValueError("No content or tool calls found in response") + return '{"a": "ok"}' + + monkeypatch.setattr(BaseLLMClient, "_generate", fake_generate, raising=True) + c = _FakeValidating(prompt_based_validation=True, client=object()) + out = c.generate( + [], + schema={ + "type": "object", + "properties": {"a": {"type": "string"}}, + "required": ["a"], + }, + retries=3, + ) + assert out == {"a": "ok"} + assert calls["n"] == 2 + + def test_empty_reply_retries_original_prompt_not_a_growing_thread( + self, monkeypatch + ): + """No empty assistant turn is appended, and the retry re-sends the + original prompt: several backends answer a padded conversation with + another empty response, which would burn every attempt.""" + observed: list = [] + _install_scripted_generate(monkeypatch, observed, ["", '{"a": "ok"}']) + c = _FakeValidating(prompt_based_validation=True, client=object()) + out = c.generate( + [{"role": "user", "content": "hi"}], + schema={ + "type": "object", + "properties": {"a": {"type": "string"}}, + "required": ["a"], + }, + retries=3, + ) + assert out == {"a": "ok"} + retry_msgs = observed[-1]["prompt"] + assert not any( + m.get("role") == "assistant" and not (m.get("content") or "") + for m in retry_msgs + ), "an empty assistant turn must never be sent" + assert len(retry_msgs) == len(observed[0]["prompt"]) + + def test_truncated_reply_escalates_max_tokens(self, monkeypatch): + """finish_reason='length' means the budget was too small; re-asking with + the same budget would truncate identically, so it must grow.""" + observed: list = [] + from altk.core.llm.base import BaseLLMClient + + truncated = { + "choices": [{"message": {"content": ""}, "finish_reason": "length"}] + } + scripted = [truncated, '{"a": "ok"}'] + + def fake_generate(self, **kwargs): + observed.append(kwargs) + raw = scripted.pop(0) + return self._parse_llm_response(raw) + + monkeypatch.setattr(BaseLLMClient, "_generate", fake_generate, raising=True) + + class _RealisticParse(_FakeValidating): + """A truncated reply carries no text: a real provider parser raises + on the missing content and the wrapper turns that into ``""``.""" + + def _parse_llm_response(self, raw): + if isinstance(raw, dict): + raise ValueError("No content or tool calls found in response") + return str(raw) + + c = _RealisticParse(prompt_based_validation=True, client=object()) + out = c.generate( + [{"role": "user", "content": "hi"}], + schema={ + "type": "object", + "properties": {"a": {"type": "string"}}, + "required": ["a"], + }, + retries=3, + max_tokens=1024, + ) + assert out == {"a": "ok"} + assert observed[0]["max_tokens"] == 1024 + assert observed[-1]["max_tokens"] > 1024 + + def test_native_schema_skipped_when_model_lacks_support(self, monkeypatch): + """A model that ignores response_format must be steered by the prompt + instead — sending the kwarg is a no-op there at best.""" + observed: list = [] + _install_scripted_generate(monkeypatch, observed, ['{"a": "ok"}']) + + class _NoNative(_FakeValidating): + def supports_native_structured_output(self) -> bool: + return False + + c = _NoNative(client=object()) + c.generate( + [{"role": "user", "content": "hi"}], + schema={ + "type": "object", + "properties": {"a": {"type": "string"}}, + "required": ["a"], + }, + schema_field="response_format", + ) + assert "response_format" not in observed[-1] + # schema went into a system message instead + assert observed[-1]["prompt"][0]["role"] == "system"