diff --git a/altk/core/llm/output_parser.py b/altk/core/llm/output_parser.py index d5a9fc1b..307319b8 100644 --- a/altk/core/llm/output_parser.py +++ b/altk/core/llm/output_parser.py @@ -5,6 +5,7 @@ Any, Dict, List, + Literal, Optional, Type, TypeVar, @@ -59,25 +60,59 @@ def json_schema_to_pydantic_model( } def _map_object_for_prop(prop_schema: Dict[str, Any]) -> Type: - """Return dict/str for a property whose declared type is ``object``. + """Return a model/dict/str for a property whose declared type is ``object``. A property is "free-form" if it has no ``properties`` sub-schema; the - OpenAI workaround only applies to those. + OpenAI workaround only applies to those. An object *with* properties is + recursed into so nested constraints survive the conversion — a bare + ``dict`` would erase them and let providers emit output that fails the + original JSON Schema. """ - if free_form_object_as_str and "properties" not in prop_schema: + if "properties" in prop_schema: + return json_schema_to_pydantic_model( + prop_schema, + model_name=f"{model_name}_{_next_nested_id()}", + free_form_object_as_str=free_form_object_as_str, + ) + if free_form_object_as_str: return str return dict + _nested_count = [0] + + def _next_nested_id() -> int: + _nested_count[0] += 1 + return _nested_count[0] + def parse_type( type_def: Union[str, List[str], None], prop_schema: Dict[str, Any], ) -> Type[T]: def _lookup(t: str) -> Type: - return ( - _map_object_for_prop(prop_schema) - if t == "object" - else type_mapping.get(t, Any) - ) + if t == "object": + return _map_object_for_prop(prop_schema) + if t == "array": + return _map_array_for_prop(prop_schema) + return type_mapping.get(t, Any) + + def _map_array_for_prop(prop_schema: Dict[str, Any]) -> Type: + """Preserve ``items`` so array element constraints are not lost.""" + items = prop_schema.get("items") + if not isinstance(items, dict): + return list + item_type = parse_type(items.get("type"), items) + return List[item_type] # type: ignore[valid-type] + + # ``enum`` becomes a real ``Literal`` type so the choices survive even + # inside ``items``, where a Field-level constraint could not reach. + enum_values = prop_schema.get("enum") + if enum_values and all( + isinstance(v, (str, int, bool)) or v is None for v in enum_values + ): + literal = Literal[tuple(enum_values)] # type: ignore[valid-type] + if isinstance(type_def, list) and "null" in type_def: + return Optional[literal] # type: ignore[return-value] + return literal # type: ignore[return-value] if isinstance(type_def, list): python_types = [_lookup(t) for t in type_def] @@ -93,14 +128,38 @@ def _lookup(t: str) -> Type: return _lookup(type_def) return Any # type: ignore[return-value] + # JSON Schema keyword -> Pydantic ``Field`` argument. Carrying these over + # keeps provider-native structured output faithful to the source schema + # (a dropped ``minimum``/``enum`` shows up later as a validation failure). + _CONSTRAINT_ARGS = { + "minimum": "ge", + "maximum": "le", + "exclusiveMinimum": "gt", + "exclusiveMaximum": "lt", + "minLength": "min_length", + "maxLength": "max_length", + "minItems": "min_length", + "maxItems": "max_length", + "pattern": "pattern", + } + for prop_name, prop_schema in schema.get("properties", {}).items(): field_type: Any = parse_type(prop_schema.get("type"), prop_schema) default = ... if prop_name in required_fields else None description = prop_schema.get("description", None) - field_args = {"description": description} if description else {} + field_args: Dict[str, Any] = {"description": description} if description else {} + for json_kw, field_kw in _CONSTRAINT_ARGS.items(): + if json_kw in prop_schema and field_kw not in field_args: + field_args[field_kw] = prop_schema[json_kw] fields[prop_name] = (field_type, Field(default, **field_args)) - return create_model(model_name, **fields) # type: ignore + model = create_model(model_name, **fields) # type: ignore + # Mirror ``additionalProperties: false`` — providers with strict structured + # output need it, and without it the model may invent extra keys that the + # original schema then rejects. + if schema.get("additionalProperties") is False: + model.model_config["extra"] = "forbid" + return model def relax_freeform_object_schema(schema: Dict[str, Any]) -> Dict[str, Any]: @@ -129,6 +188,26 @@ class OutputValidationError(Exception): """Raised when LLM output cannot be validated against the provided schema.""" +def _is_truncated(raw: Any) -> bool: + """Return ``True`` when *raw* was cut off by the token limit. + + A ``finish_reason`` of ``"length"`` means the model never finished writing, + so whatever came back cannot be valid JSON. Reasoning ("thinking") models + hit this routinely: the reasoning tokens consume the whole budget and the + content field arrives empty. + """ + choices = getattr(raw, "choices", None) or ( + raw.get("choices", []) if isinstance(raw, dict) else [] + ) + if not choices: + return False + first = choices[0] + finish = getattr(first, "finish_reason", None) or ( + first.get("finish_reason") if isinstance(first, dict) else None + ) + return finish == "length" + + class ValidatingLLMClient(BaseLLMClient, ABC): """ An LLMClient wrapper enforcing output structure via: @@ -178,6 +257,10 @@ def __init__( self.default_generation_kwargs: Dict[str, Any] = dict( default_generation_kwargs or {} ) + # Set by the wrapped parser: was the most recent reply cut off by the + # token limit? Retries use it to grow ``max_tokens`` instead of + # re-asking with a budget already known to be too small. + self._last_response_truncated: bool = False super().__init__(**base_kwargs) # Wrap the subclass's _parse_llm_response so empty / malformed LLM # outputs retry gracefully (the retry loop treats "" as invalid) @@ -204,8 +287,18 @@ def configure_validation( self.default_generation_kwargs = dict(default_generation_kwargs) return self - @staticmethod - def _build_safe_parse(orig): # noqa: ANN001, ANN205 + #: Budget assumed to be in play when the provider applied its own default + #: (``max_tokens`` was never passed) and the reply came back truncated. + ASSUMED_PROVIDER_MAX_TOKENS: int = 1024 + #: Ceiling for the retry escalation, so a stuck model cannot grow forever. + MAX_TOKENS_ESCALATION_LIMIT: int = 16384 + + def _escalate_max_tokens(self, current: Optional[int]) -> int: + """Return a larger ``max_tokens`` for the next attempt after truncation.""" + base = current or self.ASSUMED_PROVIDER_MAX_TOKENS + return min(base * 4, self.MAX_TOKENS_ESCALATION_LIMIT) + + def _build_safe_parse(self, orig): # noqa: ANN001, ANN205 """Wrap ``_parse_llm_response`` so parse failures become retry-worthy empty strings instead of raising. Also surfaces a targeted warning when a reasoning-only response exhausted the token budget.""" @@ -214,6 +307,7 @@ def _build_safe_parse(orig): # noqa: ANN001, ANN205 _logger = _logging.getLogger("altk.core.llm.output_parser") def _safe_parse(raw): # noqa: ANN001, ANN202 + self._last_response_truncated = _is_truncated(raw) try: return orig(raw) except (ValueError, KeyError): @@ -251,6 +345,34 @@ def _safe_parse(raw): # noqa: ANN001, ANN202 def provider_class(cls) -> Type[Any]: """Return the underlying SDK client class, e.g. openai.OpenAI.""" + def supports_native_structured_output(self) -> bool: + """Whether the target model honors a native structured-output kwarg. + + Defaults to ``True`` (previous behavior). Providers that can tell which + models support it override this; when it returns ``False`` the schema is + injected into the system prompt instead, because a model that ignores + ``response_format`` cannot be constrained by it. + """ + return True + + def _render_native_schema( + self, schema: Union[Dict[str, Any], Type[BaseModel], Type[Any]] + ) -> Any: + """Render *schema* into the value this provider expects for its native + structured-output kwarg (``schema_field``). + + The default converts a JSON Schema dict into a Pydantic model, which is + what litellm accepts. Providers whose SDK rejects a model class + override this — see the OpenAI/Azure clients, which need a + ``{"type": "json_schema", ...}`` dict for ``chat.completions.create``. + """ + if isinstance(schema, dict): + return json_schema_to_pydantic_model( + schema, + free_form_object_as_str=self.free_form_object_as_str, + ) + return schema + @abstractmethod def _register_methods(self) -> None: """ @@ -406,19 +528,19 @@ def generate( if self.prompt_based_validation: include_schema_in_system_prompt = True schema_field = None + # Models that ignore a native schema kwarg must be steered by the + # prompt instead; sending ``response_format`` to them is at best a + # no-op and at worst returns empty content. + elif schema_field and not self.supports_native_structured_output(): + include_schema_in_system_prompt = True + schema_field = None current = prompt instr = None if include_schema_in_system_prompt: instr = self._make_instruction(schema) current = self._inject_system(prompt, instr) if schema_field: - kwargs[schema_field] = schema - if isinstance(schema, dict): - new_schema = json_schema_to_pydantic_model( - schema, - free_form_object_as_str=self.free_form_object_as_str, - ) - kwargs[schema_field] = new_schema + kwargs[schema_field] = self._render_native_schema(schema) last_error: Optional[str] = None for _ in range(1, retries + 1): @@ -434,13 +556,39 @@ def generate( "include_schema_in_system_prompt", ] } - raw = super()._generate(**{"prompt": current, **filtered_kwargs}) + raw = "" try: + # Inside the try: ``_parse_llm_response`` raises ValueError for a + # contentless reply, and that must be retried like any other + # invalid output rather than aborting the whole call. + raw = super()._generate(**{"prompt": current, **filtered_kwargs}) if isinstance(raw, str): return self._validate(raw, schema) return raw - except OutputValidationError as e: + except (OutputValidationError, ValueError) as e: + # ValueError covers providers whose ``_parse_llm_response`` + # rejects an empty/contentless response ("No content or tool + # calls found in response"). Without it, a single blank reply + # from the backend aborts the whole call and the configured + # ``retries`` are never used. last_error = str(e) + # An empty response carries no mistake to correct: it is a + # transient backend failure (empty content under load). Retry + # the original prompt untouched — appending correction turns + # would grow a conversation that several backends answer with + # another empty response, burning every remaining attempt. + if not (isinstance(raw, str) and raw.strip()): + # Truncated by the token limit? Re-asking with the same + # budget yields the identical truncation, so grow it. This + # is the common failure for reasoning models, whose + # "thinking" tokens can consume a small default budget + # (watsonx defaults to 1024) before any content is emitted. + if self._last_response_truncated: + kwargs["max_tokens"] = self._escalate_max_tokens( + kwargs.get("max_tokens") + ) + current = self._inject_system(prompt, instr) if instr else prompt + continue correction = ( f"The previous response did not conform: {last_error}\nPlease correct it." " And remember to output ONLY the requested schema, without any additional text." @@ -484,19 +632,19 @@ async def generate_async( if self.prompt_based_validation: include_schema_in_system_prompt = True schema_field = None + # Models that ignore a native schema kwarg must be steered by the + # prompt instead; sending ``response_format`` to them is at best a + # no-op and at worst returns empty content. + elif schema_field and not self.supports_native_structured_output(): + include_schema_in_system_prompt = True + schema_field = None current = prompt instr = None if include_schema_in_system_prompt: instr = self._make_instruction(schema) current = self._inject_system(prompt, instr) if schema_field: - kwargs[schema_field] = schema - if isinstance(schema, dict): - new_schema = json_schema_to_pydantic_model( - schema, - free_form_object_as_str=self.free_form_object_as_str, - ) - kwargs[schema_field] = new_schema + kwargs[schema_field] = self._render_native_schema(schema) last_error: Optional[str] = None for _ in range(1, retries + 1): @@ -512,15 +660,41 @@ async def generate_async( "include_schema_in_system_prompt", ] } - raw = await super()._generate_async( - **{"prompt": current, **filtered_kwargs} - ) + raw = "" try: + # Inside the try: ``_parse_llm_response`` raises ValueError for a + # contentless reply, and that must be retried like any other + # invalid output rather than aborting the whole call. + raw = await super()._generate_async( + **{"prompt": current, **filtered_kwargs} + ) if isinstance(raw, str): return self._validate(raw, schema) return raw - except OutputValidationError as e: + except (OutputValidationError, ValueError) as e: + # ValueError covers providers whose ``_parse_llm_response`` + # rejects an empty/contentless response ("No content or tool + # calls found in response"). Without it, a single blank reply + # from the backend aborts the whole call and the configured + # ``retries`` are never used. last_error = str(e) + # An empty response carries no mistake to correct: it is a + # transient backend failure (empty content under load). Retry + # the original prompt untouched — appending correction turns + # would grow a conversation that several backends answer with + # another empty response, burning every remaining attempt. + if not (isinstance(raw, str) and raw.strip()): + # Truncated by the token limit? Re-asking with the same + # budget yields the identical truncation, so grow it. This + # is the common failure for reasoning models, whose + # "thinking" tokens can consume a small default budget + # (watsonx defaults to 1024) before any content is emitted. + if self._last_response_truncated: + kwargs["max_tokens"] = self._escalate_max_tokens( + kwargs.get("max_tokens") + ) + current = self._inject_system(prompt, instr) if instr else prompt + continue correction = ( f"The previous response did not conform: {last_error}\nPlease correct it." " And remember to output ONLY the requested schema, without any additional text." diff --git a/altk/core/llm/providers/litellm/litellm.py b/altk/core/llm/providers/litellm/litellm.py index 7ec917fc..9d7cd060 100644 --- a/altk/core/llm/providers/litellm/litellm.py +++ b/altk/core/llm/providers/litellm/litellm.py @@ -158,6 +158,16 @@ def _parse_llm_response(self, raw: Any) -> Union[str, LLMResponse]: if not content: content = first.get("delta", {}).get("content", first.get("text", "")) + # Chain-of-thought models (e.g. gpt-oss on watsonx) can put the answer in + # ``reasoning_content`` and leave ``content`` empty. Prefer it over + # raising, so a usable response isn't discarded as "no content". + if not content and not tool_calls and msg: + reasoning = getattr(msg, "reasoning_content", None) or ( + msg.get("reasoning_content") if isinstance(msg, dict) else None + ) + if reasoning: + content = reasoning + if not content and not tool_calls: raise ValueError("No content or tool calls found in response") @@ -303,6 +313,28 @@ def provider_class(cls) -> Type[Any]: """ return litellm # type: ignore + def supports_native_structured_output(self) -> bool: + """Whether this model honors a native ``response_format`` schema. + + Uses litellm's per-model capability data, so newly supported models are + picked up without changes here. Models that lack it (e.g. gpt-oss on + watsonx, ollama, gemini) silently ignore ``response_format`` — some + return empty content when it is sent — so the caller falls back to + injecting the schema into the system prompt instead. + """ + try: + if litellm.supports_response_schema(model=self.model_path): + return True + # ``False`` is also what litellm returns for a model it has no + # metadata for, which would silently downgrade every unknown model. + # Only trust a negative answer when the model is actually known. + litellm.get_model_info(model=self.model_path) + return False + except Exception: + # Unknown model: assume native support and let validation + retries + # catch it, preserving the previous behavior. + return True + def _register_methods(self) -> None: """ Register how to call litellm methods - only chat modes are supported: @@ -421,6 +453,16 @@ def _parse_llm_response(self, raw: Any) -> Union[str, LLMResponse]: if not content: content = first.get("delta", {}).get("content", first.get("text", "")) + # Chain-of-thought models (e.g. gpt-oss on watsonx) can put the answer in + # ``reasoning_content`` and leave ``content`` empty. Prefer it over + # raising, so a usable response isn't discarded as "no content". + if not content and not tool_calls and msg: + reasoning = getattr(msg, "reasoning_content", None) or ( + msg.get("reasoning_content") if isinstance(msg, dict) else None + ) + if reasoning: + content = reasoning + if not content and not tool_calls: raise ValueError("No content or tool calls found in response") diff --git a/altk/core/llm/providers/openai/openai.py b/altk/core/llm/providers/openai/openai.py index 765e5c18..34bb8eed 100644 --- a/altk/core/llm/providers/openai/openai.py +++ b/altk/core/llm/providers/openai/openai.py @@ -81,6 +81,31 @@ def transform_min_tokens(value: Any, mode: Any) -> dict[str, Any]: class BaseValidatingOpenAIClient(ValidatingLLMClient): """Base class for validating OpenAI and Azure OpenAI clients with shared parameter mapping""" + def _render_native_schema(self, schema: Any) -> Any: + """Render *schema* as an OpenAI ``response_format`` payload. + + ``chat.completions.create`` rejects a Pydantic model class outright + ("You tried to pass a `BaseModel` class ... use chat.completions.parse() + instead"), so the base implementation cannot be used as-is. Emitting the + equivalent ``{"type": "json_schema", ..., "strict": True}`` dict keeps + native structured output working on the ``create`` method that both the + OpenAI and Azure clients already register. + """ + from openai.lib._pydantic import to_strict_json_schema + from pydantic import BaseModel + + model = super()._render_native_schema(schema) + if not (isinstance(model, type) and issubclass(model, BaseModel)): + return model + return { + "type": "json_schema", + "json_schema": { + "name": model.__name__, + "strict": True, + "schema": to_strict_json_schema(model), + }, + } + def _setup_parameter_mapper(self) -> None: """Set up parameter mapper for OpenAI-compatible APIs""" self._parameter_mapper = ParameterMapper() diff --git a/tests/core/test_litellm_structured_output.py b/tests/core/test_litellm_structured_output.py new file mode 100644 index 00000000..dfc9c38e --- /dev/null +++ b/tests/core/test_litellm_structured_output.py @@ -0,0 +1,76 @@ +"""Offline tests for the LiteLLM validating client's response handling. + +Covers two production failures seen with reasoning ("thinking") models such as +``gpt-oss`` on watsonx: + +- the answer arrives in ``reasoning_content`` while ``content`` is empty, and +- the model ignores a native ``response_format`` schema entirely, so the schema + has to be injected into the system prompt instead. +""" + +from __future__ import annotations + +import pytest + +from altk.core.llm.providers.litellm.litellm import LiteLLMClientOutputVal +from altk.core.llm.providers.litellm.watsonx import WatsonxLiteLLMClientOutputVal + + +def _response(content=None, reasoning=None, finish="stop"): + """Build a litellm-shaped response object.""" + from litellm.types.utils import Choices, Message, ModelResponse + + kwargs = {"content": content, "role": "assistant"} + if reasoning is not None: + kwargs["reasoning_content"] = reasoning + return ModelResponse( + choices=[Choices(message=Message(**kwargs), finish_reason=finish, index=0)] + ) + + +class TestReasoningContentFallback: + """``_parse_llm_response`` must not discard a usable reasoning-only reply.""" + + def test_reasoning_content_used_when_content_empty(self): + client = LiteLLMClientOutputVal.__new__(LiteLLMClientOutputVal) + raw = _response(content="", reasoning='{"a": 1}') + # Call the unwrapped parser directly: the instance-level wrapper is + # installed in __init__, which we skip here on purpose. + assert LiteLLMClientOutputVal._parse_llm_response(client, raw) == '{"a": 1}' + + def test_content_still_preferred_over_reasoning(self): + client = LiteLLMClientOutputVal.__new__(LiteLLMClientOutputVal) + raw = _response(content='{"real": true}', reasoning="thinking out loud") + assert ( + LiteLLMClientOutputVal._parse_llm_response(client, raw) == '{"real": true}' + ) + + def test_still_raises_when_nothing_usable(self): + client = LiteLLMClientOutputVal.__new__(LiteLLMClientOutputVal) + with pytest.raises(ValueError, match="No content or tool calls"): + LiteLLMClientOutputVal._parse_llm_response(client, _response(content="")) + + +class TestNativeStructuredOutputCapability: + """Native ``response_format`` is only used where the model honors it.""" + + @pytest.mark.parametrize( + "model_name, expected", + [ + # Reasoning model with no response-schema support: must fall back. + ("openai/gpt-oss-120b", False), + # Model litellm reports as supporting response schemas. + ("mistralai/mistral-large", True), + ], + ) + def test_watsonx_capability_is_per_model(self, model_name, expected): + client = WatsonxLiteLLMClientOutputVal.__new__(WatsonxLiteLLMClientOutputVal) + client.model_path = f"watsonx/{model_name}" + assert client.supports_native_structured_output() is expected + + def test_unknown_model_assumes_native_support(self): + """Unknown models keep the previous behavior rather than silently + switching every call to prompt-based validation.""" + client = LiteLLMClientOutputVal.__new__(LiteLLMClientOutputVal) + client.model_path = "some-provider/not-a-real-model-xyz" + assert client.supports_native_structured_output() is True diff --git a/tests/core/test_validating_llm_client.py b/tests/core/test_validating_llm_client.py index a0c7f1dd..9e0a665f 100644 --- a/tests/core/test_validating_llm_client.py +++ b/tests/core/test_validating_llm_client.py @@ -16,11 +16,13 @@ from __future__ import annotations +import json import logging from types import SimpleNamespace from typing import Any, Type import pytest +from pydantic import BaseModel from altk.core.llm.output_parser import ( OutputValidationError, @@ -92,10 +94,11 @@ def test_freeform_object_flag_switches_to_str(self): ) assert m.model_fields["a"].annotation is str - def test_freeform_flag_keeps_nested_objects_as_dict(self): - # only free-form (no properties) converts; an object with properties - # keeps its dict shape (OpenAI can still satisfy additionalProperties - # when the sub-schema is fully specified). + def test_freeform_flag_recurses_into_nested_objects(self): + # Only a free-form object (no properties) becomes a str. An object that + # *has* properties is recursed into as a nested model, so its fields + # survive into the schema handed to the provider — a bare ``dict`` would + # erase them and let the model emit output the real schema rejects. m = json_schema_to_pydantic_model( { "type": "object", @@ -110,7 +113,9 @@ def test_freeform_flag_keeps_nested_objects_as_dict(self): free_form_object_as_str=True, ) assert m.model_fields["flat"].annotation is str - assert m.model_fields["structured"].annotation is dict + nested = m.model_fields["structured"].annotation + assert issubclass(nested, BaseModel) + assert set(nested.model_fields) == {"x"} # --------------------------------------------------------------------------- @@ -304,3 +309,209 @@ def _parse_llm_response(self, raw): # will be wrapped out = c2._parse_llm_response(raw) assert out == "" assert any("reasoning" in r.message.lower() for r in caplog.records) + + +# --------------------------------------------------------------------------- +# Schema fidelity: constraints must survive the Pydantic round-trip, otherwise +# provider-native structured output is handed a weaker schema than the one the +# response is later validated against. +# --------------------------------------------------------------------------- + + +class TestSchemaFidelity: + def test_numeric_bounds_survive(self): + m = json_schema_to_pydantic_model( + { + "type": "object", + "properties": { + "score": {"type": "integer", "minimum": 1, "maximum": 5} + }, + } + ) + prop = m.model_json_schema()["properties"]["score"] + assert prop["minimum"] == 1 + assert prop["maximum"] == 5 + + def test_enum_survives_inside_array_items(self): + # A Field-level constraint cannot reach into ``items``; only a real + # Literal type does. This is what made reasoning models emit + # out-of-vocabulary values that failed the original schema. + m = json_schema_to_pydantic_model( + { + "type": "object", + "properties": { + "kinds": { + "type": "array", + "items": {"type": "string", "enum": ["A", "B"]}, + } + }, + } + ) + emitted = json.dumps(m.model_json_schema()) + assert '"A"' in emitted and '"B"' in emitted + + def test_nested_array_object_properties_survive(self): + m = json_schema_to_pydantic_model( + { + "type": "object", + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "properties": {"rationale": {"type": "string"}}, + "required": ["rationale"], + }, + } + }, + } + ) + assert "rationale" in json.dumps(m.model_json_schema()) + + def test_additional_properties_false_forbids_extras(self): + m = json_schema_to_pydantic_model( + { + "type": "object", + "properties": {"a": {"type": "string"}}, + "additionalProperties": False, + } + ) + assert m.model_config.get("extra") == "forbid" + + def test_enum_with_null_type_is_optional(self): + m = json_schema_to_pydantic_model( + { + "type": "object", + "properties": { + "t": {"type": ["string", "null"], "enum": ["x", "y", None]} + }, + } + ) + # Must accept None without raising. + assert m(t=None).t is None + + +# --------------------------------------------------------------------------- +# Retry behavior for empty / truncated responses (altk-boost#115). +# --------------------------------------------------------------------------- + + +class TestEmptyResponseRetries: + def test_value_error_is_retried_not_propagated(self, monkeypatch): + """A provider raising ValueError('No content...') must consume retries.""" + observed: list = [] + calls = {"n": 0} + from altk.core.llm.base import BaseLLMClient + + def fake_generate(self, **kwargs): + observed.append(kwargs) + calls["n"] += 1 + if calls["n"] == 1: + raise ValueError("No content or tool calls found in response") + return '{"a": "ok"}' + + monkeypatch.setattr(BaseLLMClient, "_generate", fake_generate, raising=True) + c = _FakeValidating(prompt_based_validation=True, client=object()) + out = c.generate( + [], + schema={ + "type": "object", + "properties": {"a": {"type": "string"}}, + "required": ["a"], + }, + retries=3, + ) + assert out == {"a": "ok"} + assert calls["n"] == 2 + + def test_empty_reply_retries_original_prompt_not_a_growing_thread( + self, monkeypatch + ): + """No empty assistant turn is appended, and the retry re-sends the + original prompt: several backends answer a padded conversation with + another empty response, which would burn every attempt.""" + observed: list = [] + _install_scripted_generate(monkeypatch, observed, ["", '{"a": "ok"}']) + c = _FakeValidating(prompt_based_validation=True, client=object()) + out = c.generate( + [{"role": "user", "content": "hi"}], + schema={ + "type": "object", + "properties": {"a": {"type": "string"}}, + "required": ["a"], + }, + retries=3, + ) + assert out == {"a": "ok"} + retry_msgs = observed[-1]["prompt"] + assert not any( + m.get("role") == "assistant" and not (m.get("content") or "") + for m in retry_msgs + ), "an empty assistant turn must never be sent" + assert len(retry_msgs) == len(observed[0]["prompt"]) + + def test_truncated_reply_escalates_max_tokens(self, monkeypatch): + """finish_reason='length' means the budget was too small; re-asking with + the same budget would truncate identically, so it must grow.""" + observed: list = [] + from altk.core.llm.base import BaseLLMClient + + truncated = { + "choices": [{"message": {"content": ""}, "finish_reason": "length"}] + } + scripted = [truncated, '{"a": "ok"}'] + + def fake_generate(self, **kwargs): + observed.append(kwargs) + raw = scripted.pop(0) + return self._parse_llm_response(raw) + + monkeypatch.setattr(BaseLLMClient, "_generate", fake_generate, raising=True) + + class _RealisticParse(_FakeValidating): + """A truncated reply carries no text: a real provider parser raises + on the missing content and the wrapper turns that into ``""``.""" + + def _parse_llm_response(self, raw): + if isinstance(raw, dict): + raise ValueError("No content or tool calls found in response") + return str(raw) + + c = _RealisticParse(prompt_based_validation=True, client=object()) + out = c.generate( + [{"role": "user", "content": "hi"}], + schema={ + "type": "object", + "properties": {"a": {"type": "string"}}, + "required": ["a"], + }, + retries=3, + max_tokens=1024, + ) + assert out == {"a": "ok"} + assert observed[0]["max_tokens"] == 1024 + assert observed[-1]["max_tokens"] > 1024 + + def test_native_schema_skipped_when_model_lacks_support(self, monkeypatch): + """A model that ignores response_format must be steered by the prompt + instead — sending the kwarg is a no-op there at best.""" + observed: list = [] + _install_scripted_generate(monkeypatch, observed, ['{"a": "ok"}']) + + class _NoNative(_FakeValidating): + def supports_native_structured_output(self) -> bool: + return False + + c = _NoNative(client=object()) + c.generate( + [{"role": "user", "content": "hi"}], + schema={ + "type": "object", + "properties": {"a": {"type": "string"}}, + "required": ["a"], + }, + schema_field="response_format", + ) + assert "response_format" not in observed[-1] + # schema went into a system message instead + assert observed[-1]["prompt"][0]["role"] == "system"