diff --git a/example/agents/tests/units/agents/test_router_agent.py b/example/agents/tests/units/agents/test_router_agent.py index 26ddfae7..17a71d18 100644 --- a/example/agents/tests/units/agents/test_router_agent.py +++ b/example/agents/tests/units/agents/test_router_agent.py @@ -1,16 +1,18 @@ from langchain_core.messages import AIMessage, HumanMessage from app.agents.chat import RouterAgent +from tests.test_case import TestCase -class TestRouterAgent: +class TestRouterAgent(TestCase): async def test_the_router_agent(self): with RouterAgent.record("record_stream.json") as agent: await agent.prompt("hello") agent.assert_text_response() agent.assert_tool_not_called(["job_search_tool"]) - agent.assert_response_judged( - model="gpt-3.5-turbo", + await agent.assert_response_judged( + model="gemini-3.5-flash-lite", + provider="google", expectation="The llm should respond with greetings", ) agent.assert_response_time_lt(5) diff --git a/fastapi_startkit/src/fastapi_startkit/ai/__init__.py b/fastapi_startkit/src/fastapi_startkit/ai/__init__.py index 1e8f8b8c..47abea1b 100644 --- a/fastapi_startkit/src/fastapi_startkit/ai/__init__.py +++ b/fastapi_startkit/src/fastapi_startkit/ai/__init__.py @@ -10,6 +10,7 @@ from .image import Image, ImageResponse from .image_factory import ImageFactory from .ai import Ai +from .judge import JudgeAgent from .providers.ai_provider import AIProvider from .response import AgentResponse, AgentSnapshot from .testing import AgentBinding, AgentModelFake, RecordingAgent, ToolCallView @@ -25,6 +26,7 @@ "AIConfig", "AIProvider", "AnthropicConfig", + "JudgeAgent", "RecordingAgent", "ToolCallView", "Audio", diff --git a/fastapi_startkit/src/fastapi_startkit/ai/agent.py b/fastapi_startkit/src/fastapi_startkit/ai/agent.py index 96926185..1931cd60 100644 --- a/fastapi_startkit/src/fastapi_startkit/ai/agent.py +++ b/fastapi_startkit/src/fastapi_startkit/ai/agent.py @@ -188,27 +188,37 @@ def _build_messages( return messages - def _build_model(self, model: str | None = None, provider_options: dict | None = None) -> Any: + def _build_model( + self, model: str | None = None, provider_options: dict | None = None, structured: bool = True + ) -> Any: from .ai import Ai # noqa: PLC0415 - return Ai().get_model_for(self, model, provider_options) + return Ai().get_model_for(self, model, provider_options, structured) def _to_agent_response(self, result: Any) -> AgentResponse: + parsed = None + structured = isinstance(result, dict) and "parsed" in result and "raw" in result + if structured: + parsed = result.get("parsed") + result = result.get("raw") + messages = result.get("messages", []) if isinstance(result, dict) else [] final = messages[-1] if messages else result content = getattr(final, "content", "") if not isinstance(content, str): content = str(content) + if structured and not content and hasattr(parsed, "model_dump_json"): + content = parsed.model_dump_json() - tool_calls = list(getattr(final, "tool_calls", None) or []) + tool_calls = [] if structured else list(getattr(final, "tool_calls", None) or []) usage: dict[str, Any] = {} meta = getattr(final, "usage_metadata", None) if meta: usage = {"input": meta.get("input_tokens", 0), "output": meta.get("output_tokens", 0)} - return AgentResponse(content=content, tool_calls=tool_calls, usage=usage, raw=result) + return AgentResponse(content=content, tool_calls=tool_calls, usage=usage, raw=result, parsed=parsed) def _apply_schema(self, response: AgentResponse) -> AgentResponse: schema = self.schema() @@ -253,7 +263,7 @@ async def _stream( from .runner import StreamRunner # noqa: PLC0415 messages = self._build_messages(message) - chat_model = self._build_model(model, provider_options) + chat_model = self._build_model(model, provider_options, structured=False) chain = list(self.middleware()) def core(m: Any) -> Response: diff --git a/fastapi_startkit/src/fastapi_startkit/ai/ai.py b/fastapi_startkit/src/fastapi_startkit/ai/ai.py index da1dc599..c7433e0a 100644 --- a/fastapi_startkit/src/fastapi_startkit/ai/ai.py +++ b/fastapi_startkit/src/fastapi_startkit/ai/ai.py @@ -21,11 +21,6 @@ def _key(agent: "Agent | str") -> str: @classmethod def fake(cls, agent: "Agent | str", messages: list) -> Any: - """Register a deterministic stand-in chat model for ``agent``. - - Replays ``messages`` in order via a GenericFakeChatModel — no live - LLM call. Plain strings are coerced into ``AIMessage(content=...)``. - """ from langchain_core.language_models.fake_chat_models import GenericFakeChatModel from langchain_core.messages import AIMessage @@ -51,14 +46,24 @@ def reset_fakes(cls) -> None: cls.fake_agent_models.clear() cls.fake_agent_responses.clear() - def get_model_for(self, agent: "Agent", model: str | None = None, provider_options: dict | None = None) -> Any: - """Resolve the model to run: a registered fake if one exists for - ``agent``, otherwise a freshly-built provider model.""" + def get_model_for( + self, + agent: "Agent", + model: str | None = None, + provider_options: dict | None = None, + structured: bool = True, + ) -> Any: if self.has_fake_model_for(agent): return self.get_fake_model_for(agent) - return self.build(agent, model, provider_options) - - def build(self, agent: "Agent", model: str | None = None, provider_options: dict | None = None) -> Any: + return self.build(agent, model, provider_options, structured) + + def build( + self, + agent: "Agent", + model: str | None = None, + provider_options: dict | None = None, + structured: bool = True, + ) -> Any: from langchain.chat_models import init_chat_model # noqa: PLC0415 lab = Lab.get_provider(agent.provider) @@ -79,7 +84,13 @@ def build(self, agent: "Agent", model: str | None = None, provider_options: dict chat_model = init_chat_model(self._resolve_model(agent, model), **kwargs) tools = list(agent.tools()) - return chat_model.bind_tools(tools) if tools else chat_model + chat_model = chat_model.bind_tools(tools) if tools else chat_model + + schema = agent.schema() + if structured and schema is not None: + chat_model = chat_model.with_structured_output(schema, include_raw=True) + + return chat_model def _resolve_model(self, agent: "Agent", override: str | None = None) -> str: return Lab.get_provider(agent.provider).get_model(override or agent.model or None) diff --git a/fastapi_startkit/src/fastapi_startkit/ai/judge.py b/fastapi_startkit/src/fastapi_startkit/ai/judge.py new file mode 100644 index 00000000..6923bd3d --- /dev/null +++ b/fastapi_startkit/src/fastapi_startkit/ai/judge.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +from pydantic import BaseModel + +from .agent import Agent + + +class Verdict(BaseModel): + passed: bool + reasoning: str = "" + + +class JudgeAgent(Agent): + """Grades a response against a natural-language expectation. + + A plain ``Agent`` whose ``schema()`` is a ``Verdict`` model, so the JSON + reply is parsed into a typed result through the standard structured-output + path — no hand-rolled verdict parsing. Set ``.model``/``.provider`` like + any other agent; it's fakeable via ``fake()`` and replayable via + ``record()`` for free. + """ + + def instructions(self) -> str: + return ( + "You are grading whether an AI agent's response satisfies an expectation. " + 'Reply with strict JSON only, no prose: {"passed": true|false, "reasoning": ""}' + ) + + def schema(self): + return Verdict + + async def judge(self, expectation: str, content: str) -> dict: + response = await self.prompt(f"Expectation: {expectation}\n\nResponse to grade:\n{content}") + return response.parsed.model_dump() diff --git a/fastapi_startkit/src/fastapi_startkit/ai/runner.py b/fastapi_startkit/src/fastapi_startkit/ai/runner.py index 49fe5a44..ba6ded73 100644 --- a/fastapi_startkit/src/fastapi_startkit/ai/runner.py +++ b/fastapi_startkit/src/fastapi_startkit/ai/runner.py @@ -27,6 +27,9 @@ async def run(self, messages: Sequence[Message]) -> BaseMessage: history: list[Message] = list(messages) response: AIMessage = await self.model.ainvoke(history) # type: ignore[assignment] + if isinstance(response, dict) and "parsed" in response: + return response # type: ignore[return-value] + if not response.tool_calls: return response diff --git a/fastapi_startkit/src/fastapi_startkit/ai/testing.py b/fastapi_startkit/src/fastapi_startkit/ai/testing.py index 639fda71..c2fed401 100644 --- a/fastapi_startkit/src/fastapi_startkit/ai/testing.py +++ b/fastapi_startkit/src/fastapi_startkit/ai/testing.py @@ -5,7 +5,6 @@ import hashlib import inspect import json -import re import sys import time from collections.abc import AsyncIterator @@ -244,49 +243,38 @@ def assert_response_time_lt(self, seconds: float) -> None: assert self.last_elapsed is not None, "No prompt() call has been made yet." assert self.last_elapsed < seconds, f"Expected response time < {seconds}s, took {self.last_elapsed:.3f}s" - def assert_response_judged(self, *, model: str, expectation: str) -> None: + async def assert_response_judged(self, *, model: str, expectation: str, provider: str | None = None) -> None: response = self._require_response() - verdict = self._judge(model, expectation, response.content) + verdict = await self._judge(model, expectation, response.content, provider) assert verdict.get("passed"), ( f"Judge ({model}) rejected the response for expectation {expectation!r}: " f"{verdict.get('reasoning', '')!r} — response was {response.content!r}" ) - def _judge(self, model: str, expectation: str, content: str) -> dict: + async def _judge(self, model: str, expectation: str, content: str, provider: str | None = None) -> dict: cassette, store = self._load() - key = self._judge_key(model, expectation, content) + key = self._judge_key(model, expectation, content, provider) if key in store: return store[key] - verdict = self._judge_live(model, expectation, content) + verdict = await self._judge_live(model, expectation, content, provider) self._save(cassette, store, key, verdict) return verdict @staticmethod - def _judge_key(model: str, expectation: str, content: str) -> str: + def _judge_key(model: str, expectation: str, content: str, provider: str | None = None) -> str: payload = json.dumps( - {"judge_model": model, "expectation": expectation, "content": content}, + {"judge_model": model, "judge_provider": provider, "expectation": expectation, "content": content}, sort_keys=True, ) return "judge:" + hashlib.sha256(payload.encode()).hexdigest() - def _judge_live(self, model: str, expectation: str, content: str) -> dict: - from langchain.chat_models import init_chat_model # noqa: PLC0415 + async def _judge_live(self, model: str, expectation: str, content: str, provider: str | None = None) -> dict: + from .judge import JudgeAgent # noqa: PLC0415 - prompt = ( - "You are grading whether an AI agent's response satisfies an expectation.\n" - f"Expectation: {expectation}\n" - f"Response: {content}\n\n" - 'Reply with strict JSON only, no prose: {"passed": true|false, "reasoning": ""}' - ) - chat_model = init_chat_model(model) - result = chat_model.invoke(prompt) - return self._parse_verdict(result.content) - - @staticmethod - def _parse_verdict(raw: str) -> dict: - match = re.search(r"\{.*\}", raw, re.DOTALL) - data = json.loads(match.group(0) if match else raw) - return {"passed": bool(data.get("passed")), "reasoning": data.get("reasoning", "")} + judge = JudgeAgent() + judge.model = model + judge.provider = provider + return await judge.judge(expectation, content) class AgentBinding: diff --git a/fastapi_startkit/tests/ai/test_agent_record_fluent.py b/fastapi_startkit/tests/ai/test_agent_record_fluent.py index 30d12989..0f2c42b2 100644 --- a/fastapi_startkit/tests/ai/test_agent_record_fluent.py +++ b/fastapi_startkit/tests/ai/test_agent_record_fluent.py @@ -20,7 +20,6 @@ import unittest from unittest import mock -import langchain.chat_models as chat_models from langchain_core.messages import AIMessage, HumanMessage from fastapi_startkit.ai.agent import Agent @@ -255,11 +254,13 @@ async def test_passes_when_judge_approves(self): self.setup_agent("Hello there, welcome!") with tempfile.TemporaryDirectory() as tmp: with mock.patch.object( - RecordingAgent, "_judge_live", return_value={"passed": True, "reasoning": "greets the user"} + RecordingAgent, + "_judge_live", + mock.AsyncMock(return_value={"passed": True, "reasoning": "greets the user"}), ): with SimpleAgent.record(os.path.join(tmp, "c.json")) as agent: await agent.prompt("hello") - agent.assert_response_judged( + await agent.assert_response_judged( model="gpt-3.5-turbo", expectation="The llm should respond with greetings" ) @@ -267,25 +268,27 @@ async def test_fails_when_judge_rejects(self): self.setup_agent("Completely unrelated content") with tempfile.TemporaryDirectory() as tmp: with mock.patch.object( - RecordingAgent, "_judge_live", return_value={"passed": False, "reasoning": "not a greeting"} + RecordingAgent, + "_judge_live", + mock.AsyncMock(return_value={"passed": False, "reasoning": "not a greeting"}), ): with SimpleAgent.record(os.path.join(tmp, "c.json")) as agent: await agent.prompt("hello") with self.assertRaises(AssertionError): - agent.assert_response_judged( + await agent.assert_response_judged( model="gpt-3.5-turbo", expectation="The llm should respond with greetings" ) async def test_verdict_is_cached_in_the_cassette_and_not_re_judged(self): self.setup_agent("Hello there!") - judge = mock.Mock(return_value={"passed": True, "reasoning": "ok"}) + judge = mock.AsyncMock(return_value={"passed": True, "reasoning": "ok"}) with tempfile.TemporaryDirectory() as tmp: cassette = os.path.join(tmp, "c.json") with mock.patch.object(RecordingAgent, "_judge_live", judge): with SimpleAgent.record(cassette) as agent: await agent.prompt("hello") - agent.assert_response_judged(model="gpt-3.5-turbo", expectation="greet") - agent.assert_response_judged(model="gpt-3.5-turbo", expectation="greet") + await agent.assert_response_judged(model="gpt-3.5-turbo", expectation="greet") + await agent.assert_response_judged(model="gpt-3.5-turbo", expectation="greet") judge.assert_called_once() @@ -293,16 +296,18 @@ async def test_verdict_persists_to_disk_for_a_later_replay(self): self.setup_agent("Hello there!") with tempfile.TemporaryDirectory() as tmp: cassette = os.path.join(tmp, "c.json") - with mock.patch.object(RecordingAgent, "_judge_live", return_value={"passed": True, "reasoning": "ok"}): + with mock.patch.object( + RecordingAgent, "_judge_live", mock.AsyncMock(return_value={"passed": True, "reasoning": "ok"}) + ): with SimpleAgent.record(cassette) as agent: await agent.prompt("hello") - agent.assert_response_judged(model="gpt-3.5-turbo", expectation="greet") + await agent.assert_response_judged(model="gpt-3.5-turbo", expectation="greet") - judge = mock.Mock(side_effect=AssertionError("must not be called on replay")) + judge = mock.AsyncMock(side_effect=AssertionError("must not be called on replay")) with mock.patch.object(RecordingAgent, "_judge_live", judge): with SimpleAgent.record(cassette) as agent: await agent.prompt("hello") - agent.assert_response_judged(model="gpt-3.5-turbo", expectation="greet") + await agent.assert_response_judged(model="gpt-3.5-turbo", expectation="greet") judge.assert_not_called() @@ -310,31 +315,18 @@ async def test_fails_when_no_prompt_has_been_made(self): with tempfile.TemporaryDirectory() as tmp: with SimpleAgent.record(os.path.join(tmp, "c.json")) as agent: with self.assertRaises(AssertionError): - agent.assert_response_judged(model="gpt-3.5-turbo", expectation="greet") - - -class TestJudgeLiveModelCall(unittest.TestCase): - def test_calls_init_chat_model_and_parses_json_verdict(self): - captured = {} - - class FakeResult: - content = '{"passed": true, "reasoning": "Greets the user politely."}' - - class FakeModel: - def invoke(self, prompt): - captured["prompt"] = prompt - return FakeResult() - - patcher = mock.patch.object(chat_models, "init_chat_model", lambda *a, **k: FakeModel()) - patcher.start() - self.addCleanup(patcher.stop) + await agent.assert_response_judged(model="gpt-3.5-turbo", expectation="greet") - agent = RecordingAgent(SimpleAgent()) - verdict = agent._judge_live("gpt-3.5-turbo", "The llm should respond with greetings", "Hello there!") + async def test_provider_is_forwarded_to_the_judge(self): + self.setup_agent("Hello there!") + judge = mock.AsyncMock(return_value={"passed": True, "reasoning": "ok"}) + with tempfile.TemporaryDirectory() as tmp: + with mock.patch.object(RecordingAgent, "_judge_live", judge): + with SimpleAgent.record(os.path.join(tmp, "c.json")) as agent: + await agent.prompt("hello") + await agent.assert_response_judged(model="gpt-3.5-turbo", provider="openai", expectation="greet") - self.assertTrue(verdict["passed"]) - self.assertIn("Greets", verdict["reasoning"]) - self.assertIn("Hello there!", captured["prompt"]) + judge.assert_called_once_with("gpt-3.5-turbo", "greet", "Hello there!", "openai") class TestExistingRecordApiIsUnaffected(unittest.IsolatedAsyncioTestCase): diff --git a/fastapi_startkit/tests/ai/test_judge_agent.py b/fastapi_startkit/tests/ai/test_judge_agent.py new file mode 100644 index 00000000..e1d5a964 --- /dev/null +++ b/fastapi_startkit/tests/ai/test_judge_agent.py @@ -0,0 +1,79 @@ +"""Tests for JudgeAgent — grades a response against an expectation. + +It's a plain Agent whose ``schema()`` is a ``Verdict`` model, so the model's +JSON reply is turned into a typed result through the standard structured-output +path (``response.parsed``) — no hand-rolled verdict parsing. +""" + +import os +import tempfile +import unittest +from unittest import mock + +from langchain_core.messages import AIMessage + +from fastapi_startkit.ai.judge import JudgeAgent, Verdict +from fastapi_startkit.ai.response import AgentResponse + + +class TestJudgeAgent(unittest.IsolatedAsyncioTestCase): + async def test_judge_returns_a_passing_verdict_dict(self): + with JudgeAgent.fake(['{"passed": true, "reasoning": "Greets the user politely."}']): + result = await JudgeAgent().judge("The llm should respond with greetings", "Hello there!") + + self.assertEqual(result, {"passed": True, "reasoning": "Greets the user politely."}) + + async def test_judge_returns_a_failing_verdict(self): + with JudgeAgent.fake(['{"passed": false, "reasoning": "Not a greeting."}']): + result = await JudgeAgent().judge("greet", "Completely unrelated") + + self.assertFalse(result["passed"]) + + def test_schema_is_the_verdict_model(self): + self.assertIs(JudgeAgent().schema(), Verdict) + + async def test_judge_feeds_expectation_and_response_to_the_model(self): + seen = {} + + class Capturing: + def bind_tools(self, tools, **kwargs): + return self + + async def ainvoke(self, messages): + seen["messages"] = messages + return AIMessage(content='{"passed": true, "reasoning": "ok"}') + + with mock.patch.object(JudgeAgent, "_build_model", lambda self, *a, **k: Capturing()): + await JudgeAgent().judge("The llm should respond with greetings", "Hello there!") + + blob = " ".join(str(getattr(m, "content", m)) for m in seen["messages"]) + self.assertIn("The llm should respond with greetings", blob) + self.assertIn("Hello there!", blob) + + def test_model_and_provider_are_plain_agent_attributes(self): + """No custom constructor — set like any other Agent's model/provider.""" + judge = JudgeAgent() + judge.model = "gpt-4o-mini" + judge.provider = "openai" + + self.assertEqual(judge.model, "gpt-4o-mini") + self.assertEqual(judge.provider, "openai") + + def test_model_and_provider_default_to_agent_defaults(self): + judge = JudgeAgent() + + self.assertIsNone(judge.model) + self.assertIsNone(judge.provider) + + async def test_judge_is_usable_via_the_record_fluent_dsl(self): + async def fake_run(agent_self, message, **kwargs): + return AgentResponse(content='{"passed": true, "reasoning": "ok"}') + + with tempfile.TemporaryDirectory() as tmp: + cassette = os.path.join(tmp, "judge.json") + with mock.patch.object(JudgeAgent, "_run", fake_run): + with JudgeAgent.record(cassette) as agent: + response = await agent.prompt("grade this") + + self.assertIn('"passed"', response.content) + self.assertTrue(os.path.exists(cassette)) diff --git a/fastapi_startkit/tests/ai/test_structured_output.py b/fastapi_startkit/tests/ai/test_structured_output.py new file mode 100644 index 00000000..bfa60a46 --- /dev/null +++ b/fastapi_startkit/tests/ai/test_structured_output.py @@ -0,0 +1,199 @@ +"""Structured output. + +When an Agent declares a schema(), the built model is wrapped with +model.with_structured_output(schema, include_raw=True) so the provider returns +the parsed object. Tools are bound as usual and left untouched. The wrapped +model yields {"raw", "parsed", "parsing_error"}; the Runner passes that through +and _to_agent_response unwraps it into response.parsed. + +The fake/record paths bypass build(), so they keep parsing the JSON-string +content via schema() for deterministic replay. +""" + +import unittest +from unittest import mock + +import langchain.chat_models as chat_models +from langchain_core.messages import AIMessage +from langchain_core.tools import tool +from pydantic import BaseModel + +from fastapi_startkit.ai import AIConfig +from fastapi_startkit.ai.agent import Agent +from fastapi_startkit.ai.ai import Ai +from fastapi_startkit.ai.runner import Runner +from fastapi_startkit.application import app + + +class Movie(BaseModel): + title: str + year: int + + +class MovieAgent(Agent): + def schema(self): + return Movie + + +@tool +def noop(query: str) -> str: + """A no-op tool that echoes its query.""" + return query + + +class ToolMovieAgent(Agent): + def schema(self): + return Movie + + def tools(self): + return [noop] + + +def _real_tool_call(**args) -> dict: + return {"name": "noop", "args": args, "id": "1", "type": "tool_call"} + + +class _FakeModel: + """Records the build calls made against it.""" + + def __init__(self): + self.calls = [] + + def bind_tools(self, tools, **kwargs): + self.calls.append(("bind_tools", list(tools))) + return self + + def with_structured_output(self, schema, **kwargs): + self.calls.append(("with_structured_output", schema, kwargs)) + return "STRUCTURED" + + +class TestBuild(unittest.TestCase): + def setUp(self): + container = app() + container.bind("ai", AIConfig()) + container.make("config").set("ai", AIConfig()) + + def tearDown(self): + Ai.reset_fakes() + + def _patch(self, fake): + patcher = mock.patch.object(chat_models, "init_chat_model", lambda *a, **k: fake) + patcher.start() + self.addCleanup(patcher.stop) + + def test_schema_wraps_model_with_structured_output(self): + fake = _FakeModel() + self._patch(fake) + + result = Ai().build(MovieAgent()) + + self.assertEqual(result, "STRUCTURED") + self.assertEqual(fake.calls, [("with_structured_output", Movie, {"include_raw": True})]) + + def test_tools_are_bound_then_structured_output_is_applied(self): + fake = _FakeModel() + self._patch(fake) + + result = Ai().build(ToolMovieAgent()) + + self.assertEqual(result, "STRUCTURED") + self.assertEqual( + fake.calls, + [("bind_tools", [noop]), ("with_structured_output", Movie, {"include_raw": True})], + ) + + def test_tools_only_binds_tools_without_structured_output(self): + fake = _FakeModel() + self._patch(fake) + + class ToolAgent(Agent): + def tools(self): + return [noop] + + result = Ai().build(ToolAgent()) + + self.assertIs(result, fake) + self.assertEqual(fake.calls, [("bind_tools", [noop])]) + + def test_streaming_skips_structured_output(self): + fake = _FakeModel() + self._patch(fake) + + result = Ai().build(ToolMovieAgent(), structured=False) + + self.assertIs(result, fake) + self.assertEqual(fake.calls, [("bind_tools", [noop])]) + + def test_no_schema_no_tools_returns_the_plain_model(self): + fake = _FakeModel() + self._patch(fake) + + self.assertIs(Ai().build(Agent()), fake) + self.assertEqual(fake.calls, []) + + +class TestRunner(unittest.IsolatedAsyncioTestCase): + async def test_passes_structured_output_dict_through(self): + parsed = Movie(title="Inception", year=2010) + payload = {"raw": AIMessage(content=""), "parsed": parsed, "parsing_error": None} + + class Model: + async def ainvoke(self, messages): + return payload + + result = await Runner(MovieAgent(), Model()).run(["hi"]) + + self.assertEqual(result, payload) + + async def test_runs_the_tool_when_model_calls_a_real_tool(self): + class Model: + async def ainvoke(self, messages): + return AIMessage(content="", tool_calls=[_real_tool_call(query="hello")]) + + result = await Runner(ToolMovieAgent(), Model()).run(["hi"]) + + self.assertEqual(result.content, "hello") + + +class TestResponseMapping(unittest.TestCase): + def test_unwraps_include_raw_into_parsed_and_content(self): + parsed = Movie(title="Inception", year=2010) + + response = MovieAgent()._to_agent_response( + {"raw": AIMessage(content=""), "parsed": parsed, "parsing_error": None} + ) + + self.assertIs(response.parsed, parsed) + self.assertEqual(response.content, parsed.model_dump_json()) + self.assertEqual(response.tool_calls, []) + + +class TestPromptEndToEnd(unittest.IsolatedAsyncioTestCase): + def setUp(self): + container = app() + container.bind("ai", AIConfig()) + container.make("config").set("ai", AIConfig()) + + def tearDown(self): + Ai.reset_fakes() + + async def test_prompt_populates_parsed_via_structured_output(self): + parsed = Movie(title="Inception", year=2010) + + class Structured: + async def ainvoke(self, messages): + return {"raw": AIMessage(content=""), "parsed": parsed, "parsing_error": None} + + class FakeModel: + def with_structured_output(self, schema, **kwargs): + return Structured() + + patcher = mock.patch.object(chat_models, "init_chat_model", lambda *a, **k: FakeModel()) + patcher.start() + self.addCleanup(patcher.stop) + + response = await MovieAgent().prompt("best nolan movie") + + self.assertEqual(response.parsed, parsed) + self.assertEqual(response.content, parsed.model_dump_json())