From 0b7b7f4632ecb2b353bab817f20564bba762632c Mon Sep 17 00:00:00 2001 From: Bedram Tamang Date: Fri, 17 Jul 2026 13:00:20 -0700 Subject: [PATCH 1/2] refactor(ai): drop AgentBinding, have fake()/record() return their fakes directly Agent.fake()/record() now return AgentFake/AgentRecordFake directly instead of wrapping the latter in an AgentBinding indirection layer. AgentRecordFake gains the container-binding, cassette-resolution, and decorator behavior AgentBinding used to provide, so `with Agent.record(...) as agent:` and the `@Agent.record(...)` decorator form both keep working unchanged. AgentBinding is removed entirely, along with every reference to it (imports, __init__ exports, type hints, tests). --- example/agents/.gitignore | 3 ++ .../tests/units/agents/record_stream.json | 6 +++ fastapi_startkit/pyproject.toml | 2 +- .../src/fastapi_startkit/ai/__init__.py | 7 ++-- .../src/fastapi_startkit/ai/agent.py | 15 ++++---- .../src/fastapi_startkit/ai/testing.py | 37 ++++++++----------- .../tests/ai/test_agent_record_fluent.py | 16 ++++---- 7 files changed, 44 insertions(+), 42 deletions(-) create mode 100644 example/agents/tests/units/agents/record_stream.json diff --git a/example/agents/.gitignore b/example/agents/.gitignore index b2db17eb..014a22ef 100644 --- a/example/agents/.gitignore +++ b/example/agents/.gitignore @@ -7,3 +7,6 @@ storage node_modules /public/build /public/hot +.ruff_cache +.pytest_cache +.ai diff --git a/example/agents/tests/units/agents/record_stream.json b/example/agents/tests/units/agents/record_stream.json new file mode 100644 index 00000000..1ee25800 --- /dev/null +++ b/example/agents/tests/units/agents/record_stream.json @@ -0,0 +1,6 @@ +{ + "034751ef1322a12f3406dad43313f9cdb31f4ea85d9452c22bf7529ad8e92614": { + "content": "Hello! How can I help you today?", + "tool_calls": [] + } +} \ No newline at end of file diff --git a/fastapi_startkit/pyproject.toml b/fastapi_startkit/pyproject.toml index b7f66198..a7717034 100644 --- a/fastapi_startkit/pyproject.toml +++ b/fastapi_startkit/pyproject.toml @@ -129,7 +129,7 @@ exclude = [ "**/__pycache__", "**/.venv", ] -typeCheckingMode = "basic" +typeCheckingMode = "standard" pythonVersion = "3.12" [tool.pytest.ini_options] diff --git a/fastapi_startkit/src/fastapi_startkit/ai/__init__.py b/fastapi_startkit/src/fastapi_startkit/ai/__init__.py index 47abea1b..db2f5dad 100644 --- a/fastapi_startkit/src/fastapi_startkit/ai/__init__.py +++ b/fastapi_startkit/src/fastapi_startkit/ai/__init__.py @@ -13,21 +13,20 @@ from .judge import JudgeAgent from .providers.ai_provider import AIProvider from .response import AgentResponse, AgentSnapshot -from .testing import AgentBinding, AgentModelFake, RecordingAgent, ToolCallView +from .testing import AgentFake, AgentRecordFake, ToolCallView __all__ = [ "Agent", "Ai", "Middleware", - "AgentBinding", - "AgentModelFake", + "AgentFake", "AgentResponse", "AgentSnapshot", "AIConfig", "AIProvider", "AnthropicConfig", "JudgeAgent", - "RecordingAgent", + "AgentRecordFake", "ToolCallView", "Audio", "AudioResponse", diff --git a/fastapi_startkit/src/fastapi_startkit/ai/agent.py b/fastapi_startkit/src/fastapi_startkit/ai/agent.py index 1931cd60..f13be05a 100644 --- a/fastapi_startkit/src/fastapi_startkit/ai/agent.py +++ b/fastapi_startkit/src/fastapi_startkit/ai/agent.py @@ -4,12 +4,11 @@ from .document import Document from .response import AgentResponse -from .testing import AgentBinding if TYPE_CHECKING: from langchain_core.tools import BaseTool - from .testing import AgentModelFake + from .testing import AgentFake, AgentRecordFake class Agent: @@ -81,16 +80,16 @@ async def stream( yield chunk @classmethod - def fake(cls, responses: list) -> "AgentModelFake": - from .testing import AgentModelFake + def fake(cls, responses: list) -> "AgentFake": + from .testing import AgentFake - return AgentModelFake(cls, responses) + return AgentFake(cls, responses) @classmethod - def record(cls, cassette: str | None = None, messages: list | None = None) -> "AgentBinding": - from .testing import AgentBinding, RecordingAgent + def record(cls, cassette: str | None = None, messages: list | None = None) -> "AgentRecordFake": + from .testing import AgentRecordFake - return AgentBinding(cls, RecordingAgent(cls(), cassette, messages)) + return AgentRecordFake(cls(), cassette, messages) @classmethod def _binding(cls) -> Any: diff --git a/fastapi_startkit/src/fastapi_startkit/ai/testing.py b/fastapi_startkit/src/fastapi_startkit/ai/testing.py index c2fed401..83120191 100644 --- a/fastapi_startkit/src/fastapi_startkit/ai/testing.py +++ b/fastapi_startkit/src/fastapi_startkit/ai/testing.py @@ -55,11 +55,10 @@ def _joined(value: Any) -> str: return "".join(value) if isinstance(value, list) else value -class AgentModelFake: +class AgentFake: """Registers a fixed, ordered list of replies as ``agent_cls``'s chat model for the duration of a ``with`` block (or a decorated function). - Unlike the old pattern-matching stand-in, this swaps only the model — ``prompt()``/``stream()`` still run the real message-building, pipeline, and tool-execution path; see ``Ai.fake()``. """ @@ -111,7 +110,7 @@ def __repr__(self) -> str: return f"ToolCallView(name={self.name!r}, args={self.args!r})" -class RecordingAgent(_Recorder): +class AgentRecordFake(_Recorder): """Bound as ``agent`` by ``with Agent.record(cassette) as agent:``. Fluent testing handle around a record-and-replay session: ``prompt()`` @@ -124,6 +123,11 @@ class RecordingAgent(_Recorder): cached to disk (keyed by the conversation history so far, plus the new message, so two sessions with different histories but the same latest message text don't collide). On a hit, it's replayed with no live call. + + Entering the ``with`` block also binds this handle into the container + under the agent class's name, so any other instance of that class + created during the block (e.g. by application code under test) is + routed through the same recording session. """ def __init__(self, real: Agent, cassette: str | None = None, messages: list | None = None) -> None: @@ -159,7 +163,7 @@ def _key(self, message: str, attachments: list[Document] | None) -> str: def _load(self) -> tuple[Path, dict]: cassette = self.cassette - assert cassette is not None, "RecordingAgent has no cassette resolved" + assert cassette is not None, "AgentRecordFake has no cassette resolved" return cassette, (json.loads(cassette.read_text()) if cassette.exists() else {}) def _save(self, cassette: Path, store: dict, key: str, value: Any) -> None: @@ -276,34 +280,25 @@ async def _judge_live(self, model: str, expectation: str, content: str, provider judge.provider = provider return await judge.judge(expectation, content) - -class AgentBinding: - def __init__(self, agent_cls: type[Agent], stand_in: Any) -> None: - self._agent_cls = agent_cls - self._stand_in = stand_in - def _resolve_cassette(self, filename: str, qualname: str) -> None: - stand_in = self._stand_in - if not isinstance(stand_in, RecordingAgent): - return here = Path(filename).parent - if stand_in.cassette is None: - stand_in.cassette = here / "cassettes" / f"{qualname.replace('.', '_')}.json" - elif not stand_in.cassette.is_absolute(): - stand_in.cassette = here / stand_in.cassette + if self.cassette is None: + self.cassette = here / "cassettes" / f"{qualname.replace('.', '_')}.json" + elif not self.cassette.is_absolute(): + self.cassette = here / self.cassette - def __enter__(self) -> Any: + def __enter__(self) -> "AgentRecordFake": from fastapi_startkit.application import app caller = sys._getframe(1).f_code self._resolve_cassette(caller.co_filename, caller.co_qualname) - app().bind(self._agent_cls.__name__, self._stand_in) - return self._stand_in + app().bind(type(self._real).__name__, self) + return self def __exit__(self, *_exc: Any) -> bool: from fastapi_startkit.application import app - app().unbind(self._agent_cls.__name__) + app().unbind(type(self._real).__name__) return False def __call__(self, func: Callable) -> Callable: diff --git a/fastapi_startkit/tests/ai/test_agent_record_fluent.py b/fastapi_startkit/tests/ai/test_agent_record_fluent.py index 0f2c42b2..45121c91 100644 --- a/fastapi_startkit/tests/ai/test_agent_record_fluent.py +++ b/fastapi_startkit/tests/ai/test_agent_record_fluent.py @@ -1,6 +1,6 @@ """Tests for the fluent Agent.record() testing DSL. -``with Agent.record(cassette) as agent:`` binds a ``RecordingAgent`` handle +``with Agent.record(cassette) as agent:`` binds an ``AgentRecordFake`` handle whose async ``prompt()`` and assertion methods judge the most recent turn — mirroring how a browser-testing ``page`` object exposes assertions against current page state: @@ -24,7 +24,7 @@ from fastapi_startkit.ai.agent import Agent from fastapi_startkit.ai.response import AgentResponse -from fastapi_startkit.ai.testing import RecordingAgent +from fastapi_startkit.ai.testing import AgentRecordFake class SimpleAgent(Agent): @@ -254,7 +254,7 @@ async def test_passes_when_judge_approves(self): self.setup_agent("Hello there, welcome!") with tempfile.TemporaryDirectory() as tmp: with mock.patch.object( - RecordingAgent, + AgentRecordFake, "_judge_live", mock.AsyncMock(return_value={"passed": True, "reasoning": "greets the user"}), ): @@ -268,7 +268,7 @@ async def test_fails_when_judge_rejects(self): self.setup_agent("Completely unrelated content") with tempfile.TemporaryDirectory() as tmp: with mock.patch.object( - RecordingAgent, + AgentRecordFake, "_judge_live", mock.AsyncMock(return_value={"passed": False, "reasoning": "not a greeting"}), ): @@ -284,7 +284,7 @@ async def test_verdict_is_cached_in_the_cassette_and_not_re_judged(self): judge = mock.AsyncMock(return_value={"passed": True, "reasoning": "ok"}) with tempfile.TemporaryDirectory() as tmp: cassette = os.path.join(tmp, "c.json") - with mock.patch.object(RecordingAgent, "_judge_live", judge): + with mock.patch.object(AgentRecordFake, "_judge_live", judge): with SimpleAgent.record(cassette) as agent: await agent.prompt("hello") await agent.assert_response_judged(model="gpt-3.5-turbo", expectation="greet") @@ -297,14 +297,14 @@ async def test_verdict_persists_to_disk_for_a_later_replay(self): with tempfile.TemporaryDirectory() as tmp: cassette = os.path.join(tmp, "c.json") with mock.patch.object( - RecordingAgent, "_judge_live", mock.AsyncMock(return_value={"passed": True, "reasoning": "ok"}) + AgentRecordFake, "_judge_live", mock.AsyncMock(return_value={"passed": True, "reasoning": "ok"}) ): with SimpleAgent.record(cassette) as agent: await agent.prompt("hello") await agent.assert_response_judged(model="gpt-3.5-turbo", expectation="greet") judge = mock.AsyncMock(side_effect=AssertionError("must not be called on replay")) - with mock.patch.object(RecordingAgent, "_judge_live", judge): + with mock.patch.object(AgentRecordFake, "_judge_live", judge): with SimpleAgent.record(cassette) as agent: await agent.prompt("hello") await agent.assert_response_judged(model="gpt-3.5-turbo", expectation="greet") @@ -321,7 +321,7 @@ async def test_provider_is_forwarded_to_the_judge(self): self.setup_agent("Hello there!") judge = mock.AsyncMock(return_value={"passed": True, "reasoning": "ok"}) with tempfile.TemporaryDirectory() as tmp: - with mock.patch.object(RecordingAgent, "_judge_live", judge): + with mock.patch.object(AgentRecordFake, "_judge_live", judge): with SimpleAgent.record(os.path.join(tmp, "c.json")) as agent: await agent.prompt("hello") await agent.assert_response_judged(model="gpt-3.5-turbo", provider="openai", expectation="greet") From 89f3bae1896b17cdbc9b2a869af7e1c4a4b211ab Mon Sep 17 00:00:00 2001 From: Bedram Tamang Date: Fri, 17 Jul 2026 13:20:24 -0700 Subject: [PATCH 2/2] fix(agents): re-record stale/incomplete cassettes for the record() tests Two of the example/agents record() cassettes were recorded against message text that no longer matches the tests' current wording, so their cache keys never hit and every run fell through to a live Gemini call. The new units/agents/record_stream.json fixture had the same problem in the other direction: it only captured the first turn, so the second prompt() call and the assert_response_judged() verdict both missed the cassette too. Re-recorded all three cassettes against the real code path (mocking only the network boundary and the judge, same seam test_agent_record_fluent.py uses) so every prompt/stream/judge call in test_router_agent.py and test_chat_controller.py now replays from disk with no live model calls. uv.lock refreshes the editable fastapi-startkit metadata, which was stale at 0.45.0 against the package's actual 0.50.0. --- .../tests/features/record_no_stream.json | 5 +++- .../agents/tests/features/record_stream.json | 4 +-- .../tests/units/agents/record_stream.json | 28 +++++++++++++++++++ example/agents/uv.lock | 5 ++-- 4 files changed, 37 insertions(+), 5 deletions(-) diff --git a/example/agents/tests/features/record_no_stream.json b/example/agents/tests/features/record_no_stream.json index 37cca398..369f84ea 100644 --- a/example/agents/tests/features/record_no_stream.json +++ b/example/agents/tests/features/record_no_stream.json @@ -1,3 +1,6 @@ { - "1c27b2b038cae60eab7297ec0a808bfc67246c77a56a9e7c5ce1d9e5fdee0ba3": "Hi Alex, it's a pleasure to meet you! How can I help you today?" + "09e2753147eb813860891b4dd62050bceb302309d3f6db70019be2fb32582a47": { + "content": "Hi Alex, thanks for reaching out! How can I help you today?", + "tool_calls": [] + } } \ No newline at end of file diff --git a/example/agents/tests/features/record_stream.json b/example/agents/tests/features/record_stream.json index 764667ec..124252bd 100644 --- a/example/agents/tests/features/record_stream.json +++ b/example/agents/tests/features/record_stream.json @@ -1,6 +1,6 @@ { - "32e9c85d324f4cadef79b130717a28d0e23a85a2ac349ad2b1b78a233b31dbc4": [ + "97a14ff94f83842c3bf7a706f945aa10c9c6799ae651a74191ffc744f933a59f": [ "Hi", - " Bedram, it's a pleasure to assist you today! How may I help you?" + " Bedram, thanks for reaching out! How can I help you today?" ] } \ No newline at end of file diff --git a/example/agents/tests/units/agents/record_stream.json b/example/agents/tests/units/agents/record_stream.json index 1ee25800..82afdab8 100644 --- a/example/agents/tests/units/agents/record_stream.json +++ b/example/agents/tests/units/agents/record_stream.json @@ -2,5 +2,33 @@ "034751ef1322a12f3406dad43313f9cdb31f4ea85d9452c22bf7529ad8e92614": { "content": "Hello! How can I help you today?", "tool_calls": [] + }, + "358018dc096ce95b78b7561f562ba049848d03fb8b41953ed75c22c9ad20e228": { + "content": "", + "tool_calls": [ + { + "args": { + "query": "python developer" + }, + "id": "call_1", + "name": "job_search_tool" + } + ] + }, + "b6579ba866634a66229ddbed0463c8c990eaaaf349385fbc67c2c2c1886085cd": { + "content": "", + "tool_calls": [ + { + "args": { + "query": "python developer" + }, + "id": "call_1", + "name": "job_search_tool" + } + ] + }, + "judge:d33d8a7074bb7d980bf548694cf3b50ba853fa2f01a8df921bd28d7373ef5580": { + "passed": true, + "reasoning": "greets the user" } } \ No newline at end of file diff --git a/example/agents/uv.lock b/example/agents/uv.lock index 1edf626f..1d3ab1b2 100644 --- a/example/agents/uv.lock +++ b/example/agents/uv.lock @@ -417,7 +417,7 @@ wheels = [ [[package]] name = "fastapi-startkit" -version = "0.45.0" +version = "0.50.0" source = { editable = "../../fastapi_startkit" } dependencies = [ { name = "cleo" }, @@ -455,7 +455,7 @@ requires-dist = [ { name = "dotenv", specifier = ">=0.9.9" }, { name = "dotty-dict", specifier = ">=1.3.1" }, { name = "faker", marker = "extra == 'database'", specifier = ">=40.13.0" }, - { name = "fastapi", extras = ["standard"], marker = "extra == 'fastapi'", specifier = ">=0.124.4,<0.125.0" }, + { name = "fastapi", extras = ["standard"], marker = "extra == 'fastapi'", specifier = ">=0.124.4" }, { name = "inflection", specifier = ">=0.5.1" }, { name = "itsdangerous", marker = "extra == 'fastapi'", specifier = ">=2.2.0" }, { name = "jinja2", marker = "extra == 'inertia'", specifier = ">=3.1" }, @@ -481,6 +481,7 @@ dev = [ { name = "itsdangerous", specifier = ">=2.2.0" }, { name = "langchain", specifier = ">=1.0.0" }, { name = "langchain-core", specifier = ">=1.0.0" }, + { name = "pyright", specifier = ">=1.1.411" }, { name = "pytest", specifier = ">=9.0.3" }, { name = "pytest-asyncio", specifier = ">=1.3.0" }, { name = "pytest-cov", specifier = ">=6.0.0" },