From 70c8e74877d17c80a489daf7998ff42a73add746 Mon Sep 17 00:00:00 2001 From: basil-k-aji-dev <70605804+basil-k-aji-dev@users.noreply.github.com> Date: Mon, 14 Sep 2026 00:31:19 +0530 Subject: [PATCH] test(flash): isolate the summarizer's model factories from provider credentials Four Flash test modules could not run without Gemini credentials, contrary to the deterministic-suite contract in CONTRIBUTING.md. `VisualStepSummarizer.__init__` resolves its utility model through `get_llm` and falls back to `get_google_llm` on any exception, so with no GOOGLE_API_KEY both paths end in a real ChatGoogleGenerativeAI construction that raises. The tests assign their fake to `summarizer._llm` after construction, which is too late -- the object never gets built. The runner and compressor modules reach the same constructor indirectly. Add a `stub_summarizer_model_factories` fixture that replaces both factories at the summarizer module boundary, and opt the four modules into it. The real summarizer, its bounded retry loop, the compressor, the ledger and the runner all stay under test, and every existing assertion is unchanged. `test_flash_config_and_builder` only asserts Flash configuration values but ran the builder's provider validation; it now uses the `validate_profiles=False` option the builder already exposes for configuration-only use. No production credential validation is relaxed. Closes #41. --- tests/unit/agents/conftest.py | 68 +++++++++++++++++++ tests/unit/agents/test_flash_runner.py | 5 ++ tests/unit/agents/test_flash_runner_ledger.py | 5 ++ tests/unit/agents/test_flash_scrub_edge.py | 5 ++ .../unit/agents/test_flash_step_summarizer.py | 7 +- 5 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 tests/unit/agents/conftest.py diff --git a/tests/unit/agents/conftest.py b/tests/unit/agents/conftest.py new file mode 100644 index 00000000..2f8f1310 --- /dev/null +++ b/tests/unit/agents/conftest.py @@ -0,0 +1,68 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shared fixtures for agent unit tests.""" + +from typing import Any + +import pytest + + +class StubChatModel: + """Stand-in for a resolved provider client. + + Mirrors the only attribute ``VisualStepSummarizer.__init__`` reads back + (``model``), so the constructor's model-name resolution is still + exercised, and refuses to be invoked: a test that needs generation must + install its own fake on ``summarizer._llm``. + """ + + def __init__(self, model_name: str) -> None: + self.model = model_name + + async def ainvoke(self, *_args: Any, **_kwargs: Any) -> Any: + raise AssertionError( + "The stubbed summarizer model was invoked. A unit test that needs " + "generation must assign its own fake to `summarizer._llm`." + ) + + def __repr__(self) -> str: # pragma: no cover - debugging aid + return f"" + + +@pytest.fixture +def stub_summarizer_model_factories(monkeypatch): + """Resolve the summarizer's model without provider credentials. + + ``VisualStepSummarizer.__init__`` asks ``get_llm`` for the utility model + and falls back to ``get_google_llm`` on *any* exception, so with no + ``GOOGLE_API_KEY`` present both paths end in a real + ``ChatGoogleGenerativeAI`` construction that raises. Replacing + ``summarizer._llm`` after the fact is too late -- the object never gets + built. Both factories are therefore stubbed at the summarizer module + boundary, which leaves the real summarizer, its retry loop, the + compressor, the ledger and the runner under test. + """ + + def _fake_get_google_llm(model_name: str | None = None, **_kwargs: Any) -> StubChatModel: + return StubChatModel(model_name or "gemini-2.5-flash-lite") + + def _fake_get_llm(_ctx: Any, *, name: str = "", is_utils: bool = False, **_kwargs: Any): + return StubChatModel("gemini-2.5-flash-lite") + + monkeypatch.setattr( + "artemis.agents.flash.summarizer.get_google_llm", + _fake_get_google_llm, + ) + monkeypatch.setattr("artemis.agents.flash.summarizer.get_llm", _fake_get_llm) diff --git a/tests/unit/agents/test_flash_runner.py b/tests/unit/agents/test_flash_runner.py index f188b5e1..cc1eae73 100644 --- a/tests/unit/agents/test_flash_runner.py +++ b/tests/unit/agents/test_flash_runner.py @@ -29,6 +29,11 @@ from artemis.context import ArtemisContext +@pytest.fixture(autouse=True) +def _isolate_summarizer_model_factories(stub_summarizer_model_factories): + """Keep this module runnable without provider credentials (see conftest).""" + + @pytest.fixture def mock_context(): ctx = Mock(spec=ArtemisContext) diff --git a/tests/unit/agents/test_flash_runner_ledger.py b/tests/unit/agents/test_flash_runner_ledger.py index 57ff1ef8..78f257f6 100644 --- a/tests/unit/agents/test_flash_runner_ledger.py +++ b/tests/unit/agents/test_flash_runner_ledger.py @@ -56,6 +56,11 @@ ) +@pytest.fixture(autouse=True) +def _isolate_summarizer_model_factories(stub_summarizer_model_factories): + """Keep this module runnable without provider credentials (see conftest).""" + + @pytest.fixture def mock_context(): ctx = Mock(spec=ArtemisContext) diff --git a/tests/unit/agents/test_flash_scrub_edge.py b/tests/unit/agents/test_flash_scrub_edge.py index 1524c3b9..9f4de4d0 100644 --- a/tests/unit/agents/test_flash_scrub_edge.py +++ b/tests/unit/agents/test_flash_scrub_edge.py @@ -30,6 +30,11 @@ from artemis.context import ArtemisContext +@pytest.fixture(autouse=True) +def _isolate_summarizer_model_factories(stub_summarizer_model_factories): + """Keep this module runnable without provider credentials (see conftest).""" + + @pytest.fixture def mock_context(): ctx = Mock(spec=ArtemisContext) diff --git a/tests/unit/agents/test_flash_step_summarizer.py b/tests/unit/agents/test_flash_step_summarizer.py index a26bc57b..d053d472 100644 --- a/tests/unit/agents/test_flash_step_summarizer.py +++ b/tests/unit/agents/test_flash_step_summarizer.py @@ -31,6 +31,11 @@ from artemis.sdk.builders import Builders +@pytest.fixture(autouse=True) +def _isolate_summarizer_model_factories(stub_summarizer_model_factories): + """Keep this module runnable without provider credentials (see conftest).""" + + @pytest.fixture def mock_context(): ctx = Mock(spec=ArtemisContext) @@ -840,7 +845,7 @@ def test_flash_config_and_builder(): step_summarizer=True, step_summarizer_model="gemini-2.5-flash-lite", prune_history_xml=True, - ).build() + ).build(validate_profiles=False) assert cfg.flash.max_turns == 25 assert cfg.flash.explorer_mode == "flash"