diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/coder.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/coder.py index ab3a3620e0..d4463ccd6e 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/coder.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/coder.py @@ -51,6 +51,11 @@ class CodeEditBuilderConfig(BaseModel): default=2, description="Max LLM repair iterations inside integration_check before giving up on a candidate.", ) + max_architecture_doc_iterations: int = Field( + default=100, + gt=0, + description="Max CodeAct iterations allowed to write architecture.md. Raise it for agents with many source files.", + ) model_catalog_path: Path | None = Field( default=None, description="Optional YAML model catalog path overriding the packaged assets/models.yaml.", @@ -578,6 +583,11 @@ async def run(self, dataset: Dataset) -> None: """ +def _architecture_doc_codeact(max_iterations: int) -> CodeActConfig: + """Bound one architecture-doc generation, which reads source through slow shell work.""" + return CodeActConfig(max_iterations=max_iterations, cell_timeout=3600.0) + + # Keeps its own Evaluator rather than using ctx.evaluate: its smoke checks run against an # artifact that is deliberately not yet a Candidate, and ctx.evaluate exists to associate a # result with one. An internal check has nothing to associate and must not be recorded @@ -1041,11 +1051,32 @@ async def run_smoke_eval( options=smoke_options, ) + async def create_architecture_doc( + self, workdir: Path, source_path: str | None = None, entrypoint: str | None = None + ) -> None: + """Update (or, only if absent, create) architecture.md for the given agent. + + Args: + workdir: The agent directory to document. + source_path: Directory holding the agent source, relative to the agent directory. + entrypoint: File the evaluation harness invokes, relative to the agent directory. + + """ + # A @strategy config is fixed when the class is defined, but how many iterations + # documenting an agent takes scales with the source this builder was handed. + codeact = _architecture_doc_codeact(self._config.max_architecture_doc_iterations) + await self._create_architecture_doc( + workdir, + source_path=source_path, + entrypoint=entrypoint, + _strategy=CodeActStrategy(config=codeact), # ty: ignore[unknown-argument] + ) + @strategy( - CodeActStrategy(config=CodeActConfig(max_iterations=50, cell_timeout=3600.0)), + CodeActStrategy(config=_architecture_doc_codeact(CodeEditBuilderConfig().max_architecture_doc_iterations)), llm=lambda self: self._architecture_model, ) - async def create_architecture_doc( + async def _create_architecture_doc( self, workdir: Path, source_path: str | None = None, entrypoint: str | None = None ) -> None: """Update (or, only if absent, create) architecture.md for the given agent. diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/skills/nemo-experimentalist/SKILL.md b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/skills/nemo-experimentalist/SKILL.md index 0cf2c1c9b1..f684021fba 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/skills/nemo-experimentalist/SKILL.md +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/skills/nemo-experimentalist/SKILL.md @@ -283,6 +283,13 @@ eval_author: max_traces: 3 ``` +`builder_config.max_architecture_doc_iterations` is how many iterations the +builder can use to write `architecture.md`, and it defaults to `100`. It is +separate from the evaluation budget above, but each additional iteration adds +model usage and run time to that step. Increase it when a run stops with +`Generation failed after 100 iterations (max_iterations=100)`. An agent that has +many source files needs more iterations. + ### Create a low-cost smoke dataset When the full dataset is expensive, create small **copied** train and diff --git a/plugins/nemo-experimentalist/tests/experimentalist/test_tools.py b/plugins/nemo-experimentalist/tests/experimentalist/test_tools.py index 946c59e5f1..6a229b89a1 100644 --- a/plugins/nemo-experimentalist/tests/experimentalist/test_tools.py +++ b/plugins/nemo-experimentalist/tests/experimentalist/test_tools.py @@ -1,16 +1,30 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import json from pathlib import Path from typing import cast +import pytest from nemo_experimentalist_plugin.experimentalist.components.coder import CodeEditBuilder, CodeEditBuilderConfig from nemo_experimentalist_plugin.experimentalist.components.holdout_utils import BLOCKED_MESSAGE from nemo_experimentalist_plugin.experimentalist.components.tools import GuardedShellTools from nemo_platform_plugin.nooa_model_client import ConfiguredModelClients, ConfiguredModelRefs, activate_model_clients from nooa.agentdoc import pformat +from nooa.errors import GenerationError from nooa.tools import ShellResult -from nooa.unifiedllm import CompletionClient, FakeLLMClient +from nooa.unifiedllm import CompletionClient, FakeLLMClient, LLMResponse, ToolCall + + +def _exec_response(code: str) -> LLMResponse: + """A scripted LLM turn that drives CodeAct's ``execute_python`` tool with ``code``.""" + return LLMResponse( + raw_response=None, + content="", + finish_reason="tool_calls", + assistant_message={"role": "assistant", "content": ""}, + tool_calls=[ToolCall(id="call_exec", name="execute_python", arguments=json.dumps({"code": code}))], + ) async def test_guarded_shell_tools_runs_allowed_commands(tmp_path): @@ -65,6 +79,20 @@ def test_coder_uses_default_model_for_architecture_docs(tmp_path: Path) -> None: assert coder._architecture_model is default +async def test_architecture_doc_stops_at_the_configured_iteration_limit(tmp_path: Path) -> None: + """The configured limit, not the one fixed on the @strategy decorator, must bound the run.""" + assert CodeEditBuilderConfig().max_architecture_doc_iterations == 100, "the documented default" + + # Each scripted turn runs a no-op cell instead of returning a result, so the only way + # out is exhausting the limit. A turn past the limit would report a larger count. + never_finishes = FakeLLMClient(scripted_responses=[_exec_response("x = 1") for _ in range(4)]) + builder = CodeEditBuilder(workspace=tmp_path, config=CodeEditBuilderConfig(max_architecture_doc_iterations=3)) + builder._architecture_model = never_finishes + + with pytest.raises(GenerationError, match=r"after 3 iterations \(max_iterations=3\)"): + await builder.create_architecture_doc(tmp_path) + + async def test_coder_lists_agent_mutation_models_from_catalog(tmp_path: Path) -> None: catalog = tmp_path / "models.yaml" catalog.write_text(