From 67ec1ce5898f729db89c0bb07904a876b6ec3d60 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 14 Aug 2026 20:22:56 +0000 Subject: [PATCH 1/4] fix(experimentalist): make the architecture-doc step budget configurable The Coder wrote architecture.md under a hardcoded 50-step CodeAct budget, so an agent with more than about 20 source files exhausted it and failed the run before any optimization ran. Add coder.architecture_doc_max_iterations, default 100. A @strategy budget is fixed when the class is defined, so create_architecture_doc now builds the CodeAct config from the run configuration and passes it to the generated step. Co-authored-by: Aditya Pandey Signed-off-by: Cursor Agent --- .../experimentalist/components/coder.py | 35 ++++++++++++++++--- .../skills/nemo-experimentalist/SKILL.md | 4 +++ .../tests/experimentalist/test_tools.py | 25 +++++++++++++ 3 files changed, 60 insertions(+), 4 deletions(-) diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/coder.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/coder.py index cdcd3b3560..a292ed9469 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/coder.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/coder.py @@ -40,6 +40,11 @@ class CoderConfig(BaseModel): default=2, description="Max LLM repair iterations inside integration_check before giving up on a candidate.", ) + architecture_doc_max_iterations: int = Field( + default=100, + gt=0, + description="Max CodeAct steps allowed to write architecture.md. Raise it for agents with many source files.", + ) model_catalog_path: Path | None = Field( default=None, description="Optional YAML model catalog path overriding the packaged assets/models.yaml.", @@ -998,12 +1003,34 @@ async def run_smoke_eval( options=smoke_options, ) - @strategy( - CodeActStrategy(config=CodeActConfig(max_iterations=50, cell_timeout=3600.0)), - llm=lambda self: self._architecture_model, - ) async def create_architecture_doc( self, agent_id: str, source_path: str | None = None, entrypoint: str | None = None + ) -> None: + """Document the agent, allowing ``architecture_doc_max_iterations`` steps. + + Args: + agent_id: The agent directory under eval-and-optimize/agents/ to document. + source_path: Directory holding the agent source, relative to the agent directory. + entrypoint: File the evaluation harness invokes, relative to the agent directory. + + """ + # How many steps documenting an agent takes scales with the source the Coder is + # handed, so the CodeAct config is built here rather than on the @strategy + # decorator below, where it would be fixed when the class is defined. + config = CodeActConfig( + max_iterations=self._config.architecture_doc_max_iterations, + cell_timeout=3600.0, + ) + await self._write_architecture_doc( + agent_id, + source_path=source_path, + entrypoint=entrypoint, + _strategy=CodeActStrategy(config=config), # ty: ignore[unknown-argument] + ) + + @strategy(CodeActStrategy(), llm=lambda self: self._architecture_model) + async def _write_architecture_doc( + self, agent_id: str, source_path: str | None = None, entrypoint: str | None = None ) -> None: """Update (or, only if absent, create) architecture.md for the given agent. diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/skills/nemo-experimentalist/SKILL.md b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/skills/nemo-experimentalist/SKILL.md index ac1c9112dc..c0cc628180 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/skills/nemo-experimentalist/SKILL.md +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/skills/nemo-experimentalist/SKILL.md @@ -261,6 +261,10 @@ to select the winner. | `evaluator.n_attempts` | `1` | `1`; increase only when task results are noisy | Repeats each evaluation trial. | | `eval_author.max_traces` | `3` | `10` | Representative Insight traces deeply analyzed in Insight-driven mode. | +Increase `coder.architecture_doc_max_iterations` if a run stops because the +Coder cannot complete `architecture.md`. An agent that has many source files +needs more steps than the default of `100`. + A small explicit smoke configuration looks like this: ```yaml diff --git a/plugins/nemo-experimentalist/tests/experimentalist/test_tools.py b/plugins/nemo-experimentalist/tests/experimentalist/test_tools.py index 99984b95b4..20925ab347 100644 --- a/plugins/nemo-experimentalist/tests/experimentalist/test_tools.py +++ b/plugins/nemo-experimentalist/tests/experimentalist/test_tools.py @@ -8,6 +8,7 @@ from nemo_experimentalist_plugin.experimentalist.components.holdout_utils import BLOCKED_MESSAGE from nemo_experimentalist_plugin.experimentalist.components.tools import GuardedShellTools from nemo_platform_plugin.nooa_model_client import ConfiguredModelClients, ConfiguredModelRefs, activate_model_clients +from nooa import CodeActStrategy from nooa.agentdoc import pformat from nooa.tools import ShellResult from nooa.unifiedllm import CompletionClient, FakeLLMClient @@ -65,6 +66,30 @@ def test_coder_uses_default_model_for_architecture_docs(tmp_path: Path) -> None: assert coder._architecture_model is default +async def test_architecture_doc_step_budget_comes_from_config(tmp_path: Path, monkeypatch) -> None: + """The architecture-doc step must run under the configured budget, default included.""" + budgets: list[int | None] = [] + + async def capture( + self: Coder, + agent_id: str, + source_path: str | None = None, + entrypoint: str | None = None, + _strategy: CodeActStrategy | None = None, + ) -> None: + assert _strategy is not None + budgets.append(_strategy.config.max_iterations) + + monkeypatch.setattr(Coder, "_write_architecture_doc", capture) + + await Coder(workspace=tmp_path).create_architecture_doc("agent-1") + await Coder(workspace=tmp_path, config=CoderConfig(architecture_doc_max_iterations=250)).create_architecture_doc( + "agent-1" + ) + + assert budgets == [100, 250] + + async def test_coder_lists_agent_mutation_models_from_catalog(tmp_path: Path) -> None: catalog = tmp_path / "models.yaml" catalog.write_text( From 5899da89afe60c71c89a78a4e8394e5d7d8da53e Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 14 Aug 2026 20:45:58 +0000 Subject: [PATCH 2/4] fix(experimentalist): bound the architecture-doc strategy fallback A bare CodeActStrategy() on the @strategy decorator left the generated step unbounded when it runs without the per-call override: CodeActConfig defaults max_iterations and cell_timeout to None, and nooa reads None as unlimited. That is a worse trap than the hardcoded 50 this branch removed. Derive both the decorator and the per-call config from one helper, so the fallback matches the configured default and the timeout has a single definition. Assert the budget through a real generation run instead of a stubbed method, so the test also covers nooa honoring the override rather than only the kwarg the Coder passes. Co-authored-by: Aditya Pandey Signed-off-by: Cursor Agent --- .../experimentalist/components/coder.py | 28 ++++++----- .../skills/nemo-experimentalist/SKILL.md | 9 ++-- .../tests/experimentalist/test_tools.py | 47 ++++++++++--------- 3 files changed, 46 insertions(+), 38 deletions(-) diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/coder.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/coder.py index a292ed9469..2c03363cd4 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/coder.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/coder.py @@ -572,6 +572,11 @@ async def run(self, dataset: Dataset) -> None: """ +def _architecture_doc_codeact(max_iterations: int) -> CodeActConfig: + """Bound one architecture-doc generation, which reads source through slow shell work.""" + return CodeActConfig(max_iterations=max_iterations, cell_timeout=3600.0) + + class Coder(Agent): """Create and modify agent source code as part of the optimization loop.""" @@ -1006,7 +1011,7 @@ async def run_smoke_eval( async def create_architecture_doc( self, agent_id: str, source_path: str | None = None, entrypoint: str | None = None ) -> None: - """Document the agent, allowing ``architecture_doc_max_iterations`` steps. + """Update (or, only if absent, create) architecture.md for the given agent. Args: agent_id: The agent directory under eval-and-optimize/agents/ to document. @@ -1014,22 +1019,21 @@ async def create_architecture_doc( entrypoint: File the evaluation harness invokes, relative to the agent directory. """ - # How many steps documenting an agent takes scales with the source the Coder is - # handed, so the CodeAct config is built here rather than on the @strategy - # decorator below, where it would be fixed when the class is defined. - config = CodeActConfig( - max_iterations=self._config.architecture_doc_max_iterations, - cell_timeout=3600.0, - ) - await self._write_architecture_doc( + # A @strategy config is fixed when the class is defined, but how many steps + # documenting an agent takes scales with the source this Coder was handed. + codeact = _architecture_doc_codeact(self._config.architecture_doc_max_iterations) + await self._create_architecture_doc( agent_id, source_path=source_path, entrypoint=entrypoint, - _strategy=CodeActStrategy(config=config), # ty: ignore[unknown-argument] + _strategy=CodeActStrategy(config=codeact), # ty: ignore[unknown-argument] ) - @strategy(CodeActStrategy(), llm=lambda self: self._architecture_model) - async def _write_architecture_doc( + @strategy( + CodeActStrategy(config=_architecture_doc_codeact(CoderConfig().architecture_doc_max_iterations)), + llm=lambda self: self._architecture_model, + ) + async def _create_architecture_doc( self, agent_id: str, source_path: str | None = None, entrypoint: str | None = None ) -> None: """Update (or, only if absent, create) architecture.md for the given agent. diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/skills/nemo-experimentalist/SKILL.md b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/skills/nemo-experimentalist/SKILL.md index c0cc628180..a241d36086 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/skills/nemo-experimentalist/SKILL.md +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/skills/nemo-experimentalist/SKILL.md @@ -261,10 +261,6 @@ to select the winner. | `evaluator.n_attempts` | `1` | `1`; increase only when task results are noisy | Repeats each evaluation trial. | | `eval_author.max_traces` | `3` | `10` | Representative Insight traces deeply analyzed in Insight-driven mode. | -Increase `coder.architecture_doc_max_iterations` if a run stops because the -Coder cannot complete `architecture.md`. An agent that has many source files -needs more steps than the default of `100`. - A small explicit smoke configuration looks like this: ```yaml @@ -287,6 +283,11 @@ eval_author: max_traces: 3 ``` +`coder.architecture_doc_max_iterations` is not a cost setting. It is the number +of steps the Coder can use to write `architecture.md`, and it defaults to `100`. +Increase it if a run stops because the Coder cannot complete that file. An agent +that has many source files needs more steps. + ### Create a low-cost smoke dataset When the full dataset is expensive, create small **copied** train and diff --git a/plugins/nemo-experimentalist/tests/experimentalist/test_tools.py b/plugins/nemo-experimentalist/tests/experimentalist/test_tools.py index 20925ab347..ac730ae82d 100644 --- a/plugins/nemo-experimentalist/tests/experimentalist/test_tools.py +++ b/plugins/nemo-experimentalist/tests/experimentalist/test_tools.py @@ -1,17 +1,30 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import json from pathlib import Path from typing import cast +import pytest from nemo_experimentalist_plugin.experimentalist.components.coder import Coder, CoderConfig from nemo_experimentalist_plugin.experimentalist.components.holdout_utils import BLOCKED_MESSAGE from nemo_experimentalist_plugin.experimentalist.components.tools import GuardedShellTools from nemo_platform_plugin.nooa_model_client import ConfiguredModelClients, ConfiguredModelRefs, activate_model_clients -from nooa import CodeActStrategy from nooa.agentdoc import pformat +from nooa.errors import GenerationError from nooa.tools import ShellResult -from nooa.unifiedllm import CompletionClient, FakeLLMClient +from nooa.unifiedllm import CompletionClient, FakeLLMClient, LLMResponse, ToolCall + + +def _exec_response(code: str) -> LLMResponse: + """A scripted LLM turn that drives CodeAct's ``execute_python`` tool with ``code``.""" + return LLMResponse( + raw_response=None, + content="", + finish_reason="tool_calls", + assistant_message={"role": "assistant", "content": ""}, + tool_calls=[ToolCall(id="call_exec", name="execute_python", arguments=json.dumps({"code": code}))], + ) async def test_guarded_shell_tools_runs_allowed_commands(tmp_path): @@ -66,28 +79,18 @@ def test_coder_uses_default_model_for_architecture_docs(tmp_path: Path) -> None: assert coder._architecture_model is default -async def test_architecture_doc_step_budget_comes_from_config(tmp_path: Path, monkeypatch) -> None: - """The architecture-doc step must run under the configured budget, default included.""" - budgets: list[int | None] = [] +async def test_architecture_doc_stops_at_the_configured_step_budget(tmp_path: Path) -> None: + """The configured budget, not the one fixed on the @strategy decorator, must bound the run.""" + assert CoderConfig().architecture_doc_max_iterations == 100, "the documented default" - async def capture( - self: Coder, - agent_id: str, - source_path: str | None = None, - entrypoint: str | None = None, - _strategy: CodeActStrategy | None = None, - ) -> None: - assert _strategy is not None - budgets.append(_strategy.config.max_iterations) - - monkeypatch.setattr(Coder, "_write_architecture_doc", capture) - - await Coder(workspace=tmp_path).create_architecture_doc("agent-1") - await Coder(workspace=tmp_path, config=CoderConfig(architecture_doc_max_iterations=250)).create_architecture_doc( - "agent-1" - ) + # Each scripted turn runs a no-op cell instead of returning a result, so the only way + # out is exhausting the budget. A turn past the budget would report a larger count. + never_finishes = FakeLLMClient(scripted_responses=[_exec_response("x = 1") for _ in range(4)]) + coder = Coder(workspace=tmp_path, config=CoderConfig(architecture_doc_max_iterations=3)) + coder._architecture_model = never_finishes - assert budgets == [100, 250] + with pytest.raises(GenerationError, match=r"after 3 iterations \(max_iterations=3\)"): + await coder.create_architecture_doc("agent-1") async def test_coder_lists_agent_mutation_models_from_catalog(tmp_path: Path) -> None: From 7556e70b1b52f182e773fee1a3357d24cc0b4d27 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 14 Aug 2026 21:18:15 +0000 Subject: [PATCH 3/4] refactor(experimentalist): name the setting max_architecture_doc_iterations Every other numeric field in CoderConfig and its sibling component configs leads with max_ (max_summary_tokens, max_fix_attempts, max_trials, max_depth, max_rounds). architecture_doc_max_iterations was the only one that led with its subject instead. Also settle on one word. The field said iterations, its description said steps, and the documentation said steps. Use iterations everywhere, because that is what nooa's config key and the error a user hits are called, and quote that error in the skill so the two are searchable together. Co-authored-by: Aditya Pandey Signed-off-by: Cursor Agent --- .../experimentalist/components/coder.py | 10 +++++----- .../skills/nemo-experimentalist/SKILL.md | 8 ++++---- .../tests/experimentalist/test_tools.py | 10 +++++----- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/coder.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/coder.py index 2c03363cd4..2c6884dfa8 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/coder.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/coder.py @@ -40,10 +40,10 @@ class CoderConfig(BaseModel): default=2, description="Max LLM repair iterations inside integration_check before giving up on a candidate.", ) - architecture_doc_max_iterations: int = Field( + max_architecture_doc_iterations: int = Field( default=100, gt=0, - description="Max CodeAct steps allowed to write architecture.md. Raise it for agents with many source files.", + description="Max CodeAct iterations allowed to write architecture.md. Raise it for agents with many source files.", ) model_catalog_path: Path | None = Field( default=None, @@ -1019,9 +1019,9 @@ async def create_architecture_doc( entrypoint: File the evaluation harness invokes, relative to the agent directory. """ - # A @strategy config is fixed when the class is defined, but how many steps + # A @strategy config is fixed when the class is defined, but how many iterations # documenting an agent takes scales with the source this Coder was handed. - codeact = _architecture_doc_codeact(self._config.architecture_doc_max_iterations) + codeact = _architecture_doc_codeact(self._config.max_architecture_doc_iterations) await self._create_architecture_doc( agent_id, source_path=source_path, @@ -1030,7 +1030,7 @@ async def create_architecture_doc( ) @strategy( - CodeActStrategy(config=_architecture_doc_codeact(CoderConfig().architecture_doc_max_iterations)), + CodeActStrategy(config=_architecture_doc_codeact(CoderConfig().max_architecture_doc_iterations)), llm=lambda self: self._architecture_model, ) async def _create_architecture_doc( diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/skills/nemo-experimentalist/SKILL.md b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/skills/nemo-experimentalist/SKILL.md index a241d36086..7b4b8b2bce 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/skills/nemo-experimentalist/SKILL.md +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/skills/nemo-experimentalist/SKILL.md @@ -283,10 +283,10 @@ eval_author: max_traces: 3 ``` -`coder.architecture_doc_max_iterations` is not a cost setting. It is the number -of steps the Coder can use to write `architecture.md`, and it defaults to `100`. -Increase it if a run stops because the Coder cannot complete that file. An agent -that has many source files needs more steps. +`coder.max_architecture_doc_iterations` is not a cost setting. It is how many +iterations the Coder can use to write `architecture.md`, and it defaults to +`100`. Increase it when a run stops with `Generation failed after 100 iterations +(max_iterations=100)`. An agent that has many source files needs more iterations. ### Create a low-cost smoke dataset diff --git a/plugins/nemo-experimentalist/tests/experimentalist/test_tools.py b/plugins/nemo-experimentalist/tests/experimentalist/test_tools.py index ac730ae82d..79a0b74e0d 100644 --- a/plugins/nemo-experimentalist/tests/experimentalist/test_tools.py +++ b/plugins/nemo-experimentalist/tests/experimentalist/test_tools.py @@ -79,14 +79,14 @@ def test_coder_uses_default_model_for_architecture_docs(tmp_path: Path) -> None: assert coder._architecture_model is default -async def test_architecture_doc_stops_at_the_configured_step_budget(tmp_path: Path) -> None: - """The configured budget, not the one fixed on the @strategy decorator, must bound the run.""" - assert CoderConfig().architecture_doc_max_iterations == 100, "the documented default" +async def test_architecture_doc_stops_at_the_configured_iteration_limit(tmp_path: Path) -> None: + """The configured limit, not the one fixed on the @strategy decorator, must bound the run.""" + assert CoderConfig().max_architecture_doc_iterations == 100, "the documented default" # Each scripted turn runs a no-op cell instead of returning a result, so the only way - # out is exhausting the budget. A turn past the budget would report a larger count. + # out is exhausting the limit. A turn past the limit would report a larger count. never_finishes = FakeLLMClient(scripted_responses=[_exec_response("x = 1") for _ in range(4)]) - coder = Coder(workspace=tmp_path, config=CoderConfig(architecture_doc_max_iterations=3)) + coder = Coder(workspace=tmp_path, config=CoderConfig(max_architecture_doc_iterations=3)) coder._architecture_model = never_finishes with pytest.raises(GenerationError, match=r"after 3 iterations \(max_iterations=3\)"): From c6c1f18db36d6f175621aaf943d1741dd1b57991 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 14 Aug 2026 21:46:39 +0000 Subject: [PATCH 4/4] docs(experimentalist): correct the cost claim for the architecture-doc limit Saying the setting "is not a cost setting" was wrong: every extra iteration is another model call. Say what was meant instead, which is that it sits outside the evaluation budget the table above tunes, and name the cost it does carry. Co-authored-by: Aditya Pandey Signed-off-by: Cursor Agent --- .../skills/nemo-experimentalist/SKILL.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/skills/nemo-experimentalist/SKILL.md b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/skills/nemo-experimentalist/SKILL.md index 7b4b8b2bce..2fe357cf14 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/skills/nemo-experimentalist/SKILL.md +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/skills/nemo-experimentalist/SKILL.md @@ -283,10 +283,12 @@ eval_author: max_traces: 3 ``` -`coder.max_architecture_doc_iterations` is not a cost setting. It is how many -iterations the Coder can use to write `architecture.md`, and it defaults to -`100`. Increase it when a run stops with `Generation failed after 100 iterations -(max_iterations=100)`. An agent that has many source files needs more iterations. +`coder.max_architecture_doc_iterations` is how many iterations the Coder can use +to write `architecture.md`, and it defaults to `100`. It is separate from the +evaluation budget above, but each additional iteration adds model usage and run +time to that step. Increase it when a run stops with `Generation failed after +100 iterations (max_iterations=100)`. An agent that has many source files needs +more iterations. ### Create a low-cost smoke dataset