Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
601e780
always_on: find its agent as the solver, print its response, refuse i…
sarah-chitty-1 Nov 9, 2026
010b041
task run --a2a-agent-id: swap only the tested agent when the task dep…
sarah-chitty-1 Nov 9, 2026
0ef1b8e
deploy_agent: a run-wide agent pick swaps the tested agent, never the…
sarah-chitty-1 Nov 9, 2026
fe35c84
deploy_agent: read the run's task with its seed applied, so a seeded …
sarah-chitty-1 Nov 9, 2026
2339185
deploy_agent: only a user_sim step's agent is kept out of a run-wide …
sarah-chitty-1 Nov 9, 2026
11a1ddf
always_on plumbing: tell the agent under test and the user sim apart …
sarah-chitty-1 Nov 11, 2026
2060ebf
training: a task whose user sim writes the opening trains on that ope…
sarah-chitty-1 Nov 12, 2026
91afec3
Merge branch 'main' of github.com:scaleapi/agentenv-framework into sa…
sarah-chitty-1 Nov 12, 2026
36f8dce
cli: task run shows a prompting step its own response, not a later on…
sarah-chitty-1 Nov 12, 2026
49f21ba
Merge branch 'main' of github.com:scaleapi/agentenv-framework into sa…
sarah-chitty-1 Nov 12, 2026
af98cc1
training: a row's first message skips a prompt sent to the user sim, …
sarah-chitty-1 Nov 12, 2026
68bac9f
episode rules: classify every step type in CHECKS or UNAFFECTED; drop…
earakely-scale Nov 12, 2026
4cf1943
prompt_agent: is_prompt_step tells a prompting step by its class, not…
earakely-scale Nov 12, 2026
8368183
Drop roles.py and plays_user: the user sim is the agent a prompting s…
earakely-scale Nov 12, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 5 additions & 6 deletions src/agent_env/bundle/plan.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@
from agent_env.task_step.task_steps.deploy_agent import DeployAgentTaskStep
from agent_env.task_step.task_steps.install_agent import InstallAgentTaskStep
from agent_env.task_step.task_steps.load_artifact import LoadArtifactTaskStep
from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep
from agent_env.task_step.task_steps.prompt_agent import is_prompt_step, user_sim_agents

from ._fs import fold, relative, with_article
from .authoring import AuthoringContext, entry_files, staged_build_context
Expand Down Expand Up @@ -1219,11 +1219,10 @@ def folder_files(bundle: Bundle, entry: BundleEntry) -> Mapping[str, Path]:


def solver_steps(steps: Any) -> tuple[str, ...]:
"""The ids of the steps in ``steps``, a task's config, that deploy or install an agent a ``prompt_agent`` step
prompts, and not as its user simulator: the solver's."""
prompts = [step for step in steps if isinstance(step, dict) and step.get("type") == PromptAgentTaskStep.type]
prompted = {step.get("agent_name") or TaskStep.DEFAULT_AGENT_NAME for step in prompts}
prompted -= {step.get("user_agent_name") or PromptAgentTaskStep.DEFAULT_USER_AGENT_NAME for step in prompts}
"""The ids of the steps in ``steps``, a task's config, that deploy or install an agent a step prompts as the agent
under test (``is_prompt_step``), and not as its user simulator: the solver's."""
prompts = [step for step in steps if isinstance(step, dict) and is_prompt_step(step)]
prompted = {step.get("agent_name") or TaskStep.DEFAULT_AGENT_NAME for step in prompts} - user_sim_agents(steps)
return tuple(step["id"] for step in steps if isinstance(step, dict) and isinstance(step.get("id"), str)
and step.get("type") in (DeployAgentTaskStep.type, InstallAgentTaskStep.type)
and (step.get("agent_name") or TaskStep.DEFAULT_AGENT_NAME) in prompted)
Expand Down
8 changes: 5 additions & 3 deletions src/agent_env/bundle/resolve.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
from agent_env.store.image_store.oci_registry_credentials import names_registry
from agent_env.task_step.registry import get_task_step_registry
from agent_env.task_step.task_step import TaskStep, attach_retry_config, dependencies
from agent_env.task_step.task_steps.prompt_agent import is_prompt_step, user_sim_agents

from ._fs import fold, relative, show
from .authoring import AuthoringContext, default_dockerfile
Expand Down Expand Up @@ -125,11 +126,12 @@ def resolve_bundle(bundle: Bundle) -> ResolvedBundle:


def played_deploys(entry: ResolvedEntry) -> frozenset[str]:
"""The ``deploy_agent`` steps of a task's only prompted agent, which a caller playing that agent itself
(``EpisodeSpec``) stands in for; none when the entry isn't a task, or prompts no agent or several."""
"""The ``deploy_agent`` steps of a task's only agent under test, the one a step prompts as it and no step talks to
as its user (``user_sim_agents``), which a caller playing that agent itself (``EpisodeSpec``) stands in for; none
when the entry isn't a task, or prompts no agent or several."""
if entry.entry.kind is not BundleKind.TASK:
return frozenset()
prompted = {_agent_name(step) for step in entry.config if step.get("type") == "prompt_agent"}
prompted = {_agent_name(step) for step in entry.config if is_prompt_step(step)} - user_sim_agents(entry.config)
if len(prompted) != 1:
return frozenset()
return frozenset(step.get("id") for step in entry.config
Expand Down
11 changes: 8 additions & 3 deletions src/agent_env/cli/task/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
from agent_env.task_step.task_steps.deploy_agent import DeployAgentTaskStep
from agent_env.task_step.context_ops import build_context_update_ops
from agent_env.task_step.task_steps.collect_artifacts import CollectArtifactsTaskStep
from agent_env.task_step.task_steps.prompt_agent import is_prompt_step


def _indent(text: str, prefix: str = " ") -> str:
Expand Down Expand Up @@ -176,8 +177,11 @@ def _step_output_lines(step, context) -> list[str]:
if match:
lines.append(_format_loaded_snapshot(match))
return lines
if step.type == "prompt_agent" and context.prompt_responses:
return [_format_prompt_response(context.prompt_responses[-1])]
if is_prompt_step(step):
# This step's own response: a step running alongside it may have added a later one.
match = next((r for r in reversed(context.prompt_responses) if r.step_id == step.id), None)
if match:
return [_format_prompt_response(match)]
if step.type == "snapshot_agent_state":
snapshots = context.metadata.get("agent_snapshots") or []
match = next((s for s in reversed(snapshots) if s.get("capture_step_id") == step.id), None)
Expand Down Expand Up @@ -359,7 +363,8 @@ def _stored(task, context: TaskStepContext) -> str:
@click.option("--litellm-api-key", default=None, type=str, help="Override LITELLM_API_KEY for this run; a secret:KEY or env:NAME reference is resolved where it is used")
@click.option("--judge-litellm-api-key", default=None, type=str, help="Override LITELLM_API_KEY used by rubrics_verifier judge agents (falls back to --litellm-api-key, then to config secret); a secret:KEY or env:NAME reference is resolved where it is used")
@click.option("--apply-trajectory-filter/--no-trajectory-filter", "apply_trajectory_filter", default=None, help="Force trajectory filtering on/off for rubrics_verifier (overrides the step's trajectory_filter setting)")
@click.option("--a2a-agent-id", default=None, type=str, help="Override A2A agent id for deploy_agent steps")
@click.option("--a2a-agent-id", default=None, type=str,
help="Override A2A agent id for deploy_agent steps; a user sim's agent keeps its own")
@click.option("--agent-sandbox", default=None,
help="Override agent sandbox backend(s) for deploy_agent steps; comma-separated for fallback chain (e.g. 'modal,local')")
@click.option("--env-sandbox", default=None,
Expand Down
4 changes: 2 additions & 2 deletions src/agent_env/task/episode/spec.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
from agent_env.task.task import Task, substitute_seed_in_steps
from agent_env.task_step.context import TaskStepContext, sandbox_overrides
from agent_env.task_step.task_step import TaskStep
from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep
from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep, is_prompt_step


class EpisodeSpec:
Expand Down Expand Up @@ -74,7 +74,7 @@ async def start(self, *, sandbox: str | None = None) -> Episode:


def _trainer_agents(task: Task, steps: list[TaskStep], named: Sequence[str] | None) -> list[str]:
prompted = list(dict.fromkeys(step.agent_name for step in steps if step.type == "prompt_agent"))
prompted = list(dict.fromkeys(step.agent_name for step in steps if is_prompt_step(step)))
if named is None:
if len(prompted) != 1:
raise EpisodeError(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,6 @@ class DeployUserSimAgentTaskStep(DeployAgentTaskStep):
``agent_name`` in ``metadata["user_sims"]``."""

type: ClassVar[str] = "deploy_user_sim_agent"
plays_user: ClassVar[bool] = True
system_prompt_required: ClassVar[bool] = True
entity_refs = DeployAgentTaskStep.entity_refs
# Which agent plays the user is part of the task, not the user's setup: written out for elsewhere, the task keeps it.
Expand Down
22 changes: 22 additions & 0 deletions src/agent_env/task_step/task_steps/prompt_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -1350,3 +1350,25 @@ def _trigger_answer(resp: dict) -> Optional[dict]:
if resp.get("fired"):
return {"message": resp.get("parts") or []}
return None


def is_prompt_step(step: Any) -> bool:
"""Whether ``step`` prompts an agent: a prompt_agent step or a subclass of one, such as always_on. A step dict, or
anything else with a ``type``, is looked up by that type in the step registry."""
if isinstance(step, PromptAgentTaskStep):
return True
from agent_env.task_step.registry import get_task_step_registry # its built-ins import this module

kind = step.get("type") if isinstance(step, dict) else getattr(step, "type", None)
cls = get_task_step_registry().get(kind) if isinstance(kind, str) else None
return isinstance(cls, type) and issubclass(cls, PromptAgentTaskStep)


def user_sim_agents(steps: Any) -> set[str]:
"""The agents a task's prompting steps talk to as their user: an always_on step's ``user_sim``, a prompt_agent
step's ``user_agent_name`` (its default when unset). ``steps`` are steps or step dicts."""
def field(step: Any, name: str) -> Any:
return step.get(name) if isinstance(step, dict) else getattr(step, name, None)

return {field(step, "user_sim") or field(step, "user_agent_name") or PromptAgentTaskStep.DEFAULT_USER_AGENT_NAME
for step in steps if is_prompt_step(step)}
16 changes: 15 additions & 1 deletion src/agent_env/training/harness.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
from agent_env.task.teardown import teardown_run
from agent_env.task_step.context import TaskStepContext, sandbox_overrides
from agent_env.task_step.task_steps.deploy_agent import DeployAgentTaskStep
from agent_env.task_step.task_steps.prompt_agent import is_prompt_step, user_sim_agents
from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep
from agent_env.training._openenv import on_session_loop

Expand Down Expand Up @@ -231,7 +232,20 @@ def _index_named(prompt: Any, ids: list[str]) -> int:


def _first_prompt(task: Task) -> str:
return next((step.prompt for step in task.steps if step.type == "prompt_agent"), "")
"""The user's first message to the agent under test: its step's prompt, else the opening of the user sim the step
names (an always_on step's ``user_sim``, a prompt_agent step's ``user_agent_name``), as it was written. A step that
prompts an agent another step talks to as its user (the user sim, say a briefing) isn't the agent under test's."""
users = user_sim_agents(task.steps)
step = next((each for each in task.steps
if is_prompt_step(each) and getattr(each, "agent_name", None) not in users), None)
if step is None:
return ""
if getattr(step, "prompt", None):
return step.prompt
user = getattr(step, "user_sim", None) or getattr(step, "user_agent_name", None)
sim = next((each for each in task.steps
if user and getattr(each, "agent_name", None) == user and getattr(each, "opening_prompt", None)), None)
return getattr(sim, "opening_prompt", None) or ""


def _free_port() -> int:
Expand Down
36 changes: 36 additions & 0 deletions tst/unit/bundle/agent_override_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
from agent_env.task.task import Task
from agent_env.task_step.context import TaskStepContext
from agent_env.task_step.task_steps.install_agent import InstallAgentTaskStep
from agent_env.task_step.task_steps.prompt_agent import user_sim_agents
from tst.unit.bundle._support import layout, store_agent

ROOT = "@local/~/triage"
Expand Down Expand Up @@ -119,6 +120,41 @@ def test_only_the_steps_deploying_or_installing_the_agent_a_step_prompts_are_the
assert solver_steps(steps) == ("solver", "installed")


def test_the_agent_an_always_on_step_wakes_is_the_solver():
steps = [
{"id": "solver", "type": "deploy_agent"},
{"id": "user", "type": "deploy_user_sim_agent", "agent_name": "linda"},
{"id": "loop", "type": "always_on", "user_sim": "linda"},
]

assert solver_steps(steps) == ("solver",)


def test_the_agent_an_always_on_step_talks_to_as_its_user_is_never_the_solvers():
"""Even when a step prompts it, as a briefing would, and another step installs it."""
steps = [
{"id": "solver", "type": "deploy_agent"},
{"id": "user", "type": "deploy_user_sim_agent", "agent_name": "linda"},
{"id": "installed", "type": "install_agent", "agent_name": "linda", "sandbox_name": "box"},
{"id": "brief", "type": "prompt_agent", "agent_name": "linda"},
{"id": "loop", "type": "always_on", "user_sim": "linda"},
]

assert solver_steps(steps) == ("solver",)
assert user_sim_agents(steps) == {"linda", "human_agent"}


def test_a_prompt_agent_tasks_solver_and_user_agents_are_as_on_main():
steps = [
{"id": "solver", "type": "deploy_agent"},
{"id": "user", "type": "deploy_agent", "agent_name": "human_agent"},
{"id": "ask", "type": "prompt_agent", "max_conversation_turns": 3},
]

assert solver_steps(steps) == ("solver",)
assert user_sim_agents(steps) == {"human_agent"}


def test_preflight_checks_the_agent_the_solver_deploys(bundle_dir, monkeypatch):
checked = []
original = preflight_module._Walk._agent_deploy
Expand Down
24 changes: 23 additions & 1 deletion tst/unit/bundle/resolve_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,13 @@
import json
import random
from dataclasses import replace
from types import SimpleNamespace

import pytest

from agent_env.bundle import BundleError, BundleKind, parse_bundle
from agent_env.bundle import resolve as resolve_module
from agent_env.bundle.resolve import BuiltImage, Output, Reference, ReferencedImage, resolve_bundle
from agent_env.bundle.resolve import BuiltImage, Output, Reference, ReferencedImage, played_deploys, resolve_bundle
from agent_env.entity.refs import EntityKind, EntityRef
from agent_env.env.env import Env
from agent_env.task import Task
Expand Down Expand Up @@ -594,3 +595,24 @@ def test_a_bad_depends_on_on_a_step_deriving_its_output_is_named_once(make, depe
assert problems(make(tasks={"t": [
{"id": "cli", "type": "build_mcp_cli", "env_id": "tickets", "command_name": "c", "depends_on": depends_on},
]})) == (f"tasks/t.json: step 'cli': {problem}",)


def test_the_agent_an_always_on_step_wakes_is_the_one_a_caller_plays():
task = SimpleNamespace(entry=SimpleNamespace(kind=BundleKind.TASK), config=[
{"id": "agent", "type": "deploy_agent"},
{"id": "user", "type": "deploy_user_sim_agent", "agent_name": "linda"},
{"id": "brief", "type": "prompt_agent", "agent_name": "linda"},
{"id": "loop", "type": "always_on", "user_sim": "linda"},
])

assert played_deploys(task) == frozenset({"agent"})


def test_a_prompt_agent_tasks_played_deploy_is_as_on_main():
task = SimpleNamespace(entry=SimpleNamespace(kind=BundleKind.TASK), config=[
{"id": "agent", "type": "deploy_agent"},
{"id": "user", "type": "deploy_agent", "agent_name": "human_agent"},
{"id": "ask", "type": "prompt_agent", "max_conversation_turns": 3},
])

assert played_deploys(task) == frozenset({"agent"})
30 changes: 30 additions & 0 deletions tst/unit/cli/always_on_step_output_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
"""`task run` shows an always_on step its response as it shows a prompt_agent step's."""

from __future__ import annotations

from types import SimpleNamespace

from agent_env.cli.task.run import _step_output_lines
from agent_env.task_step.context import PromptResponse, TaskStepContext


def test_an_always_on_steps_response_is_shown():
context = TaskStepContext(prompt_responses=[PromptResponse(prompt_id="loop", response="Recap sent.", step_id="loop")])

[lines] = _step_output_lines(SimpleNamespace(type="always_on", id="loop"), context)

assert "prompt_id: loop" in lines and "Recap sent." in lines


def test_an_always_on_step_without_a_response_shows_nothing():
assert _step_output_lines(SimpleNamespace(type="always_on", id="loop"), TaskStepContext()) == []


def test_an_always_on_step_shows_its_own_response_when_a_parallel_step_added_a_later_one():
context = TaskStepContext(prompt_responses=[
PromptResponse(prompt_id="loop", response="Recap sent.", step_id="loop"),
PromptResponse(prompt_id="side", response="Side answer.", step_id="side")])

[lines] = _step_output_lines(SimpleNamespace(type="always_on", id="loop"), context)

assert "Recap sent." in lines and "Side answer." not in lines
64 changes: 64 additions & 0 deletions tst/unit/cli/task_run_agent_pick_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
"""``task run --a2a-agent-id`` is a run-wide pick: the CLI leaves it run-wide, at the deploy steps' stored versions, for
each step to apply (tst/unit/task_step/run_wide_agent_pick_test.py); a step that plays the user keeps its own agent by
its class, as it does for a hub run's pick."""

import importlib
import logging

import pytest
from click.testing import CliRunner

from agent_env.cli import cli, submitted_runs
from agent_env.runner import local_runner
from agent_env.task import Task
from agent_env.task.teardown import TeardownReport

task_run = importlib.import_module("agent_env.cli.task.run") # the package's ``run`` is the click command
pytestmark = pytest.mark.usefixtures("sigint_handled", "local_stores")

AGENT = {"id": "agent", "type": "deploy_agent", "env_ids": [], "a2a_agent_id": "theirs", "a2a_agent_version": 4}
USER = {"id": "user", "type": "deploy_agent", "env_ids": [], "agent_name": "human_agent", "a2a_agent_id": "sim"}
ASK = {"id": "ask", "type": "prompt_agent", "prompt_id": "ask", "prompt": "hi", "max_conversation_turns": 3}


class _Recorded:
"""A stored task that records each context it runs with."""

id, version, requires = "t", 1, None

def __init__(self, steps):
self.steps = Task.from_dict({"id": "t", "version": 1, "steps": steps}).steps
self.contexts = []

async def run(self, context, **_):
self.contexts.append(context)
return context


@pytest.fixture
def stored(monkeypatch):
async def teardown_run(context):
return TeardownReport()

monkeypatch.setattr(task_run, "teardown_run", teardown_run)
monkeypatch.setattr(local_runner, "teardown_run", teardown_run)
monkeypatch.setattr(submitted_runs, "POLL_SECONDS", 0.02)

def store(*steps):
recorded = _Recorded(list(steps))
monkeypatch.setattr(Task, "get", classmethod(lambda cls, id, version=None: recorded))
return recorded

logging.disable(logging.CRITICAL)
yield store
logging.disable(logging.NOTSET)


def test_the_pick_stays_run_wide_with_a_user_agent_deployed(stored):
recorded = stored(AGENT, USER, ASK)
result = CliRunner().invoke(cli, ["task", "run", "--id", "t", "--a2a-agent-id", "mine"])
assert result.exit_code == 0, result.output

[context] = recorded.contexts
overrides = context.metadata["user_overrides"]
assert overrides["a2a_agent_id"] == "mine" and "step_params" not in overrides
Loading
Loading