Skip to content

Commit 204e9db

Browse files
sarah-chitty-1claudeearakely-scale
authored
feat: bundle, training, task run and agent picks understand always_on and user sims (FD-5371) (#565)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Edgar Arakelyan <edgar.arakelyan@scale.com>
1 parent 787a6d5 commit 204e9db

16 files changed

Lines changed: 337 additions & 26 deletions

File tree

‎src/agent_env/bundle/plan.py‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@
6363
from agent_env.task_step.task_steps.deploy_agent import DeployAgentTaskStep
6464
from agent_env.task_step.task_steps.install_agent import InstallAgentTaskStep
6565
from agent_env.task_step.task_steps.load_artifact import LoadArtifactTaskStep
66-
from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep
66+
from agent_env.task_step.task_steps.prompt_agent import is_prompt_step, user_sim_agents
6767

6868
from ._fs import fold, relative, with_article
6969
from .authoring import AuthoringContext, entry_files, staged_build_context
@@ -1219,11 +1219,10 @@ def folder_files(bundle: Bundle, entry: BundleEntry) -> Mapping[str, Path]:
12191219

12201220

12211221
def solver_steps(steps: Any) -> tuple[str, ...]:
1222-
"""The ids of the steps in ``steps``, a task's config, that deploy or install an agent a ``prompt_agent`` step
1223-
prompts, and not as its user simulator: the solver's."""
1224-
prompts = [step for step in steps if isinstance(step, dict) and step.get("type") == PromptAgentTaskStep.type]
1225-
prompted = {step.get("agent_name") or TaskStep.DEFAULT_AGENT_NAME for step in prompts}
1226-
prompted -= {step.get("user_agent_name") or PromptAgentTaskStep.DEFAULT_USER_AGENT_NAME for step in prompts}
1222+
"""The ids of the steps in ``steps``, a task's config, that deploy or install an agent a step prompts as the agent
1223+
under test (``is_prompt_step``), and not as its user simulator: the solver's."""
1224+
prompts = [step for step in steps if isinstance(step, dict) and is_prompt_step(step)]
1225+
prompted = {step.get("agent_name") or TaskStep.DEFAULT_AGENT_NAME for step in prompts} - user_sim_agents(steps)
12271226
return tuple(step["id"] for step in steps if isinstance(step, dict) and isinstance(step.get("id"), str)
12281227
and step.get("type") in (DeployAgentTaskStep.type, InstallAgentTaskStep.type)
12291228
and (step.get("agent_name") or TaskStep.DEFAULT_AGENT_NAME) in prompted)

‎src/agent_env/bundle/resolve.py‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
from agent_env.store.image_store.oci_registry_credentials import names_registry
3131
from agent_env.task_step.registry import get_task_step_registry
3232
from agent_env.task_step.task_step import TaskStep, attach_retry_config, dependencies
33+
from agent_env.task_step.task_steps.prompt_agent import is_prompt_step, user_sim_agents
3334

3435
from ._fs import fold, relative, show
3536
from .authoring import AuthoringContext, default_dockerfile
@@ -125,11 +126,12 @@ def resolve_bundle(bundle: Bundle) -> ResolvedBundle:
125126

126127

127128
def played_deploys(entry: ResolvedEntry) -> frozenset[str]:
128-
"""The ``deploy_agent`` steps of a task's only prompted agent, which a caller playing that agent itself
129-
(``EpisodeSpec``) stands in for; none when the entry isn't a task, or prompts no agent or several."""
129+
"""The ``deploy_agent`` steps of a task's only agent under test, the one a step prompts as it and no step talks to
130+
as its user (``user_sim_agents``), which a caller playing that agent itself (``EpisodeSpec``) stands in for; none
131+
when the entry isn't a task, or prompts no agent or several."""
130132
if entry.entry.kind is not BundleKind.TASK:
131133
return frozenset()
132-
prompted = {_agent_name(step) for step in entry.config if step.get("type") == "prompt_agent"}
134+
prompted = {_agent_name(step) for step in entry.config if is_prompt_step(step)} - user_sim_agents(entry.config)
133135
if len(prompted) != 1:
134136
return frozenset()
135137
return frozenset(step.get("id") for step in entry.config

‎src/agent_env/cli/task/run.py‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
from agent_env.task_step.task_steps.deploy_agent import DeployAgentTaskStep
3131
from agent_env.task_step.context_ops import build_context_update_ops
3232
from agent_env.task_step.task_steps.collect_artifacts import CollectArtifactsTaskStep
33+
from agent_env.task_step.task_steps.prompt_agent import is_prompt_step
3334

3435

3536
def _indent(text: str, prefix: str = " ") -> str:
@@ -176,8 +177,11 @@ def _step_output_lines(step, context) -> list[str]:
176177
if match:
177178
lines.append(_format_loaded_snapshot(match))
178179
return lines
179-
if step.type == "prompt_agent" and context.prompt_responses:
180-
return [_format_prompt_response(context.prompt_responses[-1])]
180+
if is_prompt_step(step):
181+
# This step's own response: a step running alongside it may have added a later one.
182+
match = next((r for r in reversed(context.prompt_responses) if r.step_id == step.id), None)
183+
if match:
184+
return [_format_prompt_response(match)]
181185
if step.type == "snapshot_agent_state":
182186
snapshots = context.metadata.get("agent_snapshots") or []
183187
match = next((s for s in reversed(snapshots) if s.get("capture_step_id") == step.id), None)
@@ -359,7 +363,8 @@ def _stored(task, context: TaskStepContext) -> str:
359363
@click.option("--litellm-api-key", default=None, type=str, help="Override LITELLM_API_KEY for this run; a secret:KEY or env:NAME reference is resolved where it is used")
360364
@click.option("--judge-litellm-api-key", default=None, type=str, help="Override LITELLM_API_KEY used by rubrics_verifier judge agents (falls back to --litellm-api-key, then to config secret); a secret:KEY or env:NAME reference is resolved where it is used")
361365
@click.option("--apply-trajectory-filter/--no-trajectory-filter", "apply_trajectory_filter", default=None, help="Force trajectory filtering on/off for rubrics_verifier (overrides the step's trajectory_filter setting)")
362-
@click.option("--a2a-agent-id", default=None, type=str, help="Override A2A agent id for deploy_agent steps")
366+
@click.option("--a2a-agent-id", default=None, type=str,
367+
help="Override A2A agent id for deploy_agent steps; a user sim's agent keeps its own")
363368
@click.option("--agent-sandbox", default=None,
364369
help="Override agent sandbox backend(s) for deploy_agent steps; comma-separated for fallback chain (e.g. 'modal,local')")
365370
@click.option("--env-sandbox", default=None,

‎src/agent_env/task/episode/spec.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
from agent_env.task.task import Task, substitute_seed_in_steps
1212
from agent_env.task_step.context import TaskStepContext, sandbox_overrides
1313
from agent_env.task_step.task_step import TaskStep
14-
from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep
14+
from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep, is_prompt_step
1515

1616

1717
class EpisodeSpec:
@@ -74,7 +74,7 @@ async def start(self, *, sandbox: str | None = None) -> Episode:
7474

7575

7676
def _trainer_agents(task: Task, steps: list[TaskStep], named: Sequence[str] | None) -> list[str]:
77-
prompted = list(dict.fromkeys(step.agent_name for step in steps if step.type == "prompt_agent"))
77+
prompted = list(dict.fromkeys(step.agent_name for step in steps if is_prompt_step(step)))
7878
if named is None:
7979
if len(prompted) != 1:
8080
raise EpisodeError(

‎src/agent_env/task_step/task_steps/agent/deploy_user_sim_agent.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -168,7 +168,6 @@ class DeployUserSimAgentTaskStep(DeployAgentTaskStep):
168168
``agent_name`` in ``metadata["user_sims"]``."""
169169

170170
type: ClassVar[str] = "deploy_user_sim_agent"
171-
plays_user: ClassVar[bool] = True
172171
system_prompt_required: ClassVar[bool] = True
173172
entity_refs = DeployAgentTaskStep.entity_refs
174173
# Which agent plays the user is part of the task, not the user's setup: written out for elsewhere, the task keeps it.

‎src/agent_env/task_step/task_steps/prompt_agent.py‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1350,3 +1350,25 @@ def _trigger_answer(resp: dict) -> Optional[dict]:
13501350
if resp.get("fired"):
13511351
return {"message": resp.get("parts") or []}
13521352
return None
1353+
1354+
1355+
def is_prompt_step(step: Any) -> bool:
1356+
"""Whether ``step`` prompts an agent: a prompt_agent step or a subclass of one, such as always_on. A step dict, or
1357+
anything else with a ``type``, is looked up by that type in the step registry."""
1358+
if isinstance(step, PromptAgentTaskStep):
1359+
return True
1360+
from agent_env.task_step.registry import get_task_step_registry # its built-ins import this module
1361+
1362+
kind = step.get("type") if isinstance(step, dict) else getattr(step, "type", None)
1363+
cls = get_task_step_registry().get(kind) if isinstance(kind, str) else None
1364+
return isinstance(cls, type) and issubclass(cls, PromptAgentTaskStep)
1365+
1366+
1367+
def user_sim_agents(steps: Any) -> set[str]:
1368+
"""The agents a task's prompting steps talk to as their user: an always_on step's ``user_sim``, a prompt_agent
1369+
step's ``user_agent_name`` (its default when unset). ``steps`` are steps or step dicts."""
1370+
def field(step: Any, name: str) -> Any:
1371+
return step.get(name) if isinstance(step, dict) else getattr(step, name, None)
1372+
1373+
return {field(step, "user_sim") or field(step, "user_agent_name") or PromptAgentTaskStep.DEFAULT_USER_AGENT_NAME
1374+
for step in steps if is_prompt_step(step)}

‎src/agent_env/training/harness.py‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
from agent_env.task.teardown import teardown_run
3737
from agent_env.task_step.context import TaskStepContext, sandbox_overrides
3838
from agent_env.task_step.task_steps.deploy_agent import DeployAgentTaskStep
39+
from agent_env.task_step.task_steps.prompt_agent import is_prompt_step, user_sim_agents
3940
from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep
4041
from agent_env.training._openenv import on_session_loop
4142

@@ -231,7 +232,20 @@ def _index_named(prompt: Any, ids: list[str]) -> int:
231232

232233

233234
def _first_prompt(task: Task) -> str:
234-
return next((step.prompt for step in task.steps if step.type == "prompt_agent"), "")
235+
"""The user's first message to the agent under test: its step's prompt, else the opening of the user sim the step
236+
names (an always_on step's ``user_sim``, a prompt_agent step's ``user_agent_name``), as it was written. A step that
237+
prompts an agent another step talks to as its user (the user sim, say a briefing) isn't the agent under test's."""
238+
users = user_sim_agents(task.steps)
239+
step = next((each for each in task.steps
240+
if is_prompt_step(each) and getattr(each, "agent_name", None) not in users), None)
241+
if step is None:
242+
return ""
243+
if getattr(step, "prompt", None):
244+
return step.prompt
245+
user = getattr(step, "user_sim", None) or getattr(step, "user_agent_name", None)
246+
sim = next((each for each in task.steps
247+
if user and getattr(each, "agent_name", None) == user and getattr(each, "opening_prompt", None)), None)
248+
return getattr(sim, "opening_prompt", None) or ""
235249

236250

237251
def _free_port() -> int:

‎tst/unit/bundle/agent_override_test.py‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
from agent_env.task.task import Task
1818
from agent_env.task_step.context import TaskStepContext
1919
from agent_env.task_step.task_steps.install_agent import InstallAgentTaskStep
20+
from agent_env.task_step.task_steps.prompt_agent import user_sim_agents
2021
from tst.unit.bundle._support import layout, store_agent
2122

2223
ROOT = "@local/~/triage"
@@ -119,6 +120,41 @@ def test_only_the_steps_deploying_or_installing_the_agent_a_step_prompts_are_the
119120
assert solver_steps(steps) == ("solver", "installed")
120121

121122

123+
def test_the_agent_an_always_on_step_wakes_is_the_solver():
124+
steps = [
125+
{"id": "solver", "type": "deploy_agent"},
126+
{"id": "user", "type": "deploy_user_sim_agent", "agent_name": "linda"},
127+
{"id": "loop", "type": "always_on", "user_sim": "linda"},
128+
]
129+
130+
assert solver_steps(steps) == ("solver",)
131+
132+
133+
def test_the_agent_an_always_on_step_talks_to_as_its_user_is_never_the_solvers():
134+
"""Even when a step prompts it, as a briefing would, and another step installs it."""
135+
steps = [
136+
{"id": "solver", "type": "deploy_agent"},
137+
{"id": "user", "type": "deploy_user_sim_agent", "agent_name": "linda"},
138+
{"id": "installed", "type": "install_agent", "agent_name": "linda", "sandbox_name": "box"},
139+
{"id": "brief", "type": "prompt_agent", "agent_name": "linda"},
140+
{"id": "loop", "type": "always_on", "user_sim": "linda"},
141+
]
142+
143+
assert solver_steps(steps) == ("solver",)
144+
assert user_sim_agents(steps) == {"linda", "human_agent"}
145+
146+
147+
def test_a_prompt_agent_tasks_solver_and_user_agents_are_as_on_main():
148+
steps = [
149+
{"id": "solver", "type": "deploy_agent"},
150+
{"id": "user", "type": "deploy_agent", "agent_name": "human_agent"},
151+
{"id": "ask", "type": "prompt_agent", "max_conversation_turns": 3},
152+
]
153+
154+
assert solver_steps(steps) == ("solver",)
155+
assert user_sim_agents(steps) == {"human_agent"}
156+
157+
122158
def test_preflight_checks_the_agent_the_solver_deploys(bundle_dir, monkeypatch):
123159
checked = []
124160
original = preflight_module._Walk._agent_deploy

‎tst/unit/bundle/resolve_test.py‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,13 @@
33
import json
44
import random
55
from dataclasses import replace
6+
from types import SimpleNamespace
67

78
import pytest
89

910
from agent_env.bundle import BundleError, BundleKind, parse_bundle
1011
from agent_env.bundle import resolve as resolve_module
11-
from agent_env.bundle.resolve import BuiltImage, Output, Reference, ReferencedImage, resolve_bundle
12+
from agent_env.bundle.resolve import BuiltImage, Output, Reference, ReferencedImage, played_deploys, resolve_bundle
1213
from agent_env.entity.refs import EntityKind, EntityRef
1314
from agent_env.env.env import Env
1415
from agent_env.task import Task
@@ -594,3 +595,24 @@ def test_a_bad_depends_on_on_a_step_deriving_its_output_is_named_once(make, depe
594595
assert problems(make(tasks={"t": [
595596
{"id": "cli", "type": "build_mcp_cli", "env_id": "tickets", "command_name": "c", "depends_on": depends_on},
596597
]})) == (f"tasks/t.json: step 'cli': {problem}",)
598+
599+
600+
def test_the_agent_an_always_on_step_wakes_is_the_one_a_caller_plays():
601+
task = SimpleNamespace(entry=SimpleNamespace(kind=BundleKind.TASK), config=[
602+
{"id": "agent", "type": "deploy_agent"},
603+
{"id": "user", "type": "deploy_user_sim_agent", "agent_name": "linda"},
604+
{"id": "brief", "type": "prompt_agent", "agent_name": "linda"},
605+
{"id": "loop", "type": "always_on", "user_sim": "linda"},
606+
])
607+
608+
assert played_deploys(task) == frozenset({"agent"})
609+
610+
611+
def test_a_prompt_agent_tasks_played_deploy_is_as_on_main():
612+
task = SimpleNamespace(entry=SimpleNamespace(kind=BundleKind.TASK), config=[
613+
{"id": "agent", "type": "deploy_agent"},
614+
{"id": "user", "type": "deploy_agent", "agent_name": "human_agent"},
615+
{"id": "ask", "type": "prompt_agent", "max_conversation_turns": 3},
616+
])
617+
618+
assert played_deploys(task) == frozenset({"agent"})
Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
"""`task run` shows an always_on step its response as it shows a prompt_agent step's."""
2+
3+
from __future__ import annotations
4+
5+
from types import SimpleNamespace
6+
7+
from agent_env.cli.task.run import _step_output_lines
8+
from agent_env.task_step.context import PromptResponse, TaskStepContext
9+
10+
11+
def test_an_always_on_steps_response_is_shown():
12+
context = TaskStepContext(prompt_responses=[PromptResponse(prompt_id="loop", response="Recap sent.", step_id="loop")])
13+
14+
[lines] = _step_output_lines(SimpleNamespace(type="always_on", id="loop"), context)
15+
16+
assert "prompt_id: loop" in lines and "Recap sent." in lines
17+
18+
19+
def test_an_always_on_step_without_a_response_shows_nothing():
20+
assert _step_output_lines(SimpleNamespace(type="always_on", id="loop"), TaskStepContext()) == []
21+
22+
23+
def test_an_always_on_step_shows_its_own_response_when_a_parallel_step_added_a_later_one():
24+
context = TaskStepContext(prompt_responses=[
25+
PromptResponse(prompt_id="loop", response="Recap sent.", step_id="loop"),
26+
PromptResponse(prompt_id="side", response="Side answer.", step_id="side")])
27+
28+
[lines] = _step_output_lines(SimpleNamespace(type="always_on", id="loop"), context)
29+
30+
assert "Recap sent." in lines and "Side answer." not in lines

0 commit comments

Comments
 (0)