diff --git a/src/agent_env/bundle/resolve.py b/src/agent_env/bundle/resolve.py index c5dcf930c..4a5fa6ea8 100644 --- a/src/agent_env/bundle/resolve.py +++ b/src/agent_env/bundle/resolve.py @@ -34,7 +34,7 @@ from agent_env.store.image_store.oci_registry_credentials import names_registry from agent_env.task_step.context import ENV_KEYED_METADATA, context_path, context_path_segments from agent_env.task_step.registry import get_task_step_registry -from agent_env.task_step.task_step import TaskStep, attach_retry_config, dependencies +from agent_env.task_step.task_step import TaskStep, attach_base_fields, dependencies from agent_env.task_step.task_steps.prompt_agent import is_prompt_step, user_sim_agents from ._fs import fold, relative, show @@ -562,7 +562,7 @@ def build_step(step: dict) -> TaskStep: """The step a task's write puts, built from a copy of ``step``, so the resolved config the ledger hashes never changes.""" step = copy.deepcopy(step) - return attach_retry_config(get_task_step_registry()[step["type"]].from_dict(step), step) + return attach_base_fields(get_task_step_registry()[step["type"]].from_dict(step), step) def _nfc(name: str) -> str: diff --git a/src/agent_env/cli/task/create.py b/src/agent_env/cli/task/create.py index 8bc96849b..4475219da 100644 --- a/src/agent_env/cli/task/create.py +++ b/src/agent_env/cli/task/create.py @@ -74,14 +74,13 @@ def _build_steps(steps: list[dict]) -> list[tuple[dict, object]]: raise SystemExit(1) kwargs = {k: v for k, v in step_dict.items() if k != "type"} kwargs.setdefault("version", None) - # retry_config lives on the TaskStep base but most step __init__s don't - # accept it; construct without it and hydrate (as Task.from_dict does), - # so retry_config on any step type doesn't crash create. put() below - # handles the kwargs form the same way. - from agent_env.task_step.task_step import attach_retry_config - - ctor_kwargs = {k: v for k, v in kwargs.items() if k != "retry_config"} - step = attach_retry_config(step_cls(**ctor_kwargs), step_dict) + # retry_config and step_role live on the TaskStep base but most step + # __init__s don't accept them; construct without them and hydrate (as + # Task.from_dict does). put() below handles the kwargs form the same way. + from agent_env.task_step.task_step import attach_base_fields + + ctor_kwargs = {k: v for k, v in kwargs.items() if k not in ("retry_config", "step_role")} + step = attach_base_fields(step_cls(**ctor_kwargs), step_dict) built.append((kwargs, step)) return built diff --git a/src/agent_env/task/outcome.py b/src/agent_env/task/outcome.py new file mode 100644 index 000000000..eb9192eca --- /dev/null +++ b/src/agent_env/task/outcome.py @@ -0,0 +1,129 @@ +"""What a step failure means for its run: the role the failed step played and why it failed. + +A tolerated failure (``fail_task_on_error=False``) lets the run complete, so these facts are the only record of what +the run lost. Each ``context.metadata.failed_steps`` entry carries them. +""" + +from __future__ import annotations + +import re +from collections.abc import Iterator +from dataclasses import dataclass +from enum import StrEnum +from typing import TYPE_CHECKING, Any + +from agent_env.task_step.task_step import StepRole + +if TYPE_CHECKING: + from agent_env.task_step.task_step import TaskStep + +__all__ = ["FailedStep", "FailureCause", "StepRole", "classify_failure"] + + +class FailureCause(StrEnum): + KEY_BUDGET = "key_budget" + KEY_AUTH = "key_auth" + TRANSIENT = "transient" + UNKNOWN = "unknown" + + +# Checked in order: a key that is out of budget or rejected fails every retry, so it wins over a timeout that wraps it. +_CAUSE_PATTERNS: tuple[tuple[FailureCause, re.Pattern[str]], ...] = ( + (FailureCause.KEY_BUDGET, re.compile( + r"budget\s+(?:has\s+been\s+)?exceeded|exceeded\s+(?:the\s+|your\s+)?(?:max\s+)?budget|ExceededBudget|BudgetExceeded", + re.IGNORECASE, + )), + (FailureCause.KEY_AUTH, re.compile( + r"\b401\b|unauthori[sz]ed|authentication\s*error|token_not_found_in_db|incorrect\s+api\s+key" + r"|invalid[\s_-]+(?:proxy[\s_-]+server[\s_-]+)?(?:api[\s_-]+)?(?:key|token)" + r"|(?:key|token)\s+(?:has\s+)?(?:been\s+)?(?:revoked|expired)|(?:revoked|expired)\s+(?:api[\s_-]+)?(?:key|token)", + re.IGNORECASE, + )), + (FailureCause.TRANSIENT, re.compile( + r"timed?[\s_-]*out|disconnect|connection\s*(?:reset|refused|aborted|closed|error)|broken\s*pipe|remote\s+end\s+closed" + r"|internal\s+server\s+error|bad\s+gateway|service\s+unavailable|gateway\s+time-?out" + r"|(?:HTTP|status(?:[\s_]code)?|error)[\s:=]*5\d\d\b" + r"|\b429\b|rate[\s_-]?limit|too\s+many\s+requests" + r"|OOM[\s_-]?kill|out\s+of\s+memory|\bOOM\b|exit(?:ed)?(?:\s+with)?\s+code\s+137", + re.IGNORECASE, + )), +) + + +def _chain(exc: BaseException) -> Iterator[BaseException]: + """``exc`` and every exception it was raised from or during, each once.""" + seen: set[int] = set() + stack: list[BaseException | None] = [exc] + while stack: + current = stack.pop() + if current is None or id(current) in seen: + continue + seen.add(id(current)) + yield current + stack.extend([current.__cause__, current.__context__]) + if isinstance(current, BaseExceptionGroup): + stack.extend(current.exceptions) + + +def classify_failure(exc: BaseException) -> FailureCause: + """Why ``exc`` happened, read from the type names and messages of its whole chain. + + Matching text is the only option: a judge or agent error crosses A2A and the model proxy as a message string, so its + original type is gone by the time a step raises. + """ + text = "\n".join(f"{type(e).__name__}: {e}" for e in _chain(exc)) + for cause, pattern in _CAUSE_PATTERNS: + if pattern.search(text): + return cause + return FailureCause.UNKNOWN + + +@dataclass(frozen=True) +class FailedStep: + """One ``context.metadata.failed_steps`` entry.""" + + step_id: str + step_type: str + error: str + error_type: str + started_at_utc: str + duration_seconds: float + is_fatal: bool + retried: bool = False + step_role: StepRole = StepRole.OTHER + cause: FailureCause = FailureCause.UNKNOWN + # Set once a later action (e.g. a regrade) repaired what this failure lost. + resolved_by: str | None = None + + @classmethod + def from_exception( + cls, step: TaskStep, exc: BaseException, *, started_at_utc: str, duration_seconds: float, is_fatal: bool, + ) -> FailedStep: + return cls( + step_id=step.id, + step_type=step.type, + error=str(exc), + error_type=type(exc).__name__, + started_at_utc=started_at_utc, + duration_seconds=duration_seconds, + is_fatal=is_fatal, + step_role=getattr(step, "step_role", StepRole.OTHER), + cause=classify_failure(exc), + ) + + def to_dict(self) -> dict[str, Any]: + """The stored form: the keys every reader already knows, plus ``step_role`` and ``cause``. ``retried`` and + ``resolved_by`` appear only once set, as ``retried`` always has.""" + out: dict[str, Any] = { + "step_id": self.step_id, "step_type": self.step_type, + "error": self.error, "error_type": self.error_type, + "started_at_utc": self.started_at_utc, "duration_seconds": self.duration_seconds, + "is_fatal": self.is_fatal, + "step_role": self.step_role.value, "cause": self.cause.value, + } + if self.retried: + out["retried"] = True + if self.resolved_by is not None: + out["resolved_by"] = self.resolved_by + return out + diff --git a/src/agent_env/task/task.py b/src/agent_env/task/task.py index df78a58f4..c99024f37 100644 --- a/src/agent_env/task/task.py +++ b/src/agent_env/task/task.py @@ -24,6 +24,7 @@ from agent_env.store.document_store import UnknownDocumentTypeError from agent_env.store.routing import run_scope from agent_env.task_step.preflight import pinned +from agent_env.task.outcome import FailedStep from agent_env.task.teardown import env_record_ids from agent_env.task_step.context import ENTITY_IDS_KEY, TaskStepContext, VerifierNotPassingError, regraft_redacted_keys from agent_env.task_step.context_ops import ContextUpdateOps, build_context_update_ops @@ -130,12 +131,12 @@ def _sub(value): return {k: _sub(v) for k, v in value.items()} return value - from agent_env.task_step.task_step import attach_retry_config + from agent_env.task_step.task_step import attach_base_fields out: list["TaskStep"] = [] for step in steps: data = _sub(step.to_dict()) - out.append(attach_retry_config(type(step).from_dict(data), data)) + out.append(attach_base_fields(type(step).from_dict(data), data)) return out @@ -460,9 +461,9 @@ def from_dict(cls, data: dict) -> "Task": f"{_registration.failure_note(_registration.TASK_STEPS, step_type)}", *problems]), unmet, ) - from agent_env.task_step.task_step import attach_retry_config + from agent_env.task_step.task_step import attach_base_fields - steps.append(attach_retry_config(step_cls.from_dict(step_dict), step_dict)) + steps.append(attach_base_fields(step_cls.from_dict(step_dict), step_dict)) return cls( id=data["id"], version=data.get("version"), @@ -681,12 +682,9 @@ async def _run_one(step: TaskStep, idx: int, dispatch_epoch: int) -> None: except BaseException as exc: duration = time.monotonic() - started_mono fatal = getattr(step, "fail_task_on_error", True) - context.metadata.setdefault("failed_steps", []).append({ - "step_id": step.id, "step_type": step.type, - "error": str(exc), "error_type": type(exc).__name__, - "started_at_utc": started_utc, "duration_seconds": duration, - "is_fatal": fatal, - }) + context.metadata.setdefault("failed_steps", []).append(FailedStep.from_exception( + step, exc, started_at_utc=started_utc, duration_seconds=duration, is_fatal=fatal, + ).to_dict()) if not fatal: await _heartbeat_step(step, pre_context, _store.TaskStepStatus.FAILURE, dispatch_epoch) raise diff --git a/src/agent_env/task_step/store.py b/src/agent_env/task_step/store.py index cf40652a4..db775b6fb 100644 --- a/src/agent_env/task_step/store.py +++ b/src/agent_env/task_step/store.py @@ -96,9 +96,9 @@ def _deserialize(self, doc: dict) -> TaskStep: f"Unknown task step type: {task_step_type}" f"{_registration.failure_note(_registration.TASK_STEPS, task_step_type)}" ) - from agent_env.task_step.task_step import attach_retry_config + from agent_env.task_step.task_step import attach_base_fields - return attach_retry_config(cls.from_dict(doc), doc) + return attach_base_fields(cls.from_dict(doc), doc) def execute_query(self, query: TaskStepQuery) -> list[TaskStep]: filt, sort = to_document_query(query) diff --git a/src/agent_env/task_step/task_step.py b/src/agent_env/task_step/task_step.py index c26c6dc6c..1540d5ee4 100644 --- a/src/agent_env/task_step/task_step.py +++ b/src/agent_env/task_step/task_step.py @@ -6,6 +6,7 @@ from abc import ABC, abstractmethod from collections.abc import Mapping from dataclasses import dataclass +from enum import StrEnum from typing import TYPE_CHECKING, Any, ClassVar, Optional, Self from agent_env.task_step.context import TaskStepContext @@ -40,13 +41,23 @@ def dependencies(raw: Any) -> list[TaskStepDependency] | None: return [dep if isinstance(dep, TaskStepDependency) else TaskStepDependency.from_dict(dep) for dep in raw] -def attach_retry_config(step: "TaskStep", data: dict[str, Any]) -> "TaskStep": - """Hydrate ``retry_config`` after ``step_cls.from_dict`` at the generic - parse sites, so the field survives round-trips on step types whose - ``__init__`` doesn't accept it. No-op if already set or absent.""" +class StepRole(StrEnum): + """What a step contributes to a run, so a tolerated failure says what the run lost.""" + + GRADER = "grader" + EVIDENCE = "evidence" + AGENT = "agent" + OTHER = "other" + + +def attach_base_fields(step: "TaskStep", data: dict[str, Any]) -> "TaskStep": + """Hydrate the base fields most step ``__init__``s don't accept (``retry_config``, ``step_role``) after + ``step_cls.from_dict`` at the generic parse sites, so they survive round-trips on every step type.""" raw = data.get("retry_config") if raw is not None and getattr(step, "retry_config", None) is None: step.retry_config = RetryConfig.from_dict(raw) + if data.get("step_role") is not None: + step.step_role = StepRole(data["step_role"]) return step @@ -97,6 +108,9 @@ class TaskStep(ABC): # Each field of a scope other than the task's, by its path in the stored step; any other field is task-scoped. A # subclass merges its parent's. Such a field is optional, so a step written out without it still builds and runs. field_scopes: ClassVar[Mapping[str, FieldScope]] = {"metadata.attribution": FieldScope.USER} + # A step type declares its role; one step overrides it in config, e.g. a run_code step that grades. Not + # ``role``: deploy_agent and modify_env_tool_access already use that name for their own config. + step_role: StepRole = StepRole.OTHER def __init_subclass__(cls, **kwargs: Any) -> None: super().__init_subclass__(**kwargs) @@ -163,6 +177,8 @@ def to_dict(self) -> dict[str, Any]: base["depends_on"] = [dataclasses.asdict(d) for d in self.depends_on] if self.retry_config is not None: base["retry_config"] = dataclasses.asdict(self.retry_config) + if self.step_role != type(self).step_role: + base["step_role"] = self.step_role.value return base @classmethod @@ -188,13 +204,11 @@ def put(cls, **kwargs: Any) -> Self: from .store import get_task_step_store kwargs.setdefault("version", None) - # Most step __init__s don't accept retry_config (it lives on the base but - # subclasses don't forward it); construct without it, then hydrate so a - # retry_config on any step type round-trips through the step store. - retry_config_raw = kwargs.pop("retry_config", None) + # Most step __init__s don't accept the base-only fields; construct without + # them, then hydrate so they round-trip through the step store on any type. + base_fields = {k: kwargs.pop(k) for k in ("retry_config", "step_role") if k in kwargs} instance = cls(**kwargs) - if retry_config_raw is not None: - attach_retry_config(instance, {"retry_config": retry_config_raw}) + attach_base_fields(instance, base_fields) return get_task_step_store().put_document(instance) @classmethod diff --git a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_agent_card.py b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_agent_card.py index a0d3e4a74..9c69f346e 100644 --- a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_agent_card.py +++ b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_agent_card.py @@ -7,7 +7,7 @@ from agent_env.task_step.context import TaskStepContext from agent_env.entity.refs import EntityRef -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency logger = logging.getLogger(__name__) @@ -17,6 +17,7 @@ class VerifyA2AAgentCardStep(TaskStep): type: ClassVar[str] = "verify_a2a_agent_card" + step_role = StepRole.GRADER entity_refs = (EntityRef.agent("a2a_agent_id", version_field="a2a_agent_version"),) def __init__( diff --git a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_agent_config_identity.py b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_agent_config_identity.py index 5ecd0429d..a74e1e308 100644 --- a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_agent_config_identity.py +++ b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_agent_config_identity.py @@ -10,13 +10,14 @@ from agent_env.a2a_agent import A2AAgent from agent_env.task_step.context import TaskStepContext from agent_env.entity.refs import EntityRef -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency logger = logging.getLogger(__name__) class VerifyA2AAgentConfigIdentityStep(TaskStep): type: ClassVar[str] = "verify_a2a_agent_config_identity" + step_role = StepRole.GRADER entity_refs = (EntityRef.agent("a2a_agent_id", version_field="a2a_agent_version"),) def __init__(self, id: str, version: Optional[int], a2a_agent_id: str, a2a_agent_version: Optional[int] = None, depends_on: Optional[list[TaskStepDependency]] = None, fail_task_on_error: bool = True): diff --git a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_agent_mcp.py b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_agent_mcp.py index 7842e83d4..6e9989d0a 100644 --- a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_agent_mcp.py +++ b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_agent_mcp.py @@ -12,7 +12,7 @@ from agent_env.task_step.context import TaskStepContext from agent_env.entity.refs import EntityRef -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency logger = logging.getLogger(__name__) @@ -25,6 +25,7 @@ class VerifyA2AAgentMCPStep(TaskStep): type: ClassVar[str] = "verify_a2a_agent_mcp" + step_role = StepRole.GRADER entity_refs = (EntityRef.agent("a2a_agent_id"),) def __init__( diff --git a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_builtin_tools.py b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_builtin_tools.py index b3aa1d457..a6ef9eca6 100644 --- a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_builtin_tools.py +++ b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_builtin_tools.py @@ -17,7 +17,7 @@ from agent_env.a2a_agent import A2AAgent from agent_env.entity.refs import EntityRef from agent_env.task_step.context import TaskStepContext -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency from agent_env.task_step.task_steps.deploy_agent import DeployAgentTaskStep from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep @@ -32,6 +32,7 @@ class VerifyA2ABuiltinToolsStep(TaskStep): type: ClassVar[str] = "verify_a2a_builtin_tools" + step_role = StepRole.GRADER entity_refs = (EntityRef.agent("a2a_agent_id", version_field="a2a_agent_version"),) def __init__( diff --git a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_core_protocol.py b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_core_protocol.py index 7615fe69d..0426c0c24 100644 --- a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_core_protocol.py +++ b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_core_protocol.py @@ -7,13 +7,14 @@ from agent_env.task_step.context import TaskStepContext from agent_env.entity.refs import EntityRef -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency logger = logging.getLogger(__name__) class VerifyCoreA2AProtocolStep(TaskStep): type: ClassVar[str] = "verify_a2a_core_protocol" + step_role = StepRole.GRADER entity_refs = (EntityRef.agent("a2a_agent_id"),) def __init__( diff --git a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_install.py b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_install.py index 71c04bd03..3f71d735f 100644 --- a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_install.py +++ b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_install.py @@ -14,13 +14,14 @@ from agent_env.task_step.context import TaskStepContext from agent_env.entity.refs import EntityRef -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency logger = logging.getLogger(__name__) class VerifyA2AInstallStep(TaskStep): type: ClassVar[str] = "verify_a2a_install" + step_role = StepRole.GRADER entity_refs = (EntityRef.agent("a2a_agent_id", version_field="a2a_agent_version"),) def __init__( diff --git a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_modalities.py b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_modalities.py index 4024a9bbb..497b7bca4 100644 --- a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_modalities.py +++ b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_modalities.py @@ -31,7 +31,7 @@ from agent_env.task_step.context import TaskStepContext from agent_env.entity.refs import EntityRef -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency logger = logging.getLogger(__name__) @@ -177,6 +177,7 @@ class VerifyA2AModalitiesStep(TaskStep): """ type: ClassVar[str] = "verify_a2a_modalities" + step_role = StepRole.GRADER entity_refs = (EntityRef.agent("a2a_agent_id"),) def __init__( diff --git a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_peer_agents.py b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_peer_agents.py index bba35f04b..849317c37 100644 --- a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_peer_agents.py +++ b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_peer_agents.py @@ -9,7 +9,7 @@ from agent_env.a2a_agent import A2AAgent from agent_env.task_step.context import TaskStepContext from agent_env.entity.refs import EntityRef -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency logger = logging.getLogger(__name__) @@ -22,6 +22,7 @@ class VerifyA2APeerAgentsStep(TaskStep): to use peer_send_message to relay an `expected_token` from the peer.""" type: ClassVar[str] = "verify_a2a_peer_agents" + step_role = StepRole.GRADER entity_refs = (EntityRef.agent("a2a_agent_id", version_field="a2a_agent_version"),) def __init__(self, id: str, version: Optional[int], a2a_agent_id: str, a2a_agent_version: Optional[int], peer_prompt_id: str, expected_token: str, depends_on: Optional[list[TaskStepDependency]] = None, fail_task_on_error: bool = True): diff --git a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_role.py b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_role.py index 4096ac42d..05de33aa5 100644 --- a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_role.py +++ b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_role.py @@ -28,13 +28,14 @@ from agent_env.task_step.context import TaskStepContext from agent_env.entity.refs import EntityRef -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency logger = logging.getLogger(__name__) class VerifyA2ARoleStep(TaskStep): type: ClassVar[str] = "verify_a2a_role" + step_role = StepRole.GRADER entity_refs = (EntityRef.agent("a2a_agent_id", version_field="a2a_agent_version"),) def __init__( diff --git a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_skill_config.py b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_skill_config.py index 4b800fdd0..cd1d896f7 100644 --- a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_skill_config.py +++ b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_skill_config.py @@ -19,13 +19,14 @@ from agent_env.a2a_agent.staging import transfer_store from agent_env.config import get_config from agent_env.entity.refs import EntityRef -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency logger = logging.getLogger(__name__) class VerifyA2ASkillConfigStep(TaskStep): type: ClassVar[str] = "verify_a2a_skill_config" + step_role = StepRole.GRADER entity_refs = (EntityRef.agent("a2a_agent_id"),) def __init__( diff --git a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_snapshot.py b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_snapshot.py index 434f60b47..a2a43cc1c 100644 --- a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_snapshot.py +++ b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_snapshot.py @@ -7,7 +7,7 @@ from agent_env.task_step.context import TaskStepContext from agent_env.entity.refs import EntityRef -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency logger = logging.getLogger(__name__) @@ -28,6 +28,7 @@ class VerifyA2ASnapshotStep(TaskStep): """ type: ClassVar[str] = "verify_a2a_snapshot" + step_role = StepRole.GRADER entity_refs = (EntityRef.agent("a2a_agent_id"),) def __init__( diff --git a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_system_prompt.py b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_system_prompt.py index 70daf95b8..06ec35b2a 100644 --- a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_system_prompt.py +++ b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_system_prompt.py @@ -34,13 +34,14 @@ from agent_env.task_step.context import TaskStepContext from agent_env.entity.refs import EntityRef -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency logger = logging.getLogger(__name__) class VerifyA2ASystemPromptStep(TaskStep): type: ClassVar[str] = "verify_a2a_system_prompt" + step_role = StepRole.GRADER entity_refs = (EntityRef.agent("a2a_agent_id", version_field="a2a_agent_version"),) def __init__( diff --git a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_trajectory.py b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_trajectory.py index d1f0dea80..755cbb0ce 100644 --- a/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_trajectory.py +++ b/src/agent_env/task_step/task_steps/a2a_agent_validator/verify_a2a_trajectory.py @@ -15,13 +15,14 @@ from agent_env.a2a_agent.staging import transfer_store from agent_env.task_step.snapshot_utils.agent_state_capture import trajectory_object_url from agent_env.entity.refs import EntityRef -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency logger = logging.getLogger(__name__) class VerifyA2ATrajectoryStep(TaskStep): type: ClassVar[str] = "verify_a2a_trajectory" + step_role = StepRole.GRADER entity_refs = (EntityRef.agent("a2a_agent_id"),) def __init__( diff --git a/src/agent_env/task_step/task_steps/collect_artifacts.py b/src/agent_env/task_step/task_steps/collect_artifacts.py index b021ba003..9492edee0 100644 --- a/src/agent_env/task_step/task_steps/collect_artifacts.py +++ b/src/agent_env/task_step/task_steps/collect_artifacts.py @@ -86,7 +86,7 @@ from agent_env.store.ids import derive_path_id, is_local_id, validate_local_id from agent_env.task_step.context import StepContextError, TaskStepContext from agent_env.entity.refs import EntityRef -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency from agent_env.task_step.task_steps.sandbox_utils.sandbox_utils import find_agent_container from agent_env.task_step.thread_work import finish_on_thread @@ -227,6 +227,7 @@ def _upload_file(store, file_path: str, artifact_id: str, version: int, object_n class CollectArtifactsTaskStep(TaskStep): type: ClassVar[str] = "collect_artifacts" + step_role = StepRole.EVIDENCE entity_refs = (EntityRef.env("env_id"),) def __init__( diff --git a/src/agent_env/task_step/task_steps/env_card_validator/verify_env_card.py b/src/agent_env/task_step/task_steps/env_card_validator/verify_env_card.py index 63bd1e7c2..fc32eace9 100644 --- a/src/agent_env/task_step/task_steps/env_card_validator/verify_env_card.py +++ b/src/agent_env/task_step/task_steps/env_card_validator/verify_env_card.py @@ -7,7 +7,7 @@ from agent_env.task_step.context import StepContextError, TaskStepContext from agent_env.entity.refs import EntityRef -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency logger = logging.getLogger(__name__) @@ -20,6 +20,7 @@ class VerifyEnvironmentCardStep(TaskStep): type: ClassVar[str] = "verify_env_card" + step_role = StepRole.GRADER entity_refs = (EntityRef.env("env_id"),) def __init__( diff --git a/src/agent_env/task_step/task_steps/env_card_validator/verify_env_core_protocol.py b/src/agent_env/task_step/task_steps/env_card_validator/verify_env_core_protocol.py index 20a92ff0b..c6358ac2b 100644 --- a/src/agent_env/task_step/task_steps/env_card_validator/verify_env_core_protocol.py +++ b/src/agent_env/task_step/task_steps/env_card_validator/verify_env_core_protocol.py @@ -11,7 +11,7 @@ from agent_env.task_step.context import StepContextError, TaskStepContext from agent_env.entity.refs import EntityRef -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency logger = logging.getLogger(__name__) @@ -27,6 +27,7 @@ class VerifyCoreEnvironmentProtocolStep(TaskStep): type: ClassVar[str] = "verify_env_core_protocol" + step_role = StepRole.GRADER entity_refs = (EntityRef.env("env_id"),) def __init__( diff --git a/src/agent_env/task_step/task_steps/mcp_env_validator/validation_gate_aggregator.py b/src/agent_env/task_step/task_steps/mcp_env_validator/validation_gate_aggregator.py index b0a015aef..6264b0c9d 100644 --- a/src/agent_env/task_step/task_steps/mcp_env_validator/validation_gate_aggregator.py +++ b/src/agent_env/task_step/task_steps/mcp_env_validator/validation_gate_aggregator.py @@ -15,7 +15,7 @@ from agent_env.task_step.context import TaskStepContext from agent_env.entity.refs import EntityRef -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency logger = logging.getLogger(__name__) @@ -51,6 +51,7 @@ def aggregate_gate(verifications: dict, required_gates) -> dict: class ValidationGateAggregatorStep(TaskStep): type: ClassVar[str] = "validation_gate_aggregator" + step_role = StepRole.GRADER entity_refs = (EntityRef.env("env_id"),) def __init__( diff --git a/src/agent_env/task_step/task_steps/mcp_env_validator/verify_mcp_env_assessment.py b/src/agent_env/task_step/task_steps/mcp_env_validator/verify_mcp_env_assessment.py index 72ba54990..1c1f8693a 100644 --- a/src/agent_env/task_step/task_steps/mcp_env_validator/verify_mcp_env_assessment.py +++ b/src/agent_env/task_step/task_steps/mcp_env_validator/verify_mcp_env_assessment.py @@ -8,13 +8,14 @@ from agent_env.task_step.context import StepContextError, TaskStepContext from agent_env.entity.refs import EntityRef -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency logger = logging.getLogger(__name__) class VerifyMCPEnvAssessmentStep(TaskStep): type: ClassVar[str] = "verify_mcp_env_assessment" + step_role = StepRole.GRADER entity_refs = (EntityRef.env("env_id"),) def __init__( diff --git a/src/agent_env/task_step/task_steps/mcp_env_validator/verify_mcp_tool_schema.py b/src/agent_env/task_step/task_steps/mcp_env_validator/verify_mcp_tool_schema.py index 1a7df4f06..8366ac898 100644 --- a/src/agent_env/task_step/task_steps/mcp_env_validator/verify_mcp_tool_schema.py +++ b/src/agent_env/task_step/task_steps/mcp_env_validator/verify_mcp_tool_schema.py @@ -11,7 +11,7 @@ from agent_env.task_step.context import StepContextError, TaskStepContext from agent_env.entity.refs import EntityRef -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency logger = logging.getLogger(__name__) @@ -36,6 +36,7 @@ def resolve_defs_ref(param_schema: dict, schema: dict) -> dict: class VerifyMCPToolSchemaTaskStep(TaskStep): type: ClassVar[str] = "verify_mcp_tool_schema" + step_role = StepRole.GRADER entity_refs = (EntityRef.env("env_id"),) def __init__( diff --git a/src/agent_env/task_step/task_steps/mcp_env_validator/verify_spec_conformance.py b/src/agent_env/task_step/task_steps/mcp_env_validator/verify_spec_conformance.py index 9787c3ae5..f9c69919d 100644 --- a/src/agent_env/task_step/task_steps/mcp_env_validator/verify_spec_conformance.py +++ b/src/agent_env/task_step/task_steps/mcp_env_validator/verify_spec_conformance.py @@ -29,7 +29,7 @@ from agent_env.env import legacy_protocol from agent_env.task_step.context import StepContextError, TaskStepContext from agent_env.entity.refs import EntityRef -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency from agent_env.task_step.task_steps.mcp_env_validator.verify_mcp_tool_schema import ( resolve_defs_ref, ) @@ -378,6 +378,7 @@ def _summarize(findings: list[dict], limit: int = 10) -> str: class VerifySpecConformanceTaskStep(TaskStep): type: ClassVar[str] = "verify_spec_conformance" + step_role = StepRole.GRADER entity_refs = (EntityRef.env("env_id"),) def __init__( diff --git a/src/agent_env/task_step/task_steps/multienv_validator/combine_universe_verdicts.py b/src/agent_env/task_step/task_steps/multienv_validator/combine_universe_verdicts.py index 541e31a40..2e5035dae 100644 --- a/src/agent_env/task_step/task_steps/multienv_validator/combine_universe_verdicts.py +++ b/src/agent_env/task_step/task_steps/multienv_validator/combine_universe_verdicts.py @@ -14,13 +14,14 @@ from agent_env.task_step.context import TaskStepContext from agent_env.entity.refs import EntityRef -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency logger = logging.getLogger(__name__) class CombineUniverseVerdictsStep(TaskStep): type: ClassVar[str] = "combine_universe_verdicts" + step_role = StepRole.GRADER entity_refs = ( EntityRef.env("env_id"), EntityRef.artifact("universe_artifact_id", version_field="universe_artifact_version"), diff --git a/src/agent_env/task_step/task_steps/multienv_validator/verify_universe_roundtrip.py b/src/agent_env/task_step/task_steps/multienv_validator/verify_universe_roundtrip.py index 585c2a945..7f3e1cbcb 100644 --- a/src/agent_env/task_step/task_steps/multienv_validator/verify_universe_roundtrip.py +++ b/src/agent_env/task_step/task_steps/multienv_validator/verify_universe_roundtrip.py @@ -14,7 +14,7 @@ from agent_env.store.ids import derive_id, fs_safe, is_local_id from agent_env.task_step.context import StepContextError, TaskStepContext from agent_env.entity.refs import EntityRef -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency from .universe_comparison import classify_issues, compare_dicts, normalize @@ -26,6 +26,7 @@ class VerifyUniverseLoadExportRoundtripStep(TaskStep): type: ClassVar[str] = "verify_universe_load_export_roundtrip" + step_role = StepRole.GRADER entity_refs = ( EntityRef.env("env_id"), EntityRef.artifact("universe_artifact_id", version_field="universe_artifact_version", artifact_type="environment_universe"), diff --git a/src/agent_env/task_step/task_steps/prompt_agent.py b/src/agent_env/task_step/task_steps/prompt_agent.py index 8098f1a4b..04165445f 100644 --- a/src/agent_env/task_step/task_steps/prompt_agent.py +++ b/src/agent_env/task_step/task_steps/prompt_agent.py @@ -38,7 +38,7 @@ from agent_env.task_step.context import DeployedAgent, PromptResponse, StepContextError, TaskStepContext from agent_env.entity.refs import EntityRef from agent_env.entity.scopes import FieldScope -from agent_env.task_step.task_step import RetryConfig, TaskStep, TaskStepDependency +from agent_env.task_step.task_step import RetryConfig, StepRole, TaskStep, TaskStepDependency from agent_env.task_step.task_steps.notification_delivery import delivered_parts, next_message, take_due from agent_env.task_step.task_steps.post_turn_hooks import PostTurnHooks, run_post_turn_hooks, validate_hook_specs from agent_env.task_step.task_steps.agent import user_sim_versions @@ -132,6 +132,7 @@ def _file_uris(parts: list[dict]) -> list[str]: class PromptAgentTaskStep(TaskStep): type: ClassVar[str] = "prompt_agent" + step_role = StepRole.AGENT DEFAULT_USER_AGENT_NAME: ClassVar[str] = "human_agent" entity_refs = ( EntityRef.env("snapshot_config.env_id"), diff --git a/src/agent_env/task_step/task_steps/snapshot_agent_state.py b/src/agent_env/task_step/task_steps/snapshot_agent_state.py index b491029d6..1326c9ac8 100644 --- a/src/agent_env/task_step/task_steps/snapshot_agent_state.py +++ b/src/agent_env/task_step/task_steps/snapshot_agent_state.py @@ -10,7 +10,7 @@ from agent_env.store.ids import derive_id, is_local_id, validate_local_id from agent_env.task_step.context import TaskStepContext from agent_env.entity.refs import EntityRef, RefRole -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency from agent_env.task_step.snapshot_utils import agent_state_capture from agent_env.task_step.snapshot_utils.snapshot_series import rollout_base @@ -49,6 +49,7 @@ class SnapshotAgentStateTaskStep(TaskStep): """ type: ClassVar[str] = "snapshot_agent_state" + step_role = StepRole.EVIDENCE entity_refs = ( EntityRef.artifact("artifact_id", role=RefRole.OUTPUT, artifact_type="file_artifact_universe"), EntityRef.env("env_id"), diff --git a/src/agent_env/task_step/task_steps/snapshot_env.py b/src/agent_env/task_step/task_steps/snapshot_env.py index d42d70943..64284ea91 100644 --- a/src/agent_env/task_step/task_steps/snapshot_env.py +++ b/src/agent_env/task_step/task_steps/snapshot_env.py @@ -35,7 +35,7 @@ from agent_env.task_step.context import TaskStepContext from agent_env.entity.refs import EntityRef, RefRole from agent_env.entity.scopes import FieldScope -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency if TYPE_CHECKING: from agent_env.env.legacy_protocol import BackingServer @@ -184,6 +184,7 @@ class SnapshotEnvTaskStep(TaskStep): """ type: ClassVar[str] = "snapshot_env" + step_role = StepRole.EVIDENCE entity_refs = ( EntityRef.env("env_id"), EntityRef.artifact("snapshot_id", role=RefRole.OUTPUT, artifact_type="environment_universe"), diff --git a/src/agent_env/task_step/task_steps/verifiers/agent_prompt_response_verifier.py b/src/agent_env/task_step/task_steps/verifiers/agent_prompt_response_verifier.py index 7d868f330..0c2c78672 100644 --- a/src/agent_env/task_step/task_steps/verifiers/agent_prompt_response_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/agent_prompt_response_verifier.py @@ -22,7 +22,7 @@ from typing import ClassVar, Optional from agent_env.task_step.context import StepContextError, TaskStepContext -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency from agent_env.task_step.task_steps.verifiers.scoring import ScoreAggregator, aggregate_score logger = logging.getLogger(__name__) @@ -35,6 +35,7 @@ class AgentPromptResponseVerifierTaskStep(TaskStep): type: ClassVar[str] = "agent_prompt_response_verifier" + step_role = StepRole.GRADER entity_refs = () def __init__( diff --git a/src/agent_env/task_step/task_steps/verifiers/aggregate_verifiers.py b/src/agent_env/task_step/task_steps/verifiers/aggregate_verifiers.py index 8d663a82d..ccb4541ab 100644 --- a/src/agent_env/task_step/task_steps/verifiers/aggregate_verifiers.py +++ b/src/agent_env/task_step/task_steps/verifiers/aggregate_verifiers.py @@ -16,7 +16,7 @@ from typing import ClassVar, Optional from agent_env.task_step.context import TaskStepContext -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency from agent_env.task_step.task_steps.verifiers.scoring import ScoreAggregator, aggregate_score logger = logging.getLogger(__name__) @@ -24,6 +24,7 @@ class AggregateVerifiersTaskStep(TaskStep): type: ClassVar[str] = "aggregate_verifiers" + step_role = StepRole.GRADER entity_refs = () def __init__( diff --git a/src/agent_env/task_step/task_steps/verifiers/env_outcome_verifier.py b/src/agent_env/task_step/task_steps/verifiers/env_outcome_verifier.py index 5e45774af..e5a035edb 100644 --- a/src/agent_env/task_step/task_steps/verifiers/env_outcome_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/env_outcome_verifier.py @@ -15,7 +15,7 @@ from agent_env.config import get_config from agent_env.task_step.context import PromptResponse, StepContextError, TaskStepContext from agent_env.entity.refs import EntityRef -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency from agent_env.task_step.task_steps.verifiers.run_context import load_run_context from agent_env.task_step.task_steps.verifiers.scoring import ScoreAggregator, aggregate_score @@ -32,6 +32,7 @@ class EnvOutcomeVerifierTaskStep(TaskStep): control token, so its calls reach the env's framework-only routes as written.""" type: ClassVar[str] = "env_outcome_verifier" + step_role = StepRole.GRADER entity_refs = ( EntityRef.env("env_id"), EntityRef.artifact("file_artifact_id", version_field="file_artifact_version", artifact_type="file"), diff --git a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py index a7796c22c..be2bb6eb9 100644 --- a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py @@ -47,7 +47,7 @@ from agent_env.entity.scopes import FieldScope from agent_env.store.base import NotFoundError from agent_env.store.ids import is_local_id -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency from agent_env.task_step.task_steps.verifiers.run_context import Run, load_run_context, render_run_context from agent_env.task_step.task_steps.verifiers.scoring import ScoreAggregator, aggregate_score from agent_env.task_step.task_steps.sandbox_utils.sandbox_utils import ( @@ -403,6 +403,7 @@ def _as_of_turn(prompt_response: PromptResponse, turn: Optional[int]) -> PromptR class RubricsVerifierTaskStep(TaskStep): type: ClassVar[str] = "rubrics_verifier" + step_role = StepRole.GRADER entity_refs = (EntityRef.agent("judge_a2a_agent_id"),) context_paths = ("context_facts_for_judge{}",) field_scopes = {**TaskStep.field_scopes, **dict.fromkeys( diff --git a/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py b/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py index ab8cb2e96..1b15b316b 100644 --- a/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py @@ -51,7 +51,7 @@ ) from agent_env.store.ids import derive_id, is_local_id, key_segment, validate_local_id from agent_env.task_step.context import StepContextError, TaskStepContext, dual_keyed -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency from agent_env.task_step.task_steps.run_code import _DriverTarget logger = logging.getLogger(__name__) @@ -87,6 +87,7 @@ class RunContainerUnitTestsVerifierTaskStep(TaskStep): type: ClassVar[str] = "run_container_unit_tests_verifier" + step_role = StepRole.GRADER entity_refs = (EntityRef.env("env_id"),) def __init__( diff --git a/src/agent_env/task_step/task_steps/verifiers/verify_sandbox.py b/src/agent_env/task_step/task_steps/verifiers/verify_sandbox.py index 5d5776b34..b8656d63b 100644 --- a/src/agent_env/task_step/task_steps/verifiers/verify_sandbox.py +++ b/src/agent_env/task_step/task_steps/verifiers/verify_sandbox.py @@ -36,7 +36,7 @@ get_sandbox_provider, ) from agent_env.task_step.context import StepContextError, TaskStepContext -from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency from agent_env.task_step.task_steps.sandbox_utils.sandbox_utils import find_agent_container from agent_env.task_step.task_steps.verifiers.scoring import ScoreAggregator, aggregate_score @@ -58,6 +58,7 @@ def _outcome(passed: bool, justification: str) -> dict: class VerifySandboxTaskStep(TaskStep): type: ClassVar[str] = "verify_sandbox" + step_role = StepRole.GRADER entity_refs = () def __init__( diff --git a/tst/unit/task/outcome_test.py b/tst/unit/task/outcome_test.py new file mode 100644 index 000000000..8385d1b1d --- /dev/null +++ b/tst/unit/task/outcome_test.py @@ -0,0 +1,141 @@ +"""A failed step's entry in ``context.metadata.failed_steps`` says what role the step played and why it failed.""" + +from __future__ import annotations + +import inspect + +import pytest + +from agent_env.config import set_document_store +from agent_env.store import LocalSqliteDocumentStore +from agent_env.task.outcome import FailureCause, StepRole, classify_failure +from agent_env.task.store import TASK_INSTANCES_COLLECTION, TaskInstanceStore, set_task_instance_store +from agent_env.task.task import Task +from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.registry import get_task_step_registry +from agent_env.task_step.task_steps.collect_artifacts import CollectArtifactsTaskStep +from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep +from agent_env.task_step.task_steps.run_code import RunCodeTaskStep +from agent_env.task_step.task_steps.snapshot_agent_state import SnapshotAgentStateTaskStep +from agent_env.task_step.task_steps.snapshot_env import SnapshotEnvTaskStep +from agent_env.task_step.task_steps.verifiers import AgentPromptResponseVerifierTaskStep, RubricsVerifierTaskStep + +BUDGET_400 = ( + "Judge A2A task failed (state=failed): API Error: 400 Budget exceeded for team-a: " + "spent $100.54 of $100.00 per 15d" +) + + +def _chained(outer: BaseException, inner: BaseException, *, explicit: bool = True) -> BaseException: + try: + try: + raise inner + except BaseException as e: + if explicit: + raise outer from e + raise outer + except BaseException as caught: + return caught + + +@pytest.mark.parametrize(("exc", "cause"), [ + (RuntimeError(BUDGET_400), FailureCause.KEY_BUDGET), + (Exception("litellm.BudgetExceededError: Budget has been exceeded! Current cost: 50.54, Max budget: 50.0"), + FailureCause.KEY_BUDGET), + (_chained(RuntimeError("LLM judge failed after 3 attempts"), RuntimeError(BUDGET_400)), FailureCause.KEY_BUDGET), + (_chained(RuntimeError("LLM judge failed after 3 attempts"), RuntimeError(BUDGET_400), explicit=False), + FailureCause.KEY_BUDGET), + (_chained(TimeoutError("judge timed out"), RuntimeError(BUDGET_400)), FailureCause.KEY_BUDGET), + (RuntimeError("Error code: 401 - {'error': {'message': 'Authentication Error, Invalid proxy server token passed. " + "valid_token=None.', 'type': 'auth_error', 'code': '401'}}"), FailureCause.KEY_AUTH), + (RuntimeError("Authentication Error: token_not_found_in_db"), FailureCause.KEY_AUTH), + (RuntimeError("litellm.AuthenticationError: OpenAIException - Incorrect API key provided: sk-...abcd"), + FailureCause.KEY_AUTH), + (RuntimeError("API key has been revoked"), FailureCause.KEY_AUTH), + (RuntimeError("Key has expired. Ask the key owner for a new one."), FailureCause.KEY_AUTH), + (RuntimeError("Server disconnected without sending a response."), FailureCause.TRANSIENT), + (TimeoutError(), FailureCause.TRANSIENT), + (ConnectionResetError(104, "Connection reset by peer"), FailureCause.TRANSIENT), + (RuntimeError("API Error: 503 Service Unavailable"), FailureCause.TRANSIENT), + (RuntimeError("Server error '502 Bad Gateway' for url 'https://judge.example/v1/chat'"), FailureCause.TRANSIENT), + (RuntimeError("sandbox command exited with code 137 (OOMKilled)"), FailureCause.TRANSIENT), + (RuntimeError("Error code: 429 - Too Many Requests"), FailureCause.TRANSIENT), + (ValueError("rubric item 3 has no criteria"), FailureCause.UNKNOWN), + (ValueError("judge returned invalid JSON: Expecting value: line 1 column 1"), FailureCause.UNKNOWN), +]) +def test_classify_failure(exc, cause): + assert classify_failure(exc) == cause + + +def test_step_types_declare_their_role(): + assert RubricsVerifierTaskStep.step_role == StepRole.GRADER + assert AgentPromptResponseVerifierTaskStep.step_role == StepRole.GRADER + assert SnapshotEnvTaskStep.step_role == StepRole.EVIDENCE + assert SnapshotAgentStateTaskStep.step_role == StepRole.EVIDENCE + assert CollectArtifactsTaskStep.step_role == StepRole.EVIDENCE + assert PromptAgentTaskStep.step_role == StepRole.AGENT + assert RunCodeTaskStep.step_role == StepRole.OTHER + + +def test_every_step_type_that_writes_verifications_is_a_grader(): + writers = { + name for name, cls in get_task_step_registry().items() + if '"verifications"' in inspect.getsource(inspect.getmodule(cls)) + } + assert writers, "the scan found no verifier, so it no longer reads the steps' source" + assert {name for name in writers if get_task_step_registry()[name].step_role != StepRole.GRADER} == set() + + +def _run_code(**extra) -> dict: + return {"id": "verify", "type": "run_code", "script_artifact_id": "s", "entrypoint": "main.py", **extra} + + +def test_a_step_can_declare_its_role_in_config(): + task = Task.from_dict({"id": "t", "type": "task", "version": 1, "steps": [_run_code(step_role="grader")]}) + assert task.steps[0].step_role == StepRole.GRADER + assert task.to_dict()["steps"][0]["step_role"] == "grader" + + +def test_a_step_without_a_role_override_serializes_as_before(): + task = Task.from_dict({"id": "t", "type": "task", "version": 1, "steps": [_run_code()]}) + assert "step_role" not in task.to_dict()["steps"][0] + + +def test_an_unknown_role_is_refused_when_the_task_loads(): + with pytest.raises(ValueError, match="judge"): + Task.from_dict({"id": "t", "type": "task", "version": 1, "steps": [_run_code(step_role="judge")]}) + + +class _OutOfBudgetVerifier(AgentPromptResponseVerifierTaskStep): + async def execute(self, context): + raise RuntimeError(BUDGET_400) + + +@pytest.fixture +def store(tmp_path): + doc = LocalSqliteDocumentStore(str(tmp_path / "instances.db")) + doc.ensure_index(TASK_INSTANCES_COLLECTION, ["instance_id"], unique=True) + set_document_store(doc) + s = TaskInstanceStore() + set_task_instance_store(s) + try: + yield s + finally: + set_task_instance_store(None) + + +@pytest.mark.asyncio +async def test_a_tolerated_grader_failure_records_its_role_and_cause(store): + grader = _OutOfBudgetVerifier("grade", None, prompt_id="p", fail_task_on_error=False) + ctx = TaskStepContext() + result = await Task(id="t", version=1, steps=[grader]).run(context=ctx) + + [entry] = result.metadata["failed_steps"] + assert entry == { + "step_id": "grade", "step_type": "agent_prompt_response_verifier", + "error": BUDGET_400, "error_type": "RuntimeError", + "started_at_utc": entry["started_at_utc"], "duration_seconds": entry["duration_seconds"], + "is_fatal": False, "step_role": "grader", "cause": "key_budget", + } + assert store.get(ctx.instance_id).status == "completed" + assert store.get(ctx.instance_id).context["metadata"]["failed_steps"] == [entry] diff --git a/tst/unit/task_step/retry_test.py b/tst/unit/task_step/retry_test.py index 15be8c585..62e33bcbd 100644 --- a/tst/unit/task_step/retry_test.py +++ b/tst/unit/task_step/retry_test.py @@ -812,7 +812,7 @@ def test_retry_config_from_dict_validates(): def test_retry_config_round_trips_on_non_prompt_step(): - # deploy_env's __init__ does not accept retry_config; attach_retry_config must + # deploy_env's __init__ does not accept retry_config; attach_base_fields must # still hydrate it on the generic parse path, and it must round-trip out. doc = { "id": "t", "type": "task", "version": 1,