Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions src/agent_env/task_step/context.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

import contextlib
import dataclasses
import os
import re
from collections.abc import Callable
from dataclasses import dataclass, field
Expand Down Expand Up @@ -59,6 +60,31 @@ class VerifierNotPassingError(RuntimeError):
"""A verifier set to fail its task when its verdict isn't passing recorded one that isn't. Grading the same run
again could pass it, so a runner should not retry it."""


ERRORED_TURN_UNGRADED_ENV_VAR = "AGENT_ENV_ERRORED_TURN_UNGRADED"


# Turn failures the agent did not cause: the platform or its model key. An ``agent_error`` stays a scored 0, since
# the agent made the attempt and failed it.
PLATFORM_TURN_ERROR_TYPES = frozenset({"infra_error", "model_key_budget", "model_key_auth"})


def errored_turn_ungraded(turn_error_type: str) -> bool:
"""Whether a verifier leaves this errored agent turn ungraded, raising ``AgentTurnError``, instead of scoring it
0: only behind the flag, and only for failures the agent did not cause."""
flag_on = os.environ.get(ERRORED_TURN_UNGRADED_ENV_VAR, "").strip().lower() in ("1", "true", "yes")
return flag_on and turn_error_type in PLATFORM_TURN_ERROR_TYPES


class AgentTurnError(RuntimeError):
"""The agent turn a verifier grades ended in an error, so it has no answer to grade. The run is ungraded, not
failed by the agent: a 0 would count against the agent for a failure ``turn_error_type`` may say isn't its own."""

def __init__(self, verifier_id: str, prompt_id: str, turn_error_type: str) -> None:
super().__init__(f"Verifier '{verifier_id}' can't grade prompt '{prompt_id}': its agent turn failed with "
f"error_type={turn_error_type}")
self.turn_error_type = turn_error_type

_REDACTED_KEYS = {
"litellm_api_key", "judge_litellm_api_key", "usersim_api_key", "remote_tokens", "cf_access_client_secret",
HEARTBEAT_KEY,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
import uuid
from typing import ClassVar, Optional

from agent_env.task_step.context import StepContextError, TaskStepContext
from agent_env.task_step.context import AgentTurnError, StepContextError, TaskStepContext, errored_turn_ungraded
from agent_env.task_step.task_step import TaskStep, TaskStepDependency
from agent_env.task_step.task_steps.verifiers.scoring import ScoreAggregator, aggregate_score

Expand Down Expand Up @@ -84,6 +84,8 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext:
raise StepContextError(f"PromptResponse with prompt_id='{self.prompt_id}' not found in context")

if prompt_response.error_type:
if errored_turn_ungraded(prompt_response.error_type):
raise AgentTurnError(self.verifier_id, self.prompt_id, prompt_response.error_type)
logger.warning(
f"Skipping verification '{self.verifier_id}': prompt had error_type={prompt_response.error_type}"
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,14 @@
from agent_env.config.model import ModelParam
from agent_env.providers.sandbox_providers.sandbox import into_container
from agent_env.task_step.context import (
AgentTurnError,
PromptResponse,
StepContextError,
TaskStepContext,
VerifierNotPassingError,
context_path_segments,
dual_keyed,
errored_turn_ungraded,
)
from agent_env.task_step.task_steps.verifiers.judge_utils.judge_output_format import (
SCREENSHOT_GROUNDING,
Expand Down Expand Up @@ -683,6 +685,8 @@ async def _grade(self, context: TaskStepContext) -> TaskStepContext:
stored_response, prompt_response = prompt_response, as_of

if prompt_response.error_type:
if errored_turn_ungraded(prompt_response.error_type):
raise AgentTurnError(self.verifier_id, self.prompt_id, prompt_response.error_type)
logger.warning(f"Skipping verification '{self.verifier_id}': prompt had error_type={prompt_response.error_type}")
self._write_skipped_entries(
context, "prompt_error", f"Skipped: prompt had error_type={prompt_response.error_type}")
Expand Down
106 changes: 106 additions & 0 deletions tst/unit/task_step/verifiers/test_errored_turn_ungraded.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
"""AGENT_ENV_ERRORED_TURN_UNGRADED: a verifier whose agent turn errored leaves the run ungraded, failing its own step
with AgentTurnError, instead of recording a score of 0 that counts as the agent failing the task."""
from __future__ import annotations

import pytest

from agent_env.task import Task
from agent_env.task_step.context import ERRORED_TURN_UNGRADED_ENV_VAR, PromptResponse, TaskStepContext
from agent_env.task_step.task_steps.verifiers.agent_prompt_response_verifier import (
AgentPromptResponseVerifierTaskStep,
)
from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep


def _rubrics(monkeypatch) -> RubricsVerifierTaskStep:
step = RubricsVerifierTaskStep(
id="grade", version=1, criteria=[{"id": "done", "description": "it is done"}], prompt_id="p1",
use_agent_judge=False, use_trajectory=True, verifier_id="grade", fail_task_on_error=False)

async def judge(**kwargs):
return [{"id": "done", "description": "it is done", "score": 1.0, "result": True}], 0, [], None

monkeypatch.setattr(step, "_read_trajectory_text", lambda uri: "EVENTS")
monkeypatch.setattr(step, "_run_judge_with_output_retries", judge)
return step


def _response(monkeypatch) -> AgentPromptResponseVerifierTaskStep:
return AgentPromptResponseVerifierTaskStep(
id="grade", version=None, prompt_id="p1", criteria=[{"type": "response_contains", "needles": ["done"]}],
verifier_id="grade", fail_task_on_error=False)


VERIFIERS = pytest.mark.parametrize("make_verifier", [_rubrics, _response], ids=["rubrics", "prompt-response"])


def _context(error_type: str | None) -> TaskStepContext:
return TaskStepContext(prompt_responses=[PromptResponse(
prompt_id="p1", response="done", prompt_text="do it", agent_trajectory_object_url="s3://t/1.json",
error_type=error_type)])


@VERIFIERS
@pytest.mark.asyncio
async def test_off_an_errored_turn_scores_zero(monkeypatch, local_stores, make_verifier):
monkeypatch.delenv(ERRORED_TURN_UNGRADED_ENV_VAR, raising=False)
ctx = _context("model_key_budget")

await Task(id="t", version=None, steps=[make_verifier(monkeypatch)]).run(context=ctx)

entry = ctx.metadata["verifications"]["grade"]
assert (entry["score"], entry["results"][0]["id"]) == (0, "prompt_error")
assert "failed_steps" not in ctx.metadata


@VERIFIERS
@pytest.mark.asyncio
async def test_on_an_errored_turn_is_left_ungraded_and_its_verifier_fails(monkeypatch, local_stores, make_verifier):
monkeypatch.setenv(ERRORED_TURN_UNGRADED_ENV_VAR, "1")
ctx = _context("model_key_budget")

await Task(id="t", version=None, steps=[make_verifier(monkeypatch)]).run(context=ctx)

assert "grade" not in ctx.metadata.get("verifications", {})
[failed] = ctx.metadata["failed_steps"]
assert (failed["step_id"], failed["error_type"]) == ("grade", "AgentTurnError")
assert "error_type=model_key_budget" in failed["error"]


@VERIFIERS
@pytest.mark.parametrize("platform_error", ["infra_error", "model_key_auth"])
@pytest.mark.asyncio
async def test_on_other_platform_failures_are_left_ungraded(monkeypatch, local_stores, make_verifier, platform_error):
monkeypatch.setenv(ERRORED_TURN_UNGRADED_ENV_VAR, "1")
ctx = _context(platform_error)

await Task(id="t", version=None, steps=[make_verifier(monkeypatch)]).run(context=ctx)

assert "grade" not in ctx.metadata.get("verifications", {})
[failed] = ctx.metadata["failed_steps"]
assert f"error_type={platform_error}" in failed["error"]


@VERIFIERS
@pytest.mark.asyncio
async def test_on_an_agent_error_still_scores_zero(monkeypatch, local_stores, make_verifier):
monkeypatch.setenv(ERRORED_TURN_UNGRADED_ENV_VAR, "1")
ctx = _context("agent_error")

await Task(id="t", version=None, steps=[make_verifier(monkeypatch)]).run(context=ctx)

entry = ctx.metadata["verifications"]["grade"]
assert (entry["score"], entry["results"][0]["id"]) == (0, "prompt_error")
assert "failed_steps" not in ctx.metadata


@VERIFIERS
@pytest.mark.asyncio
async def test_on_a_turn_without_an_error_is_graded(monkeypatch, local_stores, make_verifier):
monkeypatch.setenv(ERRORED_TURN_UNGRADED_ENV_VAR, "1")
ctx = _context(None)

await Task(id="t", version=None, steps=[make_verifier(monkeypatch)]).run(context=ctx)

assert ctx.metadata["verifications"]["grade"]["score"] == 1.0
assert "failed_steps" not in ctx.metadata
Loading