Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 58 additions & 5 deletions src/agent_env/task/outcome.py
Original file line number Diff line number Diff line change
@@ -1,14 +1,15 @@
"""What a step failure means for its run: the role the failed step played and why it failed.
"""What a step failure means for its run: the role the failed step played, why it failed, and whether the completed
run is degraded.

A tolerated failure (``fail_task_on_error=False``) lets the run complete, so these facts are the only record of what
the run lost. Each ``context.metadata.failed_steps`` entry carries them.
the run lost. Each ``context.metadata.failed_steps`` entry carries them, and a completed run's ``quality`` sums them up.
"""

from __future__ import annotations

import re
from collections.abc import Iterator
from dataclasses import dataclass
from collections.abc import Iterable, Iterator, Mapping
from dataclasses import dataclass, field
from enum import StrEnum
from typing import TYPE_CHECKING, Any

Expand All @@ -17,7 +18,7 @@
if TYPE_CHECKING:
from agent_env.task_step.task_step import TaskStep

__all__ = ["FailedStep", "FailureCause", "StepRole", "classify_failure"]
__all__ = ["FailedStep", "FailureCause", "RunQuality", "StepRole", "classify_failure", "run_quality"]


class FailureCause(StrEnum):
Expand Down Expand Up @@ -111,6 +112,24 @@ def from_exception(
cause=classify_failure(exc),
)

@classmethod
def from_dict(cls, data: Mapping[str, Any]) -> FailedStep:
"""An entry as stored. One written before ``step_role`` and ``cause`` existed, or holding a value this version
does not know, reads as ``other`` and ``unknown``."""
return cls(
step_id=data.get("step_id", ""),
step_type=data.get("step_type", ""),
error=data.get("error", ""),
error_type=data.get("error_type", ""),
started_at_utc=data.get("started_at_utc", ""),
duration_seconds=data.get("duration_seconds", 0.0),
is_fatal=bool(data.get("is_fatal", True)),
retried=bool(data.get("retried", False)),
step_role=StepRole(data["step_role"]) if data.get("step_role") in _ROLES else StepRole.OTHER,
cause=FailureCause(data["cause"]) if data.get("cause") in _CAUSES else FailureCause.UNKNOWN,
resolved_by=data.get("resolved_by"),
)

def to_dict(self) -> dict[str, Any]:
"""The stored form: the keys every reader already knows, plus ``step_role`` and ``cause``. ``retried`` and
``resolved_by`` appear only once set, as ``retried`` always has."""
Expand All @@ -127,3 +146,37 @@ def to_dict(self) -> dict[str, Any]:
out["resolved_by"] = self.resolved_by
return out


_ROLES = frozenset(StepRole)
_CAUSES = frozenset(FailureCause)
# Most severe first: a run with no grade is worse than one missing its agent's turn, which is worse than one missing
# evidence.
_SEVERITY = (StepRole.GRADER, StepRole.AGENT, StepRole.EVIDENCE, StepRole.OTHER)


@dataclass(frozen=True)
class RunQuality:
"""How much of a completed run is usable. ``reasons`` holds the roles of the steps whose failure the run tolerated
and nothing has repaired, most severe first; a run with none is clean."""

reasons: tuple[StepRole, ...] = ()
# Derived, never passed, so it cannot disagree with ``reasons``; a field so ``asdict`` matches the stored form.
degraded: bool = field(init=False)

def __post_init__(self) -> None:
object.__setattr__(self, "degraded", bool(self.reasons))

@classmethod
def from_dict(cls, data: Mapping[str, Any] | None) -> RunQuality:
"""``quality`` as stored. A run completed before ``quality`` existed has none and reads as clean."""
roles = {StepRole(r) for r in (data or {}).get("reasons", ()) if r in _ROLES}
return cls(tuple(r for r in _SEVERITY if r in roles))

def to_dict(self) -> dict[str, Any]:
return {"degraded": self.degraded, "reasons": [r.value for r in self.reasons]}


def run_quality(failed_steps: Iterable[FailedStep]) -> RunQuality:
"""A fatal failure fails the run instead, and a retried or resolved one lost nothing."""
open_roles = {f.step_role for f in failed_steps if not f.is_fatal and not f.retried and f.resolved_by is None}
return RunQuality(tuple(r for r in _SEVERITY if r in open_roles))
11 changes: 11 additions & 0 deletions src/agent_env/task/store.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
commit_ordered,
replay_context,
)
from agent_env.task.outcome import FailedStep, RunQuality, run_quality
from agent_env.task_step.context import TaskStepContext

if TYPE_CHECKING:
Expand Down Expand Up @@ -184,6 +185,8 @@ class TaskInstance:
# carries the epoch it was dispatched under and no-ops when older than the stored one,
# fencing a writer from a rolled-back attempt. Missing (pre-field docs) reads as 0.
attempt_epoch: int = 0
# Stamped with ``status="completed"``; clean until then, and on a run completed before the field existed.
quality: RunQuality = field(default_factory=RunQuality)

@classmethod
def from_dict(cls, data: dict) -> TaskInstance:
Expand All @@ -202,6 +205,7 @@ def from_dict(cls, data: dict) -> TaskInstance:
completed_steps=[TaskStepResult.from_dict(e) for e in data.get("completed_steps", [])],
step_attempt_failures=data.get("step_attempt_failures", []),
attempt_epoch=data.get("attempt_epoch", 0),
quality=RunQuality.from_dict(data.get("quality")),
)


Expand Down Expand Up @@ -717,6 +721,13 @@ def _mutate(doc: dict) -> UpdateSpec | None:
if len(completed) >= total_steps and doc.get("status") not in ("failed", "cancelled"):
set_fields["status"] = "completed"
set_fields["completed_at_utc"] = completed_at_utc
# In the same write as the status, so no reader sees a completed run without its quality.
after = {"context": copy.deepcopy(doc.get("context"))}
_apply_forward(after, ops, move_to_end=rerecorded)
failed_steps = _get_path(after, "context.metadata.failed_steps") or []
set_fields["quality"] = run_quality(
FailedStep.from_dict(f) for f in failed_steps if isinstance(f, dict)
).to_dict()
return UpdateSpec(set=set_fields, unset=set(ops.unsets))

result = compare_and_swap(
Expand Down
102 changes: 102 additions & 0 deletions tst/unit/task/run_quality_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
"""A completed run carries ``quality``: degraded when it tolerated a step failure that nothing repaired."""

from __future__ import annotations

import pytest

from agent_env.config import set_document_store
from agent_env.store import LocalSqliteDocumentStore
from agent_env.store.document_store import Filter
from agent_env.task.outcome import FailedStep, RunQuality, StepRole, run_quality
from agent_env.task.store import TASK_INSTANCES_COLLECTION, TaskInstance, TaskInstanceStore, set_task_instance_store
from agent_env.task.task import Task
from agent_env.task_step.context import TaskStepContext
from agent_env.task_step.task_step import RetryConfig
from agent_env.task_step.task_steps.verifiers import AgentPromptResponseVerifierTaskStep


def _failed(role: StepRole, **fields) -> FailedStep:
return FailedStep(**{
"step_id": role.value, "step_type": "t", "error": "boom", "error_type": "RuntimeError",
"started_at_utc": "2026-01-01T00:00:00+00:00", "duration_seconds": 1.0, "is_fatal": False,
"step_role": role, **fields,
})


def test_a_run_with_no_failures_is_clean():
assert run_quality([]) == RunQuality()
assert RunQuality().to_dict() == {"degraded": False, "reasons": []}


def test_open_tolerated_failures_degrade_the_run_most_severe_first():
quality = run_quality([_failed(StepRole.EVIDENCE), _failed(StepRole.GRADER), _failed(StepRole.EVIDENCE)])
assert quality.to_dict() == {"degraded": True, "reasons": ["grader", "evidence"]}


@pytest.mark.parametrize("fields", [{"is_fatal": True}, {"retried": True}, {"resolved_by": "regrade:2026-01-02"}])
def test_a_fatal_retried_or_resolved_failure_does_not_degrade_the_run(fields):
assert not run_quality([_failed(StepRole.GRADER, **fields)]).degraded


def test_an_entry_written_before_roles_existed_degrades_as_other():
legacy = {"step_id": "s", "step_type": "run_code", "error": "boom", "error_type": "RuntimeError",
"started_at_utc": "2026-01-01T00:00:00+00:00", "duration_seconds": 1.0, "is_fatal": False}
assert run_quality([FailedStep.from_dict(legacy)]).to_dict() == {"degraded": True, "reasons": ["other"]}


def test_a_run_completed_before_quality_existed_reads_as_clean():
doc = {"instance_id": "i", "task_id": "t", "task_version": 1, "status": "completed",
"current_step": 1, "total_steps": 1}
assert TaskInstance.from_dict(doc).quality == RunQuality()


class _FailingVerifier(AgentPromptResponseVerifierTaskStep):
def __init__(self, *args, fail_times: int, **kwargs):
super().__init__(*args, **kwargs)
self.fail_times = fail_times

async def execute(self, context):
if self.fail_times:
self.fail_times -= 1
raise RuntimeError("Judge A2A task failed: API Error: 503 Service Unavailable")
return context


@pytest.fixture
def store(tmp_path):
doc = LocalSqliteDocumentStore(str(tmp_path / "instances.db"))
doc.ensure_index(TASK_INSTANCES_COLLECTION, ["instance_id"], unique=True)
set_document_store(doc)
s = TaskInstanceStore()
set_task_instance_store(s)
try:
yield s
finally:
set_task_instance_store(None)


def _stored_quality(store, instance_id):
return store._doc_store.find_one(TASK_INSTANCES_COLLECTION, Filter.of(instance_id=instance_id))["quality"]


@pytest.mark.asyncio
async def test_completing_with_a_tolerated_grader_failure_stamps_a_degraded_quality(store):
grader = _FailingVerifier("grade", None, prompt_id="p", fail_task_on_error=False, fail_times=1)
ctx = TaskStepContext()
await Task(id="t", version=1, steps=[grader]).run(context=ctx)

assert _stored_quality(store, ctx.instance_id) == {"degraded": True, "reasons": ["grader"]}
instance = store.get(ctx.instance_id)
assert instance.status == "completed"
assert instance.quality == RunQuality((StepRole.GRADER,))


@pytest.mark.asyncio
async def test_completing_after_a_successful_retry_stamps_a_clean_quality(store):
grader = _FailingVerifier("grade", None, prompt_id="p", fail_task_on_error=False, fail_times=1)
grader.retry_config = RetryConfig(retry_from_step_id="grade", max_retries=1)
ctx = TaskStepContext()
result = await Task(id="t", version=1, steps=[grader]).run(context=ctx)

assert [f.get("retried") for f in result.metadata["failed_steps"]] == [True]
assert _stored_quality(store, ctx.instance_id) == {"degraded": False, "reasons": []}
Loading