Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion packages/agentenv-protocol/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -1378,7 +1378,8 @@ Expected execution failures are returned as `TaskResult.failure(code, message)`
and become failed A2A tasks with `error_type`, `error_code`, and `error_message`
in the terminal message. `error_type` is the platform classification
(`agent_error` by default, or explicitly `infra_error` for a retryable
infrastructure failure); `error_code` preserves the author's machine-readable
infrastructure failure, `model_key_budget` for a model key out of budget, or
`model_key_auth` for a model key the endpoint rejects); `error_code` preserves the author's machine-readable
code. An exception escaping `run()`, an invalid return value, or a
result-mapping failure is logged with a correlation ID and reported as an
`infra_error` with code `framework.unhandled_exception`; raw exception text is
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@
FilePart,
NativeTrajectory,
TaskError,
TaskErrorType,
TaskOutcome,
TaskPart,
TaskProgress,
Expand Down Expand Up @@ -229,6 +230,7 @@
"SnapshotWriteObjects",
"StagingStore",
"TaskError",
"TaskErrorType",
"TaskEventsTrajectoryRequest",
"TaskObjectTrajectoryRequest",
"TaskOutcome",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
from dataclasses import dataclass, field
from enum import Enum
from pathlib import Path
from typing import Annotated, Any, Generic, Literal, TypeAlias, TypeVar, Union
from typing import Annotated, Any, Generic, Literal, TypeAlias, TypeVar, Union, get_args

from a2a.types import TaskStatusUpdateEvent
from pydantic import BaseModel, ConfigDict, Field, model_validator
Expand Down Expand Up @@ -265,15 +265,21 @@ class TaskOutcome(str, Enum):
FAILED = "failed"


# model_key_budget and model_key_auth: the model endpoint refused the run's key, as out of budget or not
# authorized. Neither is the agent's fault, and a retry with the same key fails the same way.
TaskErrorType: TypeAlias = Literal["agent_error", "infra_error", "model_key_budget", "model_key_auth"]
TASK_ERROR_TYPES: frozenset[str] = frozenset(get_args(TaskErrorType))


@dataclass(frozen=True, slots=True, kw_only=True)
class TaskError:
code: str
message: str
error_type: Literal["agent_error", "infra_error"] = "agent_error"
error_type: TaskErrorType = "agent_error"

def __post_init__(self) -> None:
if self.error_type not in {"agent_error", "infra_error"}:
raise ValueError("error_type must be 'agent_error' or 'infra_error'")
if self.error_type not in TASK_ERROR_TYPES:
raise ValueError(f"error_type must be one of {sorted(TASK_ERROR_TYPES)}")


class _TaskFailed(Exception):
Expand All @@ -284,7 +290,7 @@ def __init__(
code: str,
message: str,
*,
error_type: Literal["agent_error", "infra_error"] = "agent_error",
error_type: TaskErrorType = "agent_error",
) -> None:
super().__init__(message)
self.error = TaskError(code=code, message=message, error_type=error_type)
Expand Down Expand Up @@ -423,7 +429,7 @@ def failure(
code: str,
message: str,
*,
error_type: Literal["agent_error", "infra_error"] = "agent_error",
error_type: TaskErrorType = "agent_error",
parts: Iterable[TaskPart] | None = None,
) -> "TaskResult":
return (
Expand Down Expand Up @@ -455,7 +461,7 @@ def failed(
code: str,
message: str,
*,
error_type: Literal["agent_error", "infra_error"] = "agent_error",
error_type: TaskErrorType = "agent_error",
) -> "TaskResultBuilder":
self._outcome = TaskOutcome.FAILED
self._error = TaskError(code=code, message=message, error_type=error_type)
Expand Down
2 changes: 2 additions & 0 deletions packages/agentenv-protocol/tests/test_a2a_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -1148,6 +1148,8 @@ def test_task_result_builder_validates_and_factories_share_it() -> None:
)
assert infra_failure.error is not None
assert infra_failure.error.error_type == "infra_error"
for key_error in ("model_key_budget", "model_key_auth"):
assert TaskResult.failure("cli_error", "refused", error_type=key_error).error.error_type == key_error
with pytest.raises(ValueError, match="error_type"):
TaskResult.failure("cli_error", "failed", error_type="custom") # type: ignore[arg-type]
with pytest.raises(TypeError, match="unexpected keyword argument 'details'"):
Expand Down
3 changes: 2 additions & 1 deletion src/agent_env/agents/AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,8 @@ terminal state, `completed`, `failed` or `canceled`. Your `run()` is one turn.
`{error_type, error_code, error_message}`. A success needs at least one part, and `builder().failed(...)` adds no
text part, so add one.
- **Errors:** `agent_error`, the default, means the agent tried and failed; `infra_error` means the run couldn't
happen, and agent-env may retry it. An exception that escapes `run()` becomes an opaque `infra_error`, its text
happen, and agent-env may retry it; `model_key_budget` and `model_key_auth` mean the model endpoint refused the
run's key as out of budget or not authorized, which a retry with that key doesn't fix. An exception that escapes `run()` becomes an opaque `infra_error`, its text
never sent.
- **Context and sessions:** turns of one `context_id` run one at a time; different contexts run concurrently, and the
validator relies on it, so don't serialize them. Return your runtime's conversation id as `session_ref` and the next
Expand Down
2 changes: 1 addition & 1 deletion src/agent_env/agents/agents/claude-code/a2a_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -508,7 +508,7 @@ async def _turn(self, request: TaskRequest[ClaudeCodeConfig], cfg: ClaudeCodeCon
data["cli_attempts"] = result.attempts
builder = TaskResult.builder()
if result.error:
builder.failed("cli_error", result.error[:_ERROR_MESSAGE_LIMIT])
builder.failed("cli_error", result.error[:_ERROR_MESSAGE_LIMIT], error_type=result.error_type or "agent_error")
else:
builder.succeeded()
return (
Expand Down
26 changes: 25 additions & 1 deletion src/agent_env/agents/agents/claude-code/claude_bridge.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
from pathlib import Path
from typing import Literal

from agentenv_protocol.a2a_agent import TaskErrorType
from pydantic import BaseModel, ConfigDict, model_validator

from kit import in_thread_to_its_end
Expand Down Expand Up @@ -148,6 +149,8 @@ class ClaudeResult:
structured_output: dict | None = None
# How many `claude -p` runs the turn took; more than one means it was resumed.
attempts: int = 1
# The failure's type for agent-env, when the CLI's terminal `result` names a cause that isn't the agent's.
error_type: TaskErrorType | None = None


def _attribution_headers(project_id: str | None, task_id: str | None) -> dict[str, str]:
Expand Down Expand Up @@ -279,6 +282,25 @@ def _resumable_api_failure(result: ClaudeResult) -> str | None:
return text if any(marker in text.lower() for marker in _RESUMABLE_API_ERROR_MARKERS) else None


# The CLI copies the endpoint's error body `type` into `api_error_type`: LiteLLM sends budget_exceeded and auth_error,
# Anthropic authentication_error and permission_error.
_KEY_AUTH_ERROR_TYPES = frozenset({"auth_error", "authentication_error", "permission_error"})
_KEY_AUTH_STATUSES = frozenset({401, 403})


def _key_error_type(trajectory: list[dict]) -> TaskErrorType | None:
"""model_key_budget or model_key_auth when the endpoint refused the run's key, else None."""
event = _terminal_result(trajectory)
if event is None or not event.get("is_error"):
return None
kind = event.get("api_error_type")
if kind == "budget_exceeded":
return "model_key_budget"
if kind in _KEY_AUTH_ERROR_TYPES or _status(event.get("api_error_status")) in _KEY_AUTH_STATUSES:
return "model_key_auth"
return None


def _endpoint_hint(trajectory: list[dict]) -> str | None:
"""What a 404 from the model endpoint likely means."""
event = _terminal_result(trajectory)
Expand Down Expand Up @@ -756,7 +778,9 @@ def on_line(line: str) -> None:

err: str | None = None
detail: str | None = None
error_type: TaskErrorType | None = None
if proc.returncode != 0:
error_type = _key_error_type(trajectory)
err = (f"claude CLI was killed by signal {-proc.returncode}" if proc.returncode < 0
else f"claude CLI exited with code {proc.returncode}")
detail = _cli_error_text(trajectory) or _stderr_tail(stderr.decode(errors="replace"))
Expand All @@ -770,7 +794,7 @@ def on_line(line: str) -> None:

return ClaudeResult(response=final_response or "", tool_call_count=tool_count,
session_id=cli_session_id or session_id, trajectory=trajectory, error=err,
error_detail=detail, structured_output=structured_output)
error_detail=detail, structured_output=structured_output, error_type=error_type)


_TAGS_HEADER = "x-litellm-tags"
Expand Down
6 changes: 3 additions & 3 deletions src/agent_env/agents/agents/claude-code/requirements.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# The image installs the newest agentenv-framework-protocol at or above this floor; a release doesn't pin it, so the
# image is built again only when this folder changes. The floor is the first that serves peer-agents/v1's tools, which
# the agent advertises without handlers of its own; raise it when the agent needs a newer protocol.
agentenv-framework-protocol[agent]>=0.1.351
# image is built again only when this folder changes. The floor is the first whose TaskError takes model_key_budget and
# model_key_auth; raise it when the agent needs a newer protocol.
agentenv-framework-protocol[agent]>=0.1.551
7 changes: 4 additions & 3 deletions tst/unit/a2a_agent/protocol_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -115,17 +115,18 @@ def test_sdk_success_round_trips_through_platform_parser():
assert parsed.tool_call_count == 3


def test_sdk_failure_round_trips_through_platform_parser():
@pytest.mark.parametrize("error_type", ["infra_error", "model_key_budget", "model_key_auth"])
def test_sdk_failure_round_trips_through_platform_parser(error_type):
result = TaskResult.failure(
"provider_rate_limited",
"The provider is temporarily unavailable",
error_type="infra_error",
error_type=error_type,
)

parsed = TerminalResponse.from_message(_sdk_terminal_message(result))

assert parsed.response_text == "The provider is temporarily unavailable"
assert parsed.error_type == "infra_error"
assert parsed.error_type == error_type
assert parsed.error_code == "provider_rate_limited"
assert parsed.error_message == "The provider is temporarily unavailable"

Expand Down
2 changes: 1 addition & 1 deletion tst/unit/agents/claude_code/test_image_contract.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ def test_the_image_installs_the_protocol_alone_from_its_requirements():
requirements = [line for line in (AGENT_DIR / "requirements.txt").read_text().splitlines()
if line and not line.startswith("#")]

assert requirements == ["agentenv-framework-protocol[agent]>=0.1.351"]
assert requirements == ["agentenv-framework-protocol[agent]>=0.1.551"]


def test_the_image_gives_the_model_node_alone_from_a_pinned_image():
Expand Down
57 changes: 57 additions & 0 deletions tst/unit/agents/claude_code/test_model_key_errors.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
"""A turn the model endpoint refused for the run's key fails as model_key_budget or model_key_auth, not agent_error.

The CLI exits 1 and copies the endpoint's status and error body `type` into its terminal `result` event. The events
below are trimmed from captures of CLI 2.1.325 against a local endpoint answering with each body.
"""
import asyncio
import json

import pytest

from tst.unit.agents.claude_code.agent import claude_bridge
from tst.unit.agents.claude_code.support import FakeCLI, send, terminal


def _result(status: int, kind: str, text: str, terminal_reason: str = "api_error") -> dict:
return {
"type": "result", "subtype": "success", "is_error": True, "session_id": "sess-1", "num_turns": 1,
"terminal_reason": terminal_reason, "api_error_status": status, "api_error_type": kind, "result": text,
}


@pytest.fixture
def cli(monkeypatch):
"""Every `claude -p` prints the init event and ``event``, then exits 1."""
monkeypatch.setattr(claude_bridge, "_RETRY_BACKOFF_S", (0, 0))

def _script(event: dict) -> None:
stdout = "".join(json.dumps(e) + "\n" for e in (
{"type": "system", "subtype": "init", "session_id": "sess-1"}, event)).encode()

async def _exec(*cmd, **kwargs):
return FakeCLI(stdout, 1)

monkeypatch.setattr(asyncio, "create_subprocess_exec", _exec)

return _script


@pytest.mark.parametrize(("event", "error_type"), [
(_result(400, "budget_exceeded", "API Error: 400 Budget has been exceeded! Current cost: 10.1, Max budget: 10.0"),
"model_key_budget"),
(_result(401, "auth_error",
"Failed to authenticate. API Error: 401 Authentication Error, Invalid proxy server token passed."),
"model_key_auth"),
(_result(401, "authentication_error", "Invalid API key · Fix external API key"), "model_key_auth"),
(_result(403, "permission_error", "Failed to authenticate. API Error: 403 forbidden"), "model_key_auth"),
(_result(429, "rate_limit_error", "API Error: Request rejected (429) · slow down"), "agent_error"),
(_result(400, "invalid_request_error", "Prompt is too long", terminal_reason="prompt_too_long"), "agent_error"),
], ids=["litellm-budget", "litellm-auth", "anthropic-auth", "anthropic-permission", "rate-limit", "prompt-too-long"])
def test_a_failed_turn_says_whether_the_endpoint_refused_the_key(cli, client, event, error_type):
cli(event)

state, _, data = terminal(send(client, "hello"))

assert state == "failed"
assert (data[-1]["error_type"], data[-1]["error_code"]) == (error_type, "cli_error")
assert event["result"] in data[-1]["error_message"]
Loading