diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index 94cd518fe..348a61ef3 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -1378,7 +1378,8 @@ Expected execution failures are returned as `TaskResult.failure(code, message)` and become failed A2A tasks with `error_type`, `error_code`, and `error_message` in the terminal message. `error_type` is the platform classification (`agent_error` by default, or explicitly `infra_error` for a retryable -infrastructure failure); `error_code` preserves the author's machine-readable +infrastructure failure, `model_key_budget` for a model key out of budget, or +`model_key_auth` for a model key the endpoint rejects); `error_code` preserves the author's machine-readable code. An exception escaping `run()`, an invalid return value, or a result-mapping failure is logged with a correlation ID and reported as an `infra_error` with code `framework.unhandled_exception`; raw exception text is diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/__init__.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/__init__.py index 5f17fed1e..4f36c1aa1 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/__init__.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/__init__.py @@ -111,6 +111,7 @@ FilePart, NativeTrajectory, TaskError, + TaskErrorType, TaskOutcome, TaskPart, TaskProgress, @@ -229,6 +230,7 @@ "SnapshotWriteObjects", "StagingStore", "TaskError", + "TaskErrorType", "TaskEventsTrajectoryRequest", "TaskObjectTrajectoryRequest", "TaskOutcome", diff --git a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/tasks/v1.py b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/tasks/v1.py index 920ec7226..980a46cca 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/tasks/v1.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/a2a_agent/tasks/v1.py @@ -14,7 +14,7 @@ from dataclasses import dataclass, field from enum import Enum from pathlib import Path -from typing import Annotated, Any, Generic, Literal, TypeAlias, TypeVar, Union +from typing import Annotated, Any, Generic, Literal, TypeAlias, TypeVar, Union, get_args from a2a.types import TaskStatusUpdateEvent from pydantic import BaseModel, ConfigDict, Field, model_validator @@ -265,15 +265,21 @@ class TaskOutcome(str, Enum): FAILED = "failed" +# model_key_budget and model_key_auth: the model endpoint refused the run's key, as out of budget or not +# authorized. Neither is the agent's fault, and a retry with the same key fails the same way. +TaskErrorType: TypeAlias = Literal["agent_error", "infra_error", "model_key_budget", "model_key_auth"] +TASK_ERROR_TYPES: frozenset[str] = frozenset(get_args(TaskErrorType)) + + @dataclass(frozen=True, slots=True, kw_only=True) class TaskError: code: str message: str - error_type: Literal["agent_error", "infra_error"] = "agent_error" + error_type: TaskErrorType = "agent_error" def __post_init__(self) -> None: - if self.error_type not in {"agent_error", "infra_error"}: - raise ValueError("error_type must be 'agent_error' or 'infra_error'") + if self.error_type not in TASK_ERROR_TYPES: + raise ValueError(f"error_type must be one of {sorted(TASK_ERROR_TYPES)}") class _TaskFailed(Exception): @@ -284,7 +290,7 @@ def __init__( code: str, message: str, *, - error_type: Literal["agent_error", "infra_error"] = "agent_error", + error_type: TaskErrorType = "agent_error", ) -> None: super().__init__(message) self.error = TaskError(code=code, message=message, error_type=error_type) @@ -423,7 +429,7 @@ def failure( code: str, message: str, *, - error_type: Literal["agent_error", "infra_error"] = "agent_error", + error_type: TaskErrorType = "agent_error", parts: Iterable[TaskPart] | None = None, ) -> "TaskResult": return ( @@ -455,7 +461,7 @@ def failed( code: str, message: str, *, - error_type: Literal["agent_error", "infra_error"] = "agent_error", + error_type: TaskErrorType = "agent_error", ) -> "TaskResultBuilder": self._outcome = TaskOutcome.FAILED self._error = TaskError(code=code, message=message, error_type=error_type) diff --git a/packages/agentenv-protocol/tests/test_a2a_agent.py b/packages/agentenv-protocol/tests/test_a2a_agent.py index 5e82d5501..5490adfb1 100644 --- a/packages/agentenv-protocol/tests/test_a2a_agent.py +++ b/packages/agentenv-protocol/tests/test_a2a_agent.py @@ -1148,6 +1148,8 @@ def test_task_result_builder_validates_and_factories_share_it() -> None: ) assert infra_failure.error is not None assert infra_failure.error.error_type == "infra_error" + for key_error in ("model_key_budget", "model_key_auth"): + assert TaskResult.failure("cli_error", "refused", error_type=key_error).error.error_type == key_error with pytest.raises(ValueError, match="error_type"): TaskResult.failure("cli_error", "failed", error_type="custom") # type: ignore[arg-type] with pytest.raises(TypeError, match="unexpected keyword argument 'details'"): diff --git a/src/agent_env/agents/AGENTS.md b/src/agent_env/agents/AGENTS.md index 0c1c0f558..9005d7420 100644 --- a/src/agent_env/agents/AGENTS.md +++ b/src/agent_env/agents/AGENTS.md @@ -53,7 +53,8 @@ terminal state, `completed`, `failed` or `canceled`. Your `run()` is one turn. `{error_type, error_code, error_message}`. A success needs at least one part, and `builder().failed(...)` adds no text part, so add one. - **Errors:** `agent_error`, the default, means the agent tried and failed; `infra_error` means the run couldn't - happen, and agent-env may retry it. An exception that escapes `run()` becomes an opaque `infra_error`, its text + happen, and agent-env may retry it; `model_key_budget` and `model_key_auth` mean the model endpoint refused the + run's key as out of budget or not authorized, which a retry with that key doesn't fix. An exception that escapes `run()` becomes an opaque `infra_error`, its text never sent. - **Context and sessions:** turns of one `context_id` run one at a time; different contexts run concurrently, and the validator relies on it, so don't serialize them. Return your runtime's conversation id as `session_ref` and the next diff --git a/src/agent_env/agents/agents/claude-code/a2a_server.py b/src/agent_env/agents/agents/claude-code/a2a_server.py index 7dc27625e..8cae4b165 100644 --- a/src/agent_env/agents/agents/claude-code/a2a_server.py +++ b/src/agent_env/agents/agents/claude-code/a2a_server.py @@ -508,7 +508,7 @@ async def _turn(self, request: TaskRequest[ClaudeCodeConfig], cfg: ClaudeCodeCon data["cli_attempts"] = result.attempts builder = TaskResult.builder() if result.error: - builder.failed("cli_error", result.error[:_ERROR_MESSAGE_LIMIT]) + builder.failed("cli_error", result.error[:_ERROR_MESSAGE_LIMIT], error_type=result.error_type or "agent_error") else: builder.succeeded() return ( diff --git a/src/agent_env/agents/agents/claude-code/claude_bridge.py b/src/agent_env/agents/agents/claude-code/claude_bridge.py index e8fcfc651..35be97726 100644 --- a/src/agent_env/agents/agents/claude-code/claude_bridge.py +++ b/src/agent_env/agents/agents/claude-code/claude_bridge.py @@ -18,6 +18,7 @@ from pathlib import Path from typing import Literal +from agentenv_protocol.a2a_agent import TaskErrorType from pydantic import BaseModel, ConfigDict, model_validator from kit import in_thread_to_its_end @@ -148,6 +149,8 @@ class ClaudeResult: structured_output: dict | None = None # How many `claude -p` runs the turn took; more than one means it was resumed. attempts: int = 1 + # The failure's type for agent-env, when the CLI's terminal `result` names a cause that isn't the agent's. + error_type: TaskErrorType | None = None def _attribution_headers(project_id: str | None, task_id: str | None) -> dict[str, str]: @@ -279,6 +282,25 @@ def _resumable_api_failure(result: ClaudeResult) -> str | None: return text if any(marker in text.lower() for marker in _RESUMABLE_API_ERROR_MARKERS) else None +# The CLI copies the endpoint's error body `type` into `api_error_type`: LiteLLM sends budget_exceeded and auth_error, +# Anthropic authentication_error and permission_error. +_KEY_AUTH_ERROR_TYPES = frozenset({"auth_error", "authentication_error", "permission_error"}) +_KEY_AUTH_STATUSES = frozenset({401, 403}) + + +def _key_error_type(trajectory: list[dict]) -> TaskErrorType | None: + """model_key_budget or model_key_auth when the endpoint refused the run's key, else None.""" + event = _terminal_result(trajectory) + if event is None or not event.get("is_error"): + return None + kind = event.get("api_error_type") + if kind == "budget_exceeded": + return "model_key_budget" + if kind in _KEY_AUTH_ERROR_TYPES or _status(event.get("api_error_status")) in _KEY_AUTH_STATUSES: + return "model_key_auth" + return None + + def _endpoint_hint(trajectory: list[dict]) -> str | None: """What a 404 from the model endpoint likely means.""" event = _terminal_result(trajectory) @@ -756,7 +778,9 @@ def on_line(line: str) -> None: err: str | None = None detail: str | None = None + error_type: TaskErrorType | None = None if proc.returncode != 0: + error_type = _key_error_type(trajectory) err = (f"claude CLI was killed by signal {-proc.returncode}" if proc.returncode < 0 else f"claude CLI exited with code {proc.returncode}") detail = _cli_error_text(trajectory) or _stderr_tail(stderr.decode(errors="replace")) @@ -770,7 +794,7 @@ def on_line(line: str) -> None: return ClaudeResult(response=final_response or "", tool_call_count=tool_count, session_id=cli_session_id or session_id, trajectory=trajectory, error=err, - error_detail=detail, structured_output=structured_output) + error_detail=detail, structured_output=structured_output, error_type=error_type) _TAGS_HEADER = "x-litellm-tags" diff --git a/src/agent_env/agents/agents/claude-code/requirements.txt b/src/agent_env/agents/agents/claude-code/requirements.txt index 464574902..65738e129 100644 --- a/src/agent_env/agents/agents/claude-code/requirements.txt +++ b/src/agent_env/agents/agents/claude-code/requirements.txt @@ -1,4 +1,4 @@ # The image installs the newest agentenv-framework-protocol at or above this floor; a release doesn't pin it, so the -# image is built again only when this folder changes. The floor is the first that serves peer-agents/v1's tools, which -# the agent advertises without handlers of its own; raise it when the agent needs a newer protocol. -agentenv-framework-protocol[agent]>=0.1.351 +# image is built again only when this folder changes. The floor is the first whose TaskError takes model_key_budget and +# model_key_auth; raise it when the agent needs a newer protocol. +agentenv-framework-protocol[agent]>=0.1.551 diff --git a/tst/unit/a2a_agent/protocol_test.py b/tst/unit/a2a_agent/protocol_test.py index 4c2d803bd..9c515140d 100644 --- a/tst/unit/a2a_agent/protocol_test.py +++ b/tst/unit/a2a_agent/protocol_test.py @@ -115,17 +115,18 @@ def test_sdk_success_round_trips_through_platform_parser(): assert parsed.tool_call_count == 3 -def test_sdk_failure_round_trips_through_platform_parser(): +@pytest.mark.parametrize("error_type", ["infra_error", "model_key_budget", "model_key_auth"]) +def test_sdk_failure_round_trips_through_platform_parser(error_type): result = TaskResult.failure( "provider_rate_limited", "The provider is temporarily unavailable", - error_type="infra_error", + error_type=error_type, ) parsed = TerminalResponse.from_message(_sdk_terminal_message(result)) assert parsed.response_text == "The provider is temporarily unavailable" - assert parsed.error_type == "infra_error" + assert parsed.error_type == error_type assert parsed.error_code == "provider_rate_limited" assert parsed.error_message == "The provider is temporarily unavailable" diff --git a/tst/unit/agents/claude_code/test_image_contract.py b/tst/unit/agents/claude_code/test_image_contract.py index 6445b09d6..843feb5f8 100644 --- a/tst/unit/agents/claude_code/test_image_contract.py +++ b/tst/unit/agents/claude_code/test_image_contract.py @@ -23,7 +23,7 @@ def test_the_image_installs_the_protocol_alone_from_its_requirements(): requirements = [line for line in (AGENT_DIR / "requirements.txt").read_text().splitlines() if line and not line.startswith("#")] - assert requirements == ["agentenv-framework-protocol[agent]>=0.1.351"] + assert requirements == ["agentenv-framework-protocol[agent]>=0.1.551"] def test_the_image_gives_the_model_node_alone_from_a_pinned_image(): diff --git a/tst/unit/agents/claude_code/test_model_key_errors.py b/tst/unit/agents/claude_code/test_model_key_errors.py new file mode 100644 index 000000000..aba958eb3 --- /dev/null +++ b/tst/unit/agents/claude_code/test_model_key_errors.py @@ -0,0 +1,57 @@ +"""A turn the model endpoint refused for the run's key fails as model_key_budget or model_key_auth, not agent_error. + +The CLI exits 1 and copies the endpoint's status and error body `type` into its terminal `result` event. The events +below are trimmed from captures of CLI 2.1.325 against a local endpoint answering with each body. +""" +import asyncio +import json + +import pytest + +from tst.unit.agents.claude_code.agent import claude_bridge +from tst.unit.agents.claude_code.support import FakeCLI, send, terminal + + +def _result(status: int, kind: str, text: str, terminal_reason: str = "api_error") -> dict: + return { + "type": "result", "subtype": "success", "is_error": True, "session_id": "sess-1", "num_turns": 1, + "terminal_reason": terminal_reason, "api_error_status": status, "api_error_type": kind, "result": text, + } + + +@pytest.fixture +def cli(monkeypatch): + """Every `claude -p` prints the init event and ``event``, then exits 1.""" + monkeypatch.setattr(claude_bridge, "_RETRY_BACKOFF_S", (0, 0)) + + def _script(event: dict) -> None: + stdout = "".join(json.dumps(e) + "\n" for e in ( + {"type": "system", "subtype": "init", "session_id": "sess-1"}, event)).encode() + + async def _exec(*cmd, **kwargs): + return FakeCLI(stdout, 1) + + monkeypatch.setattr(asyncio, "create_subprocess_exec", _exec) + + return _script + + +@pytest.mark.parametrize(("event", "error_type"), [ + (_result(400, "budget_exceeded", "API Error: 400 Budget has been exceeded! Current cost: 10.1, Max budget: 10.0"), + "model_key_budget"), + (_result(401, "auth_error", + "Failed to authenticate. API Error: 401 Authentication Error, Invalid proxy server token passed."), + "model_key_auth"), + (_result(401, "authentication_error", "Invalid API key · Fix external API key"), "model_key_auth"), + (_result(403, "permission_error", "Failed to authenticate. API Error: 403 forbidden"), "model_key_auth"), + (_result(429, "rate_limit_error", "API Error: Request rejected (429) · slow down"), "agent_error"), + (_result(400, "invalid_request_error", "Prompt is too long", terminal_reason="prompt_too_long"), "agent_error"), +], ids=["litellm-budget", "litellm-auth", "anthropic-auth", "anthropic-permission", "rate-limit", "prompt-too-long"]) +def test_a_failed_turn_says_whether_the_endpoint_refused_the_key(cli, client, event, error_type): + cli(event) + + state, _, data = terminal(send(client, "hello")) + + assert state == "failed" + assert (data[-1]["error_type"], data[-1]["error_code"]) == (error_type, "cli_error") + assert event["result"] in data[-1]["error_message"]