From 779ad016f9e8b59c52dc628644db9b34ed41c4a1 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sun, 15 Nov 2026 08:20:48 +0000 Subject: [PATCH] fix(task): stop retrying a model key the endpoint refuses as out of budget or unauthorized ModelKeyUnusable(cause, where) is raised by the rubrics judge (direct LiteLLM and A2A) and by prompt_agent on model_key_budget / model_key_auth. The judge loop makes one call, and a step's retry_config does not rerun it. --- src/agent_env/task/task.py | 10 +- src/agent_env/task_step/context.py | 17 ++- .../task_step/task_steps/prompt_agent.py | 11 +- .../task_steps/verifiers/rubrics_verifier.py | 26 +++++ tst/unit/task_step/retry_test.py | 28 ++++- .../task_step/test_prompt_agent_model_key.py | 65 ++++++++++++ .../verifiers/test_model_key_unusable.py | 100 ++++++++++++++++++ 7 files changed, 252 insertions(+), 5 deletions(-) create mode 100644 tst/unit/task_step/test_prompt_agent_model_key.py create mode 100644 tst/unit/task_step/verifiers/test_model_key_unusable.py diff --git a/src/agent_env/task/task.py b/src/agent_env/task/task.py index df78a58f4..6e5e2468a 100644 --- a/src/agent_env/task/task.py +++ b/src/agent_env/task/task.py @@ -25,7 +25,13 @@ from agent_env.store.routing import run_scope from agent_env.task_step.preflight import pinned from agent_env.task.teardown import env_record_ids -from agent_env.task_step.context import ENTITY_IDS_KEY, TaskStepContext, VerifierNotPassingError, regraft_redacted_keys +from agent_env.task_step.context import ( + ENTITY_IDS_KEY, + ModelKeyUnusable, + TaskStepContext, + VerifierNotPassingError, + regraft_redacted_keys, +) from agent_env.task_step.context_ops import ContextUpdateOps, build_context_update_ops from agent_env.task_step.task_step import RetryConfig, TaskStep @@ -753,7 +759,7 @@ async def _drain_in_flight() -> set[str]: continue retry_cfg = getattr(step_obj, "retry_config", None) if (retry_cfg is not None and retry_counts.get(step_id, 0) < retry_cfg.max_retries - and not isinstance(exc, VerifierNotPassingError)): + and not isinstance(exc, (VerifierNotPassingError, ModelKeyUnusable))): retryable.append((step_obj, retry_cfg, exc)) continue if not getattr(step_obj, "fail_task_on_error", True): diff --git a/src/agent_env/task_step/context.py b/src/agent_env/task_step/context.py index 3343042d5..6f93afbf3 100644 --- a/src/agent_env/task_step/context.py +++ b/src/agent_env/task_step/context.py @@ -7,7 +7,7 @@ import re from collections.abc import Callable from dataclasses import dataclass, field -from typing import Any +from typing import Any, Literal from agent_env.config.loader import ENV_REF_PREFIX, SECRET_REF_PREFIX from agent_env.env.env import DeployedEnv @@ -55,6 +55,21 @@ class StepContextError(RuntimeError): way, so a runner should not retry it.""" +ModelKeyCause = Literal["budget", "auth"] + +MODEL_KEY_ERROR_TYPES: dict[str, ModelKeyCause] = {"model_key_budget": "budget", "model_key_auth": "auth"} + + +class ModelKeyUnusable(RuntimeError): + """The model endpoint refused the run's key as out of budget or not authorized. The same key fails the same + way on every attempt, so a runner should not retry it. Carries no key material.""" + + def __init__(self, cause: ModelKeyCause, where: str) -> None: + super().__init__(f"The model key is unusable ({cause}) for the {where}; retrying with it cannot succeed") + self.cause = cause + self.where = where + + class VerifierNotPassingError(RuntimeError): """A verifier set to fail its task when its verdict isn't passing recorded one that isn't. Grading the same run again could pass it, so a runner should not retry it.""" diff --git a/src/agent_env/task_step/task_steps/prompt_agent.py b/src/agent_env/task_step/task_steps/prompt_agent.py index 8098f1a4b..599d7127a 100644 --- a/src/agent_env/task_step/task_steps/prompt_agent.py +++ b/src/agent_env/task_step/task_steps/prompt_agent.py @@ -35,7 +35,14 @@ ) from agent_env.store import DuplicateKeyError, get_config from agent_env.store.object_store import ObjectStore -from agent_env.task_step.context import DeployedAgent, PromptResponse, StepContextError, TaskStepContext +from agent_env.task_step.context import ( + MODEL_KEY_ERROR_TYPES, + DeployedAgent, + ModelKeyUnusable, + PromptResponse, + StepContextError, + TaskStepContext, +) from agent_env.entity.refs import EntityRef from agent_env.entity.scopes import FieldScope from agent_env.task_step.task_step import RetryConfig, TaskStep, TaskStepDependency @@ -826,6 +833,8 @@ def record_turn() -> None: )) if final_state == TaskState.failed: + if final_terminal.error_type in MODEL_KEY_ERROR_TYPES: + raise ModelKeyUnusable(MODEL_KEY_ERROR_TYPES[final_terminal.error_type], "agent") detail_parts = [] if final_terminal.error_type: detail_parts.append(f"error_type={final_terminal.error_type}") diff --git a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py index a7796c22c..0859217b2 100644 --- a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py @@ -22,6 +22,9 @@ from agent_env.config.model import ModelParam from agent_env.providers.sandbox_providers.sandbox import into_container from agent_env.task_step.context import ( + MODEL_KEY_ERROR_TYPES, + ModelKeyCause, + ModelKeyUnusable, PromptResponse, StepContextError, TaskStepContext, @@ -271,6 +274,21 @@ def _validated_run_context_parts(parts: Optional[list[str]]) -> Optional[list[st ) +_BUDGET_MARKERS = ("budget has been exceeded", "exceededbudget", "budget_exceeded") + + +def _model_key_refusal(exc: Exception) -> Optional[ModelKeyCause]: + """Whether a LiteLLM error says the endpoint refused the key itself. Parsed here, once, so the retry loop + branches on a cause and never on message text. Any other failure, including a 429 rate limit, returns None.""" + status = getattr(exc, "status_code", None) + text = str(exc).lower() + if type(exc).__name__ == "BudgetExceededError" or any(marker in text for marker in _BUDGET_MARKERS): + return "budget" + if status == 401 or type(exc).__name__ == "AuthenticationError": + return "auth" + return None + + def _looks_like_response_format_error(exc: Exception) -> bool: msg = str(exc).lower() return any(s in msg for s in _SCHEMA_TERMS) and any(u in msg for u in _UNSUPPORTED_TERMS) @@ -1252,6 +1270,12 @@ async def _prompt_llm_judge( return {"response": content, "trajectory_object_url": None} except Exception as exc: last_exc = exc + if (cause := _model_key_refusal(exc)) is not None: + logger.error( + f"LLM judge for verifier '{self.verifier_id}' refused its model key " + f"({cause}, {type(exc).__name__}); not retrying" + ) + raise ModelKeyUnusable(cause, "judge") from None if image_blocks and use_response_format and _looks_like_response_format_error(exc): logger.warning( f"Multimodal judge call with json_schema failed for verifier " @@ -1334,6 +1358,8 @@ async def _invoke_judge_a2a( status_msg = (result.get("status") or {}).get("message") or {} tr = protocol.TerminalResponse.from_message(status_msg) if state == "failed": + if tr.error_type in MODEL_KEY_ERROR_TYPES: + raise ModelKeyUnusable(MODEL_KEY_ERROR_TYPES[tr.error_type], "judge") sandbox_id = getattr(judge_agent, "sandbox_id", None) logger.error( "Judge A2A task %s failed (verifier=%s sandbox=%s error_type=%s exception=%s). " diff --git a/tst/unit/task_step/retry_test.py b/tst/unit/task_step/retry_test.py index 15be8c585..07b600ec0 100644 --- a/tst/unit/task_step/retry_test.py +++ b/tst/unit/task_step/retry_test.py @@ -37,7 +37,13 @@ set_task_instance_store, ) from agent_env.task.task import Task, _rolled_back_sandboxes -from agent_env.task_step.context import DeployedAgent, DeployedSandbox, PromptResponse, TaskStepContext +from agent_env.task_step.context import ( + DeployedAgent, + DeployedSandbox, + ModelKeyUnusable, + PromptResponse, + TaskStepContext, +) from agent_env.task_step.context_ops import ContextUpdateOps, build_context_update_ops from agent_env.task_step.task_step import RetryConfig from agent_env.config import set_document_store @@ -1110,3 +1116,23 @@ def test_validate_dag_accepts_ancestor_resume_point(): c = _FlakyPrompt("c", retry_config=RetryConfig(retry_from_step_id="a"), depends_on=["a"]) # Should not raise. Task(id="t", version=1, steps=[a, c]) + + +class _DeadKeyPrompt(_FlakyPrompt): + async def execute(self, context: TaskStepContext) -> TaskStepContext: + self.attempts += 1 + raise ModelKeyUnusable("budget", "agent") + + +@pytest.mark.asyncio +async def test_a_step_whose_model_key_is_unusable_is_not_retried(store): + deploy = _FakeDeployEnv("s0") + dead = _DeadKeyPrompt("s1", retry_config=RetryConfig(retry_from_step_id="s0", max_retries=3)) + task = Task(id="t", version=1, steps=[deploy, dead]) + + ctx = TaskStepContext() + with pytest.raises(ModelKeyUnusable): + await task.run(context=ctx) + + assert dead.attempts == 1 + assert store.get(ctx.instance_id).status == "failed" diff --git a/tst/unit/task_step/test_prompt_agent_model_key.py b/tst/unit/task_step/test_prompt_agent_model_key.py new file mode 100644 index 000000000..77cb5e635 --- /dev/null +++ b/tst/unit/task_step/test_prompt_agent_model_key.py @@ -0,0 +1,65 @@ +"""prompt_agent fails the step with ModelKeyUnusable when the agent turn was refused for its model key.""" + +from unittest.mock import MagicMock + +import pytest +from a2a.types import TaskState + +from agent_env.task_step.context import DeployedAgent, ModelKeyUnusable, TaskStepContext +from agent_env.task_step.task_steps import prompt_agent as pa_mod +from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep + + +def _failed(error_type): + data = {"error_type": error_type, "error_message": "refused"} + return {"status": {"state": TaskState.failed, "message": {"parts": [{"kind": "data", "data": data}]}}} + + +def _wire(monkeypatch, result): + cfg = MagicMock() + cfg.get_model_params.return_value = {} + monkeypatch.setattr(pa_mod, "get_config", lambda: cfg) + for fn in ("create_conversation", "add_a2a_task", "complete_a2a_task", "mark_closed", "get_conversation"): + monkeypatch.setattr(pa_mod.conversation_store, fn, lambda *a, **kw: None) + + async def send(url, parts, message_id, context_id, timeout): + return f"{url}#task", None + + async def poll(url, task_id, timeout, interval, *, sandbox_id=None): + return result + + monkeypatch.setattr(pa_mod.protocol, "send_a2a_message", send) + monkeypatch.setattr(pa_mod.protocol, "poll_a2a_task", poll) + + +def _context(): + ctx = TaskStepContext(instance_id="ti-1") + ctx.deployed_agents.append(DeployedAgent( + agent_name="solver", api_url="http://solver", a2a_url="http://solver", a2a_card={}, sandbox_id="sb-solver", + )) + return ctx + + +@pytest.mark.asyncio +@pytest.mark.parametrize("error_type, cause", [("model_key_budget", "budget"), ("model_key_auth", "auth")]) +async def test_a_turn_refused_for_its_model_key_raises_model_key_unusable(monkeypatch, error_type, cause): + _wire(monkeypatch, _failed(error_type)) + step = PromptAgentTaskStep(id="solve", version=None, prompt="hi", agent_name="solver") + ctx = _context() + + with pytest.raises(ModelKeyUnusable) as raised: + await step.execute(ctx) + + assert raised.value.cause == cause + assert ctx.prompt_responses[-1].error_type == error_type + + +@pytest.mark.asyncio +async def test_any_other_failed_turn_still_raises_the_plain_error(monkeypatch): + _wire(monkeypatch, _failed("agent_error")) + step = PromptAgentTaskStep(id="solve", version=None, prompt="hi", agent_name="solver") + + with pytest.raises(RuntimeError, match="A2A task failed") as raised: + await step.execute(_context()) + + assert not isinstance(raised.value, ModelKeyUnusable) diff --git a/tst/unit/task_step/verifiers/test_model_key_unusable.py b/tst/unit/task_step/verifiers/test_model_key_unusable.py new file mode 100644 index 000000000..60b382d8c --- /dev/null +++ b/tst/unit/task_step/verifiers/test_model_key_unusable.py @@ -0,0 +1,100 @@ +"""A model key the endpoint refuses fails the judge on its first call; a transient failure still retries.""" +from __future__ import annotations + +import json + +import litellm +import pytest + +from agent_env.a2a_agent import protocol +from agent_env.config.model import ModelConfig +from agent_env.store import LocalSecretStore +from agent_env.task_step.context import ModelKeyUnusable, TaskStepContext +from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep + +_VERDICT = {"results": [{"criterion_id": "c1", "passed": True}]} +_BUDGET = litellm.BadRequestError( + message="Budget has been exceeded! Current cost: 51.0, Max budget: 50.0", model="m", llm_provider="openai", +) +_AUTH = litellm.AuthenticationError( + message="Incorrect API key provided: sk-secret123", model="m", llm_provider="openai", +) + + +@pytest.fixture +def judge(monkeypatch): + import agent_env.config as cfg_mod + + config = cfg_mod.Config() + config.set_secret_store(LocalSecretStore(values={"litellm_api_key": "k"}, use_env=False)) + monkeypatch.setattr(config, "_model_cfg", ModelConfig()) + monkeypatch.setattr(cfg_mod, "get_config", lambda: config) + monkeypatch.delenv("LITELLM_BASE_URL", raising=False) + monkeypatch.delenv("LITELLM_API_KEY", raising=False) + return RubricsVerifierTaskStep( + id="v", version=1, criteria=[], prompt_id="p", use_agent_judge=False, + default_model_api_base="https://judge.invalid/v1", verifier_id="vt", + ) + + +def _answer_with(monkeypatch, *answers): + real, calls, pending = litellm.acompletion, [], list(answers) + + async def acompletion(**kwargs): + calls.append(kwargs) + return await real(**kwargs, mock_response=pending.pop(0)) + + monkeypatch.setattr(litellm, "acompletion", acompletion) + return calls + + +@pytest.mark.asyncio +@pytest.mark.parametrize("refusal, cause", [(_BUDGET, "budget"), (_AUTH, "auth")], ids=["budget", "auth"]) +async def test_a_judge_whose_key_is_refused_is_called_once_and_raises_model_key_unusable( + judge, monkeypatch, refusal, cause +): + calls = _answer_with(monkeypatch, *[refusal] * judge.DEFAULT_MAX_RETRIES) + + with pytest.raises(ModelKeyUnusable) as raised: + await judge._prompt_llm_judge("grade it", model="openai/gpt-4o", context=TaskStepContext()) + + assert len(calls) == 1 + assert raised.value.cause == cause + assert "sk-secret123" not in str(raised.value) + assert raised.value.__cause__ is None + + +@pytest.mark.asyncio +async def test_a_transient_judge_error_is_still_retried(judge, monkeypatch): + outage = litellm.ServiceUnavailableError(message="upstream down", model="m", llm_provider="openai") + rate_limit = litellm.RateLimitError(message="slow down", model="m", llm_provider="openai") + calls = _answer_with(monkeypatch, outage, rate_limit, json.dumps(_VERDICT)) + + out = await judge._prompt_llm_judge("grade it", model="openai/gpt-4o", context=TaskStepContext()) + + assert json.loads(out["response"]) == _VERDICT + assert len(calls) == 3 + + +@pytest.mark.asyncio +async def test_an_agent_judge_task_failed_for_its_key_raises_model_key_unusable(monkeypatch): + async def send(*args, **kwargs): + return "task-1", "ctx-1" + + async def poll(*args, **kwargs): + return {"status": {"state": "failed", "message": {"parts": [ + {"kind": "data", "data": {"error_type": "model_key_budget", "error_message": "Budget has been exceeded"}}, + ]}}} + + monkeypatch.setattr(protocol, "send_a2a_message", send) + monkeypatch.setattr(protocol, "poll_a2a_task", poll) + verifier = RubricsVerifierTaskStep( + id="v", version=1, criteria=[], prompt_id="p", use_agent_judge=True, verifier_id="vt", + ) + + with pytest.raises(ModelKeyUnusable) as raised: + await verifier._invoke_judge_a2a( + eval_prompt="grade it", judge_a2a_url="http://judge.example", judge_agent_card={}, judge_agent=None, + ) + + assert raised.value.cause == "budget"