diff --git a/dokimi/changelog.d/20261114_120000_fu_run-wide-credentials.md b/dokimi/changelog.d/20261114_120000_fu_run-wide-credentials.md new file mode 100644 index 000000000..400fc64f3 --- /dev/null +++ b/dokimi/changelog.d/20261114_120000_fu_run-wide-credentials.md @@ -0,0 +1,8 @@ + + +### Fixed + +- **A missing-AWS-credentials error under litellm's catch-all classes is permanent** — "Unable to locate credentials", "Unable to locate authorization token", partial credentials, a missing config profile and an expired SSO session raised as `APIConnectionError`, `APIError` or `InternalServerError` were retried three times and, in a routing run, written as failed rows call after call. `is_retryable` now returns False for them, so the retry wrappers fail at the first attempt and `model routing` stops with `permanent_error`. diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index b06951b3d..d3df36e33 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -827,7 +827,7 @@ so count the queries first, or pass `--max-usd` to stop the run at a spend limit A stored response in `DOKIMI_RESPONSE_CACHE` is replayed without a call, so a rerun can make fewer calls than the printed count. -**A permanent provider error that every call would meet stops a routing run the way the cap does.** An authentication or permission error, an unknown model ID (`not found`), a parameter the provider does not take (every call sends the same ones) or missing credentials (with `claude-cli`: not logged in, a usage limit, no `claude` binary) would fail every later call alike, so the run stops at that call, keeps the rows made before it and writes the partial result below, with `"reason": "permanent_error"` and the error text as the message (`--max-usd` is not involved; the exit line gives `calls_done` of `calls_planned`). Any other failure is one failed row and the run goes on: a refusal, a request one query makes too long, an unparseable reply, an error the client does not recognise, a transient error that ran out of retries. +**A permanent provider error that every call would meet stops a routing run the way the cap does.** An authentication or permission error, an unknown model ID (`not found`), a parameter the provider does not take (every call sends the same ones) or missing credentials (with `claude-cli`: not logged in, a usage limit, no `claude` binary; with AWS: "Unable to locate credentials", a missing config profile or an expired SSO session, whichever class litellm wraps it in) would fail every later call alike, so the run stops at that call, keeps the rows made before it and writes the partial result below, with `"reason": "permanent_error"` and the error text as the message (`--max-usd` is not involved; the exit line gives `calls_done` of `calls_planned`). Any other failure is one failed row and the run goes on: a refusal, a request one query makes too long, an unparseable reply, an error the client does not recognise, a transient error that ran out of retries. **A routing run stopped by `--max-usd` keeps what it paid for.** `skill routing --model`, single artifact or `catalog routing`, writes the result of the calls made before the cap (or a call with no diff --git a/dokimi/dokimi/llm.py b/dokimi/dokimi/llm.py index f0f9c5f68..f88854b21 100644 --- a/dokimi/dokimi/llm.py +++ b/dokimi/dokimi/llm.py @@ -274,11 +274,11 @@ def unsupported_sampling_params( "unsupportedparamserror", "contentpolicyviolationerror", }) -# So are these messages, which litellm raises under its catch-alls: no key, project or SDK. -_NON_RETRYABLE_MESSAGES = ("missing credentials", "api_key client option must be set", - "missing gemini api key", "could not resolve credentials", - "could not resolve project_id", "default credentials were not found", - "google cloud sdk not found") +# So are these messages, which litellm raises under its catch-alls: no key, project, SDK or AWS credentials. +_NON_RETRYABLE_MESSAGES = ("missing credentials", "api_key client option must be set", "missing gemini api key", + "could not resolve credentials", "could not resolve project_id", "google cloud sdk not found", + "default credentials were not found", "unable to locate credentials", "partial credentials found in", + "unable to locate authorization token", "the config profile (", "sso session associated with this profile") def is_retryable(exc: BaseException) -> bool: diff --git a/dokimi/dokimi/novelty/routing_stop.py b/dokimi/dokimi/novelty/routing_stop.py index 53f4e378d..fec78ef2b 100644 --- a/dokimi/dokimi/novelty/routing_stop.py +++ b/dokimi/dokimi/novelty/routing_stop.py @@ -10,7 +10,6 @@ # call sends. Any other error (a context-length rejection, a refusal, an odd reply) may be one query's. _RUN_WIDE_ERRORS = frozenset({"authenticationerror", "permissiondeniederror", "notfounderror", "unsupportedparamserror"}) -_NO_CREDENTIALS = (*_NON_RETRYABLE_MESSAGES, "unable to locate credentials") def is_run_wide(exc: Exception) -> bool: @@ -20,4 +19,4 @@ def is_run_wide(exc: Exception) -> bool: if isinstance(run_wide := getattr(exc, "run_wide", None), bool): # set by an error that knows (CliCallError) return run_wide return type(exc).__name__.lower() in _RUN_WIDE_ERRORS or any( - marker in str(exc).lower() for marker in _NO_CREDENTIALS) + marker in str(exc).lower() for marker in _NON_RETRYABLE_MESSAGES) diff --git a/dokimi/tests/test_llm_retry_credentials.py b/dokimi/tests/test_llm_retry_credentials.py index f51be28ac..e43e78cc9 100644 --- a/dokimi/tests/test_llm_retry_credentials.py +++ b/dokimi/tests/test_llm_retry_credentials.py @@ -76,10 +76,21 @@ def test_a_missing_key_fails_on_the_first_attempt(no_credentials, sleeps, provid "Google Cloud SDK not found. Install it with: pip install 'litellm[google]' or pip " "install google-cloud-aiplatform", "vertex_ai", "gemini-2.5-pro"), ] +# botocore's own texts, which litellm's Bedrock mapping leaves as a bare APIConnectionError +# (only "Unable to locate credentials" is mapped, to AuthenticationError, for Bedrock itself). +MISSING_CREDENTIALS += [ + litellm.APIConnectionError(text, "bedrock", "claude") + for text in ("Unable to locate credentials", "Unable to locate authorization token", + "Partial credentials found in env, missing: aws_secret_access_key", + "The config profile (prod) could not be found", + "The SSO session associated with this profile has expired or is otherwise " + "invalid. To refresh this SSO session run aws sso login with the " + "corresponding profile.")] #: The caller each error comes back to: litellm's provider name -> (model, dokimi provider). ROUTES = {"openai": ("gpt-4o", "openai"), "azure": ("gpt-4o", "azure"), + "bedrock": ("claude", "bedrock"), "vertex_ai": ("gemini-2.5-pro", "google-vertex")} diff --git a/dokimi/tests/test_novelty_routing_estimate_and_stop.py b/dokimi/tests/test_novelty_routing_estimate_and_stop.py index b4fada490..4dd92a915 100644 --- a/dokimi/tests/test_novelty_routing_estimate_and_stop.py +++ b/dokimi/tests/test_novelty_routing_estimate_and_stop.py @@ -109,6 +109,26 @@ def behaviour(n): assert result["partial"]["reason"] == "permanent_error" +class APIConnectionError(Exception): + """Named as litellm's catch-all, which is_retryable reads by class name.""" + + +@pytest.mark.parametrize("text", ["Unable to locate credentials", + "The config profile (prod) could not be found"]) +def test_a_credentials_error_under_a_catch_all_class_stops_the_run_at_the_first_call(text): + stub = Stub(lambda n: (_ for _ in ()).throw(APIConnectionError(text))) + result = model_routing(FIELD, QUERIES, client=stub, repeats=3) + assert stub.calls == 1 and result["per_query"] == [] + assert result["partial"]["reason"] == "permanent_error" + assert result["cap_stop"] == f"permanent error: {text}" + + +def test_a_connection_reset_under_the_catch_all_class_is_still_a_row(): + stub = Stub(lambda n: (_ for _ in ()).throw(APIConnectionError("connection reset"))) + result = model_routing(FIELD, QUERIES, client=stub) + assert stub.calls == 3 and result["n_failed"] == 3 and "partial" not in result + + def test_all_vs_all_stops_at_a_run_wide_error_and_counts_the_calls_of_the_run(): def behaviour(n): if n == 4: