diff --git a/.pr/design.html b/.pr/design.html new file mode 100644 index 00000000000..8f0437cb972 --- /dev/null +++ b/.pr/design.html @@ -0,0 +1,147 @@ + + + + + +Fallbacks keep the caller's accounting session + + + +
+ +
+
+
Stacked fix for SDK PR #6690
+

Fallbacks keep the caller's accounting session.

+

A fallback changes the model serving a request. Its tokens, cost and telemetry still belong to the agent, condenser or ask-agent role that made the request.

+

The session refactor already records a fallback response in a conversation ledger. Its old post-call merge records the same usage again, and a profile's usage_id="default" can route auxiliary calls into the agent ledger. This fix gives a temporary fallback LLM the caller's existing session references and removes the second accounting path for SDK calls.

+

Before: fallback_strategy.py:99 · After: fallback_strategy.py:106

+
★ One successful provider request writes once to the caller's ledger. The individual records and completion logs identify the model that actually answered.
+
+
+

1. Bind the request before calling the fallback

+
+ +An ask-agent fallback before and after the fix +Before, the fallback profile selects the default role and its metrics are merged into ask-agent, counting one response twice. After, a temporary fallback object is explicitly bound to ask-agent's session, which records one hundred tokens once. + + + + + + + + + +Before · profile role wins + + +try_fallback() → call_fn(fb) +Saved fallback keeps usage_id="default" +fallback_strategy.py:99 + + + +session_for() → sessions["default"] +Agent ledger receives +100 tokens +llm.py:975 + + + +primary_metrics.merge(metrics_diff) +Ask-agent ledger receives another +100 +fallback_strategy.py:110 +100 provider tokens → 200 recorded +Same-role fallback also doubles its single ledger. + + + +After · caller role stays + + +fb.model_copy(usage_id=caller.role) +Temporary copy disables nested fallbacks +fallback_strategy.py:106 + + + +Reuse _session + _registry_session +Existing role and aggregate ownership +fallback_strategy.py:117 + + + +session_for() → caller's session +Ask-agent records +100; its callbacks fire +llm.py:975 +100 provider tokens → 100 recorded +SDK calls skip the post-call metrics merge. + + +
Fig. 1 · The fallback profile supplies model configuration; the caller supplies accounting ownership. Orange arrows mark the changed path.
+
+
+

Context survives async fallback

asyncio.to_thread() carries the scoped call context into async fallback work. Explicit contexts already travel with the callback, so both paths preserve the conversation identity and prompt-cache key.

llm.py:1122 · _ahandle_error()
+

Records keep the answering model

Telemetry passes its current model into token, cost and latency entries. Session totals stay with the role while individual entries identify the fallback model.

telemetry.py:227 · on_response()
+
+
+
+

2. Preserve existing call signatures

+

The original positional arguments remain valid. The fallback reuses existing session references; serialized LLM and profile shapes are unchanged. fallback_strategy.py:117

+
+

Before

try_fallback(model, error, metrics, call_fn)
+metrics.add_cost(value)
+metrics.add_response_latency(value, response_id)
+metrics.add_token_usage(...)
+

After · optional keyword additions

try_fallback(model, error, metrics, call_fn,
+             *, primary=None)
+metrics.add_cost(value, *, model_name=None)
+metrics.add_response_latency(..., model_name=None)
+metrics.add_token_usage(..., model_name=None)
+
+

fallback_strategy.py:73 · metrics.py:189

+

The original four-argument try_fallback() API keeps its metrics-diff merge. SDK calls pass primary=self, reuse the caller's sessions and callbacks, and skip that merge. The fallback example's accounting description is updated to match. fallback_strategy.py:117

+
+
+

3. Five operations, five corrected ledgers

+

The same script ran on PR #6690's 0a68ab94 and on this fix. Each operation sent one request that received HTTP 503, then one successful fallback request reporting 100 prompt tokens.

+ + + + + + + + + +
Public operationBefore: token increaseAfter: token increase
run(), first turndefault +200default +100
run(), second turndefault +200default +100
run(), third turndefault +200default +100
ask_agent()default +100; ask-agent-llm +100ask-agent-llm +100
condense()default +100; condenser +100condenser +100
+

Evidence: fallback-session-evidence.json · fallback-session-repro.py

+

macOS · Python 3.13.11 · real settings construction, profile save/load, conversation lifecycle and HTTP transport. The provider runs on loopback and supplies deterministic responses. These observations establish token totals and role attribution; they do not measure callback counts, actual provider billing or remote-service behavior.

+
Reproduce with the checkout's dependencies
OH_PERSISTENCE_DIR=/tmp/fallback-repro-state \
+OPENHANDS_SUPPRESS_BANNER=1 LITELLM_LOCAL_MODEL_COST_MAP=True \
+uv run python .pr/fallback-session-repro.py --assert-fixed

The final flag makes incorrect accounting fail with exit code 1. The script starts and shuts down its own local HTTP server and uses temporary profile and conversation directories.

+
+
+

4. Regression checks cover the other boundaries

+

1,791 tests passed; 2 expected failures. The LLM, conversation-session, model-switch and agent-server event-service checks passed. Two credential-path checks ran separately without the temporary persistence-directory override. The two fallback regression files account for 58 passing cases.

+

The focused regression file checks sync and async Chat Completions and Responses calls, explicit and scoped contexts, standalone use, a colliding default role, and one LLM shared across registries. Two fallback calls followed by a recovered primary call verify cumulative tokens, cost, callback counts, returned metrics and actual-model labels. A separate case exercises the original direct try_fallback() API.

+

test_fallback_preserves_caller_accounting:41 · test_direct_fallback_preserves_supplied_metrics:230

+
Focused regression command
uv run pytest tests/sdk/llm/test_llm_fallback_sessions.py \
+  tests/sdk/llm/test_llm_fallback.py tests/sdk/llm/test_llm_session.py -q

These regression tests replace provider responses at the LiteLLM boundary. The loopback reproduction above separately exercises real HTTP transport.

+
+ +
+
+ + diff --git a/.pr/fallback-session-evidence.json b/.pr/fallback-session-evidence.json new file mode 100644 index 00000000000..65bffa51193 --- /dev/null +++ b/.pr/fallback-session-evidence.json @@ -0,0 +1,227 @@ +{ + "base_commit": "0a68ab94cb3f447254bdc68ce778048ce517be22", + "fixed_source_sha256": { + "openhands-sdk/openhands/sdk/llm/fallback_strategy.py": "c7f268068411b524ab403d53326f41ec68ae1f17b6be39f95ef2e7085845c48c", + "openhands-sdk/openhands/sdk/llm/llm.py": "a16bc975d4b126839dede49c84e0401f9236a48014e27c7c3ee3aa4639c82c05", + "openhands-sdk/openhands/sdk/llm/utils/metrics.py": "c8b6977296c1274fdd8f63a2e4556d9be2303469e9b559065048536e8996f93b", + "openhands-sdk/openhands/sdk/llm/utils/telemetry.py": "bed4636f931f6dd95f3275ac63cca3d77e533e2d4f1648d4d56a864eee5e73b4" + }, + "runtime": "macOS, Python 3.13.11, PR checkout editable dependencies", + "entry_points": [ + "OpenHandsAgentSettings.create_agent", + "LLMProfileStore.save/load", + "Conversation.run", + "Conversation.ask_agent", + "Conversation.condense" + ], + "provider": "Loopback HTTP provider: gpt-4o returns 503; gpt-4o-mini returns 200 with 100 prompt tokens and 5 completion tokens.", + "limitations": "Provider responses are simulated; SDK execution, profile persistence, HTTP transport and accounting are real. This artifact checks token totals and role attribution. It does not measure costs or callback invocation counts.", + "reproduce": "OH_PERSISTENCE_DIR=/tmp/fallback-repro-state OPENHANDS_SUPPRESS_BANNER=1 LITELLM_LOCAL_MODEL_COST_MAP=True uv run python .pr/fallback-session-repro.py --assert-fixed", + "before": [ + { + "operation": "run-1", + "http_requests": [ + { + "model": "gpt-4o", + "status": 503 + }, + { + "model": "gpt-4o-mini", + "status": 200 + } + ], + "expected_prompt_token_deltas": { + "default": 100 + }, + "actual_prompt_token_deltas": { + "default": 200 + }, + "correct": false + }, + { + "operation": "run-2", + "http_requests": [ + { + "model": "gpt-4o", + "status": 503 + }, + { + "model": "gpt-4o-mini", + "status": 200 + } + ], + "expected_prompt_token_deltas": { + "default": 100 + }, + "actual_prompt_token_deltas": { + "default": 200 + }, + "correct": false + }, + { + "operation": "run-3", + "http_requests": [ + { + "model": "gpt-4o", + "status": 503 + }, + { + "model": "gpt-4o-mini", + "status": 200 + } + ], + "expected_prompt_token_deltas": { + "default": 100 + }, + "actual_prompt_token_deltas": { + "default": 200 + }, + "correct": false + }, + { + "operation": "ask_agent", + "http_requests": [ + { + "model": "gpt-4o", + "status": 503 + }, + { + "model": "gpt-4o-mini", + "status": 200 + } + ], + "expected_prompt_token_deltas": { + "ask-agent-llm": 100 + }, + "actual_prompt_token_deltas": { + "default": 100, + "ask-agent-llm": 100 + }, + "correct": false + }, + { + "operation": "condense", + "http_requests": [ + { + "model": "gpt-4o", + "status": 503 + }, + { + "model": "gpt-4o-mini", + "status": 200 + } + ], + "expected_prompt_token_deltas": { + "condenser": 100 + }, + "actual_prompt_token_deltas": { + "default": 100, + "condenser": 100 + }, + "correct": false + } + ], + "after": [ + { + "operation": "run-1", + "http_requests": [ + { + "model": "gpt-4o", + "status": 503 + }, + { + "model": "gpt-4o-mini", + "status": 200 + } + ], + "expected_prompt_token_deltas": { + "default": 100 + }, + "actual_prompt_token_deltas": { + "default": 100 + }, + "correct": true + }, + { + "operation": "run-2", + "http_requests": [ + { + "model": "gpt-4o", + "status": 503 + }, + { + "model": "gpt-4o-mini", + "status": 200 + } + ], + "expected_prompt_token_deltas": { + "default": 100 + }, + "actual_prompt_token_deltas": { + "default": 100 + }, + "correct": true + }, + { + "operation": "run-3", + "http_requests": [ + { + "model": "gpt-4o", + "status": 503 + }, + { + "model": "gpt-4o-mini", + "status": 200 + } + ], + "expected_prompt_token_deltas": { + "default": 100 + }, + "actual_prompt_token_deltas": { + "default": 100 + }, + "correct": true + }, + { + "operation": "ask_agent", + "http_requests": [ + { + "model": "gpt-4o", + "status": 503 + }, + { + "model": "gpt-4o-mini", + "status": 200 + } + ], + "expected_prompt_token_deltas": { + "ask-agent-llm": 100 + }, + "actual_prompt_token_deltas": { + "ask-agent-llm": 100 + }, + "correct": true + }, + { + "operation": "condense", + "http_requests": [ + { + "model": "gpt-4o", + "status": 503 + }, + { + "model": "gpt-4o-mini", + "status": 200 + } + ], + "expected_prompt_token_deltas": { + "condenser": 100 + }, + "actual_prompt_token_deltas": { + "condenser": 100 + }, + "correct": true + } + ], + "fixed_commit": "6bfdc18917a3ec2847029f3b22cf1861bf8b99ec" +} diff --git a/.pr/fallback-session-repro.py b/.pr/fallback-session-repro.py new file mode 100644 index 00000000000..88c67297f2e --- /dev/null +++ b/.pr/fallback-session-repro.py @@ -0,0 +1,165 @@ +"""Exercise fallback accounting through real SDK conversations and loopback HTTP. + +Run with an isolated OH_PERSISTENCE_DIR and the checkout's Python environment: + OH_PERSISTENCE_DIR=/tmp/fallback-repro-state uv run python \ + .pr/fallback-session-repro.py + +The provider is simulated: primary requests receive HTTP 503, while successful +fallback requests report exactly 100 input and 5 output tokens. SDK execution, +settings, profile loading, HTTP transport, telemetry and accounting are real. +""" + +import argparse +import json +import os +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Any + +from openhands.sdk import LLM, Conversation +from openhands.sdk.llm.fallback_strategy import FallbackStrategy +from openhands.sdk.llm.llm_profile_store import LLMProfileStore +from openhands.sdk.settings import OpenHandsAgentSettings + + +def token_totals(conversation: Conversation) -> dict[str, int]: + return { + role: metrics.accumulated_token_usage.prompt_tokens + if metrics.accumulated_token_usage is not None + else 0 + for role, metrics in conversation.conversation_stats.usage_to_metrics.items() + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--assert-fixed", action="store_true") + args = parser.parse_args() + if not os.environ.get("OH_PERSISTENCE_DIR"): + parser.error("Set OH_PERSISTENCE_DIR to an isolated temporary directory") + + requests: list[dict[str, Any]] = [] + + class Provider(BaseHTTPRequestHandler): + def do_POST(self) -> None: + body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + model = body["model"] + failed = model == "gpt-4o" + status = 503 if failed else 200 + requests.append({"model": model, "status": status}) + if failed: + payload = {"error": {"message": "Simulated outage", "type": "server"}} + else: + payload = { + "id": f"chatcmpl-{len(requests)}", + "object": "chat.completion", + "created": 1, + "model": model, + "choices": [ + { + "index": 0, + "finish_reason": "stop", + "message": {"role": "assistant", "content": "Done."}, + } + ], + "usage": { + "prompt_tokens": 100, + "completion_tokens": 5, + "total_tokens": 105, + }, + } + data = json.dumps(payload).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def log_message(self, format: str, *args: Any) -> None: + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), Provider) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + evidence: list[dict[str, Any]] = [] + try: + with TemporaryDirectory(prefix="fallback-repro-") as directory: + root = Path(directory) + common = { + "base_url": f"http://127.0.0.1:{server.server_port}/v1", + "api_key": "loopback-only", + "num_retries": 1, + "stream": False, + "input_cost_per_token": 0.001, + "output_cost_per_token": 0.002, + } + profiles = root / "profiles" + LLMProfileStore(profiles).save( + "fallback", + LLM(model="openai/gpt-4o-mini", **common), + include_secrets=True, + ) + primary = LLM( + model="openai/gpt-4o", + fallback_strategy=FallbackStrategy( + fallback_llms=["fallback"], profile_store_dir=profiles + ), + **common, + ) + settings = OpenHandsAgentSettings(llm=primary, tools=[]) + conversation = Conversation( + agent=settings.create_agent(), + workspace=root / "workspace", + persistence_dir=root / "conversations", + visualizer=None, + ) + try: + for operation, expected_role in [ + ("run-1", "default"), + ("run-2", "default"), + ("run-3", "default"), + ("ask_agent", "ask-agent-llm"), + ("condense", "condenser"), + ]: + before = token_totals(conversation) + start = len(requests) + if operation.startswith("run"): + conversation.send_message(f"Reply briefly for {operation}.") + conversation.run() + elif operation == "ask_agent": + conversation.ask_agent("What are you doing?") + else: + conversation.condense() + after = token_totals(conversation) + deltas = { + role: total - before.get(role, 0) + for role, total in after.items() + if total != before.get(role, 0) + } + receipts = requests[start:] + successes = sum(item["status"] == 200 for item in receipts) + expected = {expected_role: 100 * successes} + evidence.append( + { + "operation": operation, + "http_requests": receipts, + "expected_prompt_token_deltas": expected, + "actual_prompt_token_deltas": deltas, + "correct": successes == 1 and deltas == expected, + } + ) + finally: + conversation.close() + finally: + server.shutdown() + server.server_close() + worker.join() + print(json.dumps(evidence, indent=2)) + if args.assert_fixed and not all(item["correct"] for item in evidence): + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/examples/01_standalone_sdk/39_llm_fallback.py b/examples/01_standalone_sdk/39_llm_fallback.py index ebd891c8f37..566cc464ea1 100644 --- a/examples/01_standalone_sdk/39_llm_fallback.py +++ b/examples/01_standalone_sdk/39_llm_fallback.py @@ -3,7 +3,7 @@ When the primary LLM fails with a transient error (rate limit, timeout, etc.), FallbackStrategy automatically tries alternate LLMs in order. Fallback is per-call: each new request starts with the primary model. Token usage and -cost from fallback calls are merged into the primary LLM's metrics. +cost from fallback calls are recorded in the caller's usage session. This example: 1. Saves two fallback LLM profiles to a temporary store. diff --git a/openhands-sdk/openhands/sdk/llm/fallback_strategy.py b/openhands-sdk/openhands/sdk/llm/fallback_strategy.py index faa2f43a4fc..c9ca0c9794c 100644 --- a/openhands-sdk/openhands/sdk/llm/fallback_strategy.py +++ b/openhands-sdk/openhands/sdk/llm/fallback_strategy.py @@ -25,6 +25,7 @@ if TYPE_CHECKING: + from openhands.sdk.llm.llm import LLM from openhands.sdk.llm.llm_response import LLMResponse from openhands.sdk.llm.utils.metrics import Metrics @@ -75,14 +76,17 @@ def try_fallback( primary_error: Exception, primary_metrics: Metrics, call_fn: Callable[[Any], LLMResponse], + *, + primary: LLM | None = None, ) -> LLMResponse | None: - """Try fallback LLMs in order. Merges metrics into primary on success. + """Try fallback models in the caller's role and session. Args: primary_model: The primary model name (for logging). primary_error: The error from the primary model. - primary_metrics: The primary LLM's Metrics to merge fallback costs into. + primary_metrics: The primary LLM's metrics for legacy direct callers. call_fn: A callable that takes an LLM instance and returns an LLMResponse. + primary: The caller whose usage ID and aggregate accounting are kept. Returns: LLMResponse from the first successful fallback, or None if all fail. @@ -99,17 +103,23 @@ def try_fallback( f"{remaining} fallback(s) remaining" ) try: - # Disable nested fallbacks to prevent recursive chains - saved_strategy = fb.fallback_strategy - fb.fallback_strategy = None - metrics_before = fb.metrics.deep_copy() - try: - result = call_fn(fb) - finally: - fb.fallback_strategy = saved_strategy - # Merge fallback metrics (cost + tokens) into primary - metrics_diff = fb.metrics.diff(metrics_before) - primary_metrics.merge(metrics_diff) + fallback = fb.model_copy( + update={ + "usage_id": primary.usage_id + if primary is not None + else fb.usage_id, + "fallback_strategy": None, + } + ) + metrics_before = ( + fallback.metrics.deep_copy() if primary is None else None + ) + if primary is not None: + fallback._session = primary.session + fallback._registry_session = primary._registered_session() + result = call_fn(fallback) + if metrics_before is not None: + primary_metrics.merge(fallback.metrics.diff(metrics_before)) logger.info(f"[Fallback Strategy] Fallback LLM ({fb.model}) succeeded") return result except Exception as fb_error: diff --git a/openhands-sdk/openhands/sdk/llm/llm.py b/openhands-sdk/openhands/sdk/llm/llm.py index 9e77e97a103..6a65bcef11c 100644 --- a/openhands-sdk/openhands/sdk/llm/llm.py +++ b/openhands-sdk/openhands/sdk/llm/llm.py @@ -977,7 +977,7 @@ def session_for(self, call_context: LLMCallContext | None = None) -> LLMSession: if session is None: session = self._registered_session() if session is None: - return self.session + session = self.session if session is not self._session: self._keep_own_session_apart() session.follow(self) @@ -1093,6 +1093,7 @@ def _handle_error( primary_error=error, primary_metrics=session.metrics, call_fn=fallback_call_fn, + primary=self, ) if result is not None: return result @@ -1111,21 +1112,20 @@ async def _ahandle_error( The *fallback_call_fn* is synchronous (it calls the fallback LLM's sync ``completion``/``responses``), so the fallback attempt is - offloaded to a thread via :func:`asyncio.loop.run_in_executor` to + offloaded to a thread via :func:`asyncio.to_thread` to avoid blocking the event loop. """ if session is None: session = self.session session.telemetry.on_error(error) if self.fallback_strategy and self.fallback_strategy.should_fallback(error): - loop = asyncio.get_running_loop() - result = await loop.run_in_executor( - None, + result = await asyncio.to_thread( self.fallback_strategy.try_fallback, self.model, error, session.metrics, fallback_call_fn, + primary=self, ) if result is not None: return result diff --git a/openhands-sdk/openhands/sdk/llm/utils/metrics.py b/openhands-sdk/openhands/sdk/llm/utils/metrics.py index 720fb27b469..e851c7d9386 100644 --- a/openhands-sdk/openhands/sdk/llm/utils/metrics.py +++ b/openhands-sdk/openhands/sdk/llm/utils/metrics.py @@ -186,24 +186,28 @@ def rolls_up_into(self) -> "Metrics | None": except AttributeError: # never attached: the slot is still empty return None - def add_cost(self, value: float) -> None: + def add_cost(self, value: float, *, model_name: str | None = None) -> None: if value < 0: raise ValueError("Added cost cannot be negative.") self.accumulated_cost += value - self.costs.append(Cost(cost=value, model=self.model_name)) + self.costs.append(Cost(cost=value, model=model_name or self.model_name)) target = self.rolls_up_into if target is not None: - target.add_cost(value) + target.add_cost(value, model_name=model_name) - def add_response_latency(self, value: float, response_id: str) -> None: + def add_response_latency( + self, value: float, response_id: str, *, model_name: str | None = None + ) -> None: self.response_latencies.append( ResponseLatency( - latency=max(0.0, value), model=self.model_name, response_id=response_id + latency=max(0.0, value), + model=model_name or self.model_name, + response_id=response_id, ) ) target = self.rolls_up_into if target is not None: - target.add_response_latency(value, response_id) + target.add_response_latency(value, response_id, model_name=model_name) def add_token_usage( self, @@ -214,13 +218,15 @@ def add_token_usage( context_window: int, response_id: str, reasoning_tokens: int = 0, + *, + model_name: str | None = None, ) -> None: """Add a single usage record.""" # Token each turn for calculating context usage. per_turn_token = prompt_tokens + completion_tokens usage = TokenUsage( - model=self.model_name, + model=model_name or self.model_name, prompt_tokens=prompt_tokens, completion_tokens=completion_tokens, cache_read_tokens=cache_read_tokens, @@ -259,6 +265,7 @@ def add_token_usage( context_window=context_window, response_id=response_id, reasoning_tokens=reasoning_tokens, + model_name=model_name, ) def merge(self, other: "Metrics") -> None: diff --git a/openhands-sdk/openhands/sdk/llm/utils/telemetry.py b/openhands-sdk/openhands/sdk/llm/utils/telemetry.py index d2d7c71ce28..a1616e2beef 100644 --- a/openhands-sdk/openhands/sdk/llm/utils/telemetry.py +++ b/openhands-sdk/openhands/sdk/llm/utils/telemetry.py @@ -227,13 +227,15 @@ def on_response( # 1) latency self._last_latency = time.time() - (self._req_start or time.time()) response_id = resp.id - self.metrics.add_response_latency(self._last_latency, response_id) + self.metrics.add_response_latency( + self._last_latency, response_id, model_name=self.model_name + ) # 2) cost cost = self._compute_cost(resp, provider_info=provider_info) # A confirmed zero charge is different from unavailable pricing. if cost is not None: - self.metrics.add_cost(cost) + self.metrics.add_cost(cost, model_name=self.model_name) # 3) tokens - normalize the provider's usage shape exactly once usage = normalize_usage(_response_usage(resp)) @@ -315,6 +317,7 @@ def _record_usage( reasoning_tokens=usage.reasoning_tokens, context_window=context_window, response_id=response_id, + model_name=self.model_name, ) # ---------- Observability span ---------- diff --git a/tests/sdk/llm/test_llm_fallback_sessions.py b/tests/sdk/llm/test_llm_fallback_sessions.py new file mode 100644 index 00000000000..b2b713571f9 --- /dev/null +++ b/tests/sdk/llm/test_llm_fallback_sessions.py @@ -0,0 +1,277 @@ +"""Fallback requests keep the caller's role and accounting session.""" + +import json +from contextlib import nullcontext +from pathlib import Path +from typing import Any, Literal +from unittest.mock import AsyncMock + +import pytest +from litellm.exceptions import APIConnectionError +from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse +from litellm.types.utils import Choices, Message as LiteLLMMessage, ModelResponse, Usage +from pydantic import SecretStr + +from openhands.sdk.llm import ( + LLM, + FallbackStrategy, + LLMCallContext, + LLMRegistry, + Message, + TextContent, +) +from openhands.sdk.llm.call_context import llm_call_context_scope +from openhands.sdk.llm.llm_profile_store import LLMProfileStore + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "method,role,context_mode,collision,shared", + [ + ("completion", "default", "explicit", False, False), + ("acompletion", "default", "scoped", False, True), + ("responses", "condenser", "explicit", True, False), + ("aresponses", "condenser", "scoped", True, False), + ("completion", "condenser", "explicit", False, False), + ("acompletion", "condenser", "scoped", False, False), + ("completion", "condenser", "standalone", False, False), + ("aresponses", "default", "standalone", False, False), + ], +) +async def test_fallback_preserves_caller_accounting( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + method: Literal["completion", "acompletion", "responses", "aresponses"], + role: str, + context_mode: Literal["explicit", "scoped", "standalone"], + collision: bool, + shared: bool, +): + fallback = LLM( + model="gpt-4o-mini", + api_key=SecretStr("test-key"), + num_retries=0, + input_cost_per_token=0.01, + output_cost_per_token=0.02, + log_completions=True, + log_completions_folder=str(tmp_path / "logs"), + ) + LLMProfileStore(base_dir=tmp_path / "profiles").save( + "backup", fallback, include_secrets=True + ) + primary = LLM( + model="gpt-4o", + api_key=SecretStr("test-key"), + usage_id=role, + num_retries=0, + input_cost_per_token=0.1, + output_cost_per_token=0.2, + log_completions=True, + log_completions_folder=str(tmp_path / "logs"), + fallback_strategy=FallbackStrategy( + fallback_llms=["backup"], profile_store_dir=tmp_path / "profiles" + ), + ) + registry = LLMRegistry() + if context_mode != "standalone": + registry.add(primary) + if collision: + registry.add(LLM(model="gpt-4o-mini")) + other_registry = LLMRegistry() + if shared: + other_registry.add(primary) + metrics = ( + primary.metrics + if context_mode == "standalone" + else registry.sessions[role].metrics + ) + notifications: list[tuple[int, float]] = [] + logs: list[tuple[str, str, dict[str, Any]]] = [] + + def on_stats() -> None: + usage = metrics.accumulated_token_usage + assert usage is not None + notifications.append((usage.prompt_tokens, metrics.accumulated_cost)) + + def on_log(filename: str, data: str, usage_id: str, model: str) -> None: + logs.append((usage_id, model, json.loads(data))) + + if context_mode == "standalone": + primary.telemetry.set_stats_update_callback(on_stats) + primary.telemetry.set_log_completions_callback( + lambda filename, data: on_log( + filename, data, role, primary.telemetry.model_name + ) + ) + else: + registry.set_stats_update_callback(on_stats) + registry.set_log_completions_callback(on_log) + + context = LLMCallContext( + prompt_cache_key="caller-cache", + session_id="caller-conversation", + sessions=registry.sessions if context_mode != "standalone" else None, + ) + requests: list[dict[str, Any]] = [] + primary_attempts = 0 + + def respond(**kwargs: Any) -> ModelResponse | ResponsesAPIResponse: + nonlocal primary_attempts + requests.append(kwargs) + model = kwargs["model"] + if model == primary.model: + primary_attempts += 1 + if primary_attempts <= 2: + raise APIConnectionError( + message="primary unavailable", llm_provider="openai", model=model + ) + if method in ("responses", "aresponses"): + return ResponsesAPIResponse( + id=f"response-{len(requests)}", + created_at=1, + model=model, + object="response", + output=[ + { + "type": "message", + "id": "message", + "role": "assistant", + "status": "completed", + "content": [ + {"type": "output_text", "text": "ok", "annotations": []} + ], + } + ], + parallel_tool_calls=False, + tool_choice="auto", + tools=[], + usage=ResponseAPIUsage( + input_tokens=10, output_tokens=5, total_tokens=15 + ), + ) + return ModelResponse( + id=f"response-{len(requests)}", + model=model, + choices=[ + Choices( + index=0, + finish_reason="stop", + message=LiteLLMMessage(role="assistant", content="ok"), + ) + ], + usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15), + ) + + monkeypatch.setattr("openhands.sdk.llm.llm.litellm_completion", respond) + monkeypatch.setattr("openhands.sdk.llm.llm.litellm_responses", respond) + monkeypatch.setattr( + "openhands.sdk.llm.llm.litellm_acompletion", AsyncMock(side_effect=respond) + ) + monkeypatch.setattr( + "openhands.sdk.llm.llm.litellm_aresponses", AsyncMock(side_effect=respond) + ) + messages = [ + Message(role="system", content=[TextContent(text="Reply briefly.")]), + Message(role="user", content=[TextContent(text="Hello.")]), + ] + supplied_context = context if context_mode == "explicit" else None + scope = ( + llm_call_context_scope(context) if context_mode == "scoped" else nullcontext() + ) + with scope: + for count, cost in enumerate((0.2, 0.4, 2.4), start=1): + if method == "completion": + result = primary.completion(messages, call_context=supplied_context) + elif method == "acompletion": + result = await primary.acompletion( + messages, call_context=supplied_context + ) + elif method == "responses": + result = primary.responses(messages, call_context=supplied_context) + else: + result = await primary.aresponses( + messages, call_context=supplied_context + ) + assert result.metrics.accumulated_token_usage is not None + assert result.metrics.accumulated_token_usage.prompt_tokens == 10 * count + assert result.metrics.accumulated_cost == pytest.approx(cost) + assert metrics.accumulated_cost == pytest.approx(cost) + assert primary.metrics.accumulated_cost == pytest.approx(cost) + assert [tokens for tokens, _ in notifications] == [ + 10 * n for n in range(1, count + 1) + ] + assert [total for _, total in notifications] == pytest.approx( + (0.2, 0.4, 2.4)[:count] + ) + + assert [record.model for record in metrics.token_usages] == [ + "gpt-4o-mini", + "gpt-4o-mini", + "gpt-4o", + ] + assert [(uid, model) for uid, model, data in logs if "response" in data] == [ + (role, "gpt-4o-mini"), + (role, "gpt-4o-mini"), + (role, "gpt-4o"), + ] + if context_mode != "standalone": + for request in requests: + assert request["prompt_cache_key"] == "caller-cache" + assert ( + request["extra_headers"]["x-litellm-session-id"] + == "caller-conversation" + ) + if collision: + assert registry.sessions["default"].metrics.accumulated_cost == 0 + if shared: + assert other_registry.sessions[role].metrics.accumulated_cost == 0 + + +def test_direct_fallback_preserves_supplied_metrics( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +): + primary = LLM(model="gpt-4o", usage_id="condenser") + registry, other_registry = LLMRegistry(), LLMRegistry() + registry.add(primary) + other_registry.add(primary) + metrics = registry.sessions["condenser"].metrics + LLMProfileStore(base_dir=tmp_path).save( + "backup", LLM(model="gpt-4o-mini", api_key=SecretStr("test-key")) + ) + strategy = FallbackStrategy(fallback_llms=["backup"], profile_store_dir=tmp_path) + response = ModelResponse( + id="fallback-response", + model="gpt-4o-mini", + choices=[ + Choices( + index=0, + finish_reason="stop", + message=LiteLLMMessage(role="assistant", content="ok"), + ) + ], + usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15), + ) + monkeypatch.setattr( + "openhands.sdk.llm.llm.litellm_completion", lambda **kwargs: response + ) + messages = [ + Message(role="system", content=[TextContent(text="Reply briefly.")]), + Message(role="user", content=[TextContent(text="Hello.")]), + ] + + result = strategy.try_fallback( + primary.model, + APIConnectionError( + message="primary unavailable", llm_provider="openai", model=primary.model + ), + metrics, + lambda fallback: fallback.completion(messages), + ) + + assert result is not None + assert metrics.accumulated_token_usage is not None + assert primary.metrics.accumulated_token_usage is not None + assert metrics.accumulated_token_usage.prompt_tokens == 10 + assert primary.metrics.accumulated_token_usage.prompt_tokens == 10 + assert len(metrics.token_usages) == 1 + assert other_registry.sessions["condenser"].metrics.token_usages == []