From 0a68ab94cb3f447254bdc68ce778048ce517be22 Mon Sep 17 00:00:00 2001 From: VascoSch92 Date: Mon, 9 Nov 2026 17:27:38 +0100 Subject: [PATCH 1/2] refactor(sdk): keep LLM usage and telemetry in sessions keyed by role An LLM object held its own Metrics and Telemetry, so per-conversation state travelled with the object: into a second conversation that reused the LLM, and away from the conversation when the LLM was switched. LLMSession now owns usage metrics and telemetry. Every LLM has one of its own (llm.metrics, llm.telemetry). A conversation's LLMRegistry keeps one per usage_id and hands it to calls through LLMCallContext, so a call records in the session of its role whichever LLM object makes it. The registry takes over the session of the first LLM of a role, which keeps llm.metrics identical to the conversation's ledger for an LLM used by one conversation. An LLM object used by a second conversation gets a separate session there and keeps counting its total on llm.metrics. - model_copy() with another usage_id starts a new session, replacing the reset_metrics() calls that followed such copies and the registry's shared-metrics detection. - switch_llm() no longer restores metrics by hand: the session stays with the usage_id. - The agent server sets its stats and completion-log callbacks on the registry, so they cover sessions opened after a switch. Fixes found on main and covered by tests: - two conversations sharing one LLM reported each other's usage; - the stats callback stopped firing after switch_llm; - condenser usage after switch_llm was missing from conversation stats; - ask_agent usage was dropped after the second switch_llm. --- .../openhands/agent_server/event_service.py | 59 ++-- .../openhands/sdk/agent/acp_agent.py | 28 +- .../sdk/conversation/conversation_stats.py | 13 +- .../conversation/impl/local_conversation.py | 15 +- openhands-sdk/openhands/sdk/hooks/executor.py | 5 +- openhands-sdk/openhands/sdk/llm/__init__.py | 2 + .../openhands/sdk/llm/call_context.py | 26 +- openhands-sdk/openhands/sdk/llm/llm.py | 262 ++++++++++------ .../openhands/sdk/llm/llm_registry.py | 133 +++++--- .../openhands/sdk/llm/llm_session.py | 102 +++++++ .../openhands/sdk/llm/utils/metrics.py | 46 +++ .../openhands/sdk/llm/utils/telemetry.py | 15 +- openhands-sdk/openhands/sdk/settings/model.py | 1 - .../openhands/sdk/subagent/registry.py | 6 +- tests/agent_server/test_event_service.py | 19 +- .../test_llm_summarizing_condenser.py | 3 +- .../local/test_llm_session_isolation.py | 186 ++++++++++++ tests/sdk/conversation/test_switch_model.py | 2 +- tests/sdk/llm/test_llm.py | 16 +- tests/sdk/llm/test_llm_metrics.py | 50 +++ tests/sdk/llm/test_llm_registry.py | 190 ++++++------ tests/sdk/llm/test_llm_serialization.py | 6 +- tests/sdk/llm/test_llm_session.py | 284 ++++++++++++++++++ .../llm/test_responses_parsing_and_kwargs.py | 2 +- 24 files changed, 1129 insertions(+), 342 deletions(-) create mode 100644 openhands-sdk/openhands/sdk/llm/llm_session.py create mode 100644 tests/sdk/conversation/local/test_llm_session_isolation.py create mode 100644 tests/sdk/llm/test_llm_session.py diff --git a/openhands-agent-server/openhands/agent_server/event_service.py b/openhands-agent-server/openhands/agent_server/event_service.py index 9bb1415a71e..db5a5f2500f 100644 --- a/openhands-agent-server/openhands/agent_server/event_service.py +++ b/openhands-agent-server/openhands/agent_server/event_service.py @@ -1118,32 +1118,30 @@ def locked_on_event(): partial(_log_emit_failure, type(event).__name__), ) - def _setup_llm_log_streaming(self, agent: AgentBase) -> None: - """Configure LLM log callbacks to stream logs via events.""" - for llm in agent.get_all_llms(): - if not llm.log_completions: - continue + def _setup_llm_log_streaming(self, conversation: LocalConversation) -> None: + """Configure LLM log callbacks to stream logs via events. - # Capture variables for closure - usage_id = llm.usage_id - model_name = llm.model + Set on the conversation's registry, so it covers every LLM session of + the conversation, including those opened by a later ``switch_llm`` / + ``switch_profile``. Only LLMs with ``log_completions`` on produce logs. + """ - def log_callback( - filename: str, log_data: str, uid=usage_id, model=model_name - ) -> None: - """Callback to emit LLM completion logs as events.""" - try: - event = LLMCompletionLogEvent( - filename=filename, - log_data=log_data, - model_name=model, - usage_id=uid, - ) - self._emit_event_from_thread(event) - except Exception: - logger.exception("Failed to emit LLM completion log event") + def log_callback( + filename: str, log_data: str, usage_id: str, model_name: str + ) -> None: + """Callback to emit LLM completion logs as events.""" + try: + event = LLMCompletionLogEvent( + filename=filename, + log_data=log_data, + model_name=model_name, + usage_id=usage_id, + ) + self._emit_event_from_thread(event) + except Exception: + logger.exception("Failed to emit LLM completion log event") - llm.telemetry.set_log_completions_callback(log_callback) + conversation.llm_registry.set_log_completions_callback(log_callback) def _setup_acp_activity_heartbeat(self, agent: AgentBase) -> None: """Wire ACP activity heartbeat to the idle timer. @@ -1195,8 +1193,12 @@ def callback(event: Event) -> None: return callback - def _setup_stats_streaming(self, agent: AgentBase) -> None: - """Configure stats update callbacks to stream stats changes via events.""" + def _setup_stats_streaming(self, conversation: LocalConversation) -> None: + """Configure stats update callbacks to stream stats changes via events. + + Set on the conversation's registry, so it keeps firing for the LLM a + later ``switch_llm`` / ``switch_profile`` installs. + """ def stats_callback() -> None: """Callback to emit stats updates. @@ -1224,8 +1226,7 @@ def stats_callback() -> None: ) self._emit_event_from_thread(event) - for llm in agent.get_all_llms(): - llm.telemetry.set_stats_update_callback(stats_callback) + conversation.llm_registry.set_stats_update_callback(stats_callback) @staticmethod def _ensure_workspace_is_git_repo(working_dir: Path) -> None: @@ -1429,10 +1430,10 @@ def _token_streaming_callback(chunk: LLMStreamChunk | str) -> None: ) # Setup LLM log streaming for remote execution - self._setup_llm_log_streaming(self._conversation.agent) + self._setup_llm_log_streaming(self._conversation) # Setup stats streaming for remote execution - self._setup_stats_streaming(self._conversation.agent) + self._setup_stats_streaming(self._conversation) # Wire ACP activity heartbeat so ACP tool calls (which run inside # the subprocess and never hit HTTP endpoints) still reset the diff --git a/openhands-sdk/openhands/sdk/agent/acp_agent.py b/openhands-sdk/openhands/sdk/agent/acp_agent.py index 7162363531e..494e00c7e60 100644 --- a/openhands-sdk/openhands/sdk/agent/acp_agent.py +++ b/openhands-sdk/openhands/sdk/agent/acp_agent.py @@ -2182,6 +2182,11 @@ def _record_usage( elapsed: Wall-clock seconds for this prompt round-trip (optional). usage_update: The synchronized ACP UsageUpdate for this turn, if any. """ + # The conversation's session for the sentinel LLM: its registry opened + # it, so this resolves on the portal thread as well as on the caller's. + session = self.llm.session_for() + metrics = session.metrics + # -- Cost recording --------------------------------------------------- # claude-agent-acp, codex-acp: report cost via UsageUpdate notification # gemini-cli: does not send UsageUpdate (cost derived from tokens below) @@ -2190,7 +2195,7 @@ def _record_usage( last_cost = self._client._last_cost_by_session.get(session_id, 0.0) delta = usage_update.cost.amount - last_cost if delta > 0: - self.llm.metrics.add_cost(delta) + metrics.add_cost(delta) cost_recorded = True self._client._last_cost_by_session[session_id] = usage_update.cost.amount self._client._last_cost = usage_update.cost.amount @@ -2200,7 +2205,7 @@ def _record_usage( _extract_token_usage(response) ) if input_tokens or output_tokens: - self.llm.metrics.add_token_usage( + metrics.add_token_usage( prompt_tokens=input_tokens, completion_tokens=output_tokens, cache_read_tokens=cache_read, @@ -2221,7 +2226,7 @@ def _record_usage( self.acp_model, input_tokens, output_tokens ) if cost > 0: - self.llm.metrics.add_cost(cost) + metrics.add_cost(cost) if not cost_recorded and not input_tokens and not output_tokens: # gemini-cli currently returns response.usage=None and @@ -2233,13 +2238,9 @@ def _record_usage( ) if elapsed is not None: - self.llm.metrics.add_response_latency(elapsed, session_id) + metrics.add_response_latency(elapsed, session_id) - if self.llm.telemetry._stats_update_callback is not None: - try: - self.llm.telemetry._stats_update_callback() - except Exception: - logger.debug("Stats update callback failed", exc_info=True) + session.telemetry.notify_stats_update() # -- Capability helpers ------------------------------------------------ @@ -4727,9 +4728,12 @@ def set_acp_model(self, model: str) -> None: # authoritative current model is persisted by # :meth:`LocalConversation.switch_acp_model` via an agent ``model_copy``. self.llm.model = model - self.llm.metrics.model_name = model - if self.llm.metrics.accumulated_token_usage is not None: - self.llm.metrics.accumulated_token_usage.model = model + # Both ledgers carry the model name: the LLM's own and the one the + # conversation keeps for it (the same object outside a conversation). + for metrics in (self.llm.metrics, self.llm.session_for().metrics): + metrics.model_name = model + if metrics.accumulated_token_usage is not None: + metrics.accumulated_token_usage.model = model # Refresh the surfaced model state so the chip/picker # (``ConversationInfo.current_model_id``) reflects the switch instead # of the stale session-start value. ``_current_model_id`` is a diff --git a/openhands-sdk/openhands/sdk/conversation/conversation_stats.py b/openhands-sdk/openhands/sdk/conversation/conversation_stats.py index e574c7caf7e..1bee51cd437 100644 --- a/openhands-sdk/openhands/sdk/conversation/conversation_stats.py +++ b/openhands-sdk/openhands/sdk/conversation/conversation_stats.py @@ -68,18 +68,19 @@ def get_metrics_for_usage(self, usage_id: str) -> Metrics: return self.usage_to_metrics[usage_id] def register_llm(self, event: RegistryEvent): - # Listen for LLM creations and track their metrics - llm = event.llm - usage_id = llm.usage_id + # Listen for LLM registrations and track the metrics of their session: + # the one the registry keeps for the usage ID, else the LLM's own. + usage_id = event.llm.usage_id + session = event.session if event.session is not None else event.llm.session # Usage costs exist but have not been restored yet if ( usage_id in self.usage_to_metrics and usage_id not in self._restored_usage_ids ): - llm.restore_metrics(self.usage_to_metrics[usage_id]) + session.restore_metrics(self.usage_to_metrics[usage_id]) self._restored_usage_ids.add(usage_id) # Usage is new, track its metrics - if usage_id not in self.usage_to_metrics and llm.metrics: - self.usage_to_metrics[usage_id] = llm.metrics + if usage_id not in self.usage_to_metrics: + self.usage_to_metrics[usage_id] = session.metrics diff --git a/openhands-sdk/openhands/sdk/conversation/impl/local_conversation.py b/openhands-sdk/openhands/sdk/conversation/impl/local_conversation.py index 25ab31339ba..9045a07acff 100644 --- a/openhands-sdk/openhands/sdk/conversation/impl/local_conversation.py +++ b/openhands-sdk/openhands/sdk/conversation/impl/local_conversation.py @@ -360,11 +360,16 @@ def __init__( # Create-or-resume: factory inspects BASE_STATE to decide desired_id = conversation_id or uuid.uuid4() + # The registry owns this conversation's LLM sessions (usage metrics and + # telemetry per usage_id); the call context carries them to every LLM + # call. Agent LLMs are registered later, in _ensure_agent_ready(). + self.llm_registry = LLMRegistry() self._llm_call_context = ( _parent_llm_call_context or LLMCallContext() ).for_conversation( str(desired_id), prompt_cache_key=prompt_cache_key, + sessions=self.llm_registry.sessions, ) # Resolve client-defined tools, then register them and inject the matching @@ -548,7 +553,6 @@ def _track_last_user_message(e): # Agent initialization is deferred to _ensure_agent_ready() for lazy loading # This ensures plugins are loaded before agent initialization - self.llm_registry = LLMRegistry() self._profile_store = LLMProfileStore(profile_store_dir) self._cipher = cipher @@ -1802,7 +1806,6 @@ def _condenser_for_switched_llm( condenser_llm = new_llm.model_copy( update={"usage_id": condenser.llm.usage_id}, ) - condenser_llm.reset_metrics() return condenser.model_copy(update={"llm": condenser_llm}) def _reject_llm_switch_on_acp(self) -> None: @@ -1821,8 +1824,8 @@ def switch_llm(self, llm: LLM) -> None: entry with that key already exists and has the same config, the cached LLM is reused. If its config differs (e.g. ``default``, which agent initialization registers, posted with another model), the - passed ``llm`` replaces the entry and takes over the metrics already - recorded under that ``usage_id``. + passed ``llm`` replaces the entry. Usage keeps accumulating in the + session the registry holds for that ``usage_id``. Args: llm: LLM to install on the agent. @@ -1843,10 +1846,6 @@ def switch_llm(self, llm: LLM) -> None: if registered is not None: self.llm_registry.remove(usage_id) self.llm_registry.add(new_llm) - if registered is not None: - # Keep one metrics object per usage_id: ConversationStats - # tracks the registered LLM's, so accrued usage is retained. - new_llm.restore_metrics(registered.metrics) # A switch_llm tool runs on a worker thread while run()/arun() holds the # state lock across the agent step on another thread, blocked awaiting # this very tool. Re-acquiring _state here would deadlock, so skip it: diff --git a/openhands-sdk/openhands/sdk/hooks/executor.py b/openhands-sdk/openhands/sdk/hooks/executor.py index 8f75ace7f99..830b5bf1a8f 100644 --- a/openhands-sdk/openhands/sdk/hooks/executor.py +++ b/openhands-sdk/openhands/sdk/hooks/executor.py @@ -244,8 +244,8 @@ def _execute_agent_hook( "timeout": hook.timeout, } ) - # Isolate Metrics so hook spend doesn't accrue to the parent's bucket. - hook_llm.reset_metrics() + # Its own usage_id gives the copy its own session, so hook spend + # doesn't accrue to the parent's bucket. # Never hand the parent's already-initialized visualizer instance to the # sub-conversation: LocalConversation.__init__ calls initialize() on it, @@ -328,7 +328,6 @@ def _execute_prompt_hook( "stream": False, } ) - hook_llm.reset_metrics() messages = [ Message( diff --git a/openhands-sdk/openhands/sdk/llm/__init__.py b/openhands-sdk/openhands/sdk/llm/__init__.py index 8cc1a58871c..53e3e94f399 100644 --- a/openhands-sdk/openhands/sdk/llm/__init__.py +++ b/openhands-sdk/openhands/sdk/llm/__init__.py @@ -34,6 +34,7 @@ ) from openhands.sdk.llm.llm_registry import LLMRegistry, RegistryEvent from openhands.sdk.llm.llm_response import LLMResponse +from openhands.sdk.llm.llm_session import LLMSession from openhands.sdk.llm.message import ( ImageContent, Message, @@ -95,6 +96,7 @@ "LLMCallContext", "LLM_PROFILE_SCHEMA_VERSION", "LLMRegistry", + "LLMSession", "LLMProfileLoader", "LLMProfileMutator", "LLMProfileStore", diff --git a/openhands-sdk/openhands/sdk/llm/call_context.py b/openhands-sdk/openhands/sdk/llm/call_context.py index a073e4a457d..7dad7fd8c4c 100644 --- a/openhands-sdk/openhands/sdk/llm/call_context.py +++ b/openhands-sdk/openhands/sdk/llm/call_context.py @@ -2,11 +2,15 @@ from __future__ import annotations -from collections.abc import Iterator, MutableMapping +from collections.abc import Iterator, Mapping, MutableMapping from contextlib import contextmanager from contextvars import ContextVar -from dataclasses import dataclass, replace -from typing import Any +from dataclasses import dataclass, field, replace +from typing import TYPE_CHECKING, Any + + +if TYPE_CHECKING: + from openhands.sdk.llm.llm_session import LLMSession @dataclass(frozen=True) @@ -21,23 +25,33 @@ class LLMCallContext: prompt_cache_key: str | None = None session_id: str | None = None + # The conversation's LLM sessions by role (``usage_id``). A call records its + # usage in the session of the calling LLM's role, when the conversation has + # one. It is a live view owned by the conversation's ``LLMRegistry``, so it + # takes no part in equality or hashing. + sessions: Mapping[str, LLMSession] | None = field( + default=None, compare=False, repr=False + ) def for_conversation( self, conversation_id: str, *, prompt_cache_key: str | None = None, + sessions: Mapping[str, LLMSession] | None = None, ) -> LLMCallContext: """Derive context for a conversation while preserving inheritable fields. - Conversation identity is always replaced. Any additional fields added to - this dataclass are inherited automatically unless this method explicitly - gives them conversation-local semantics. + Conversation identity is always replaced, and so are the sessions: a + child conversation never records into its parent's. Any additional + fields added to this dataclass are inherited automatically unless this + method explicitly gives them conversation-local semantics. """ return replace( self, prompt_cache_key=prompt_cache_key or conversation_id, session_id=conversation_id, + sessions=sessions, ) diff --git a/openhands-sdk/openhands/sdk/llm/llm.py b/openhands-sdk/openhands/sdk/llm/llm.py index b836a7265df..9e77e97a103 100644 --- a/openhands-sdk/openhands/sdk/llm/llm.py +++ b/openhands-sdk/openhands/sdk/llm/llm.py @@ -121,6 +121,8 @@ token_counter, ) +# OpenHands utilities +from openhands.sdk.llm.call_context import resolve_llm_call_context from openhands.sdk.llm.exceptions import ( LLMContextWindowTooSmallError, LLMNoResponseError, @@ -128,9 +130,8 @@ looks_like_auth_error, map_provider_exception, ) - -# OpenHands utilities from openhands.sdk.llm.llm_response import LLMResponse +from openhands.sdk.llm.llm_session import LLMSession from openhands.sdk.llm.message import ( Message, TextContent, @@ -277,6 +278,11 @@ class LLM(BaseModel, RetryMixin, NonNativeToolCallingMixin): ``` """ + # The session of the conversation whose LLMRegistry this object was added + # to (see ``session_for``). It says where the object is used, not what it + # is, so it sits in a slot: pydantic neither compares nor copies it. + __slots__ = ("_registry_session",) + # ========================================================================= # Config fields # ========================================================================= @@ -678,12 +684,14 @@ class LLM(BaseModel, RetryMixin, NonNativeToolCallingMixin): default=None, exclude=True, ) - _metrics: Metrics | None = PrivateAttr(default=None) + # Usage metrics and telemetry live in a session, not on the configuration. + # This is the one behind ``llm.metrics`` / ``llm.telemetry``; see + # ``session_for`` for how it relates to a conversation's. + _session: LLMSession | None = PrivateAttr(default=None) # Runtime-only private attrs _model_info: Any = PrivateAttr(default=None) _tokenizer: dict[str, object] | None = PrivateAttr(default=None) _chat_template_tokenizer: ChatTemplateTokenizer | None = PrivateAttr(default=None) - _telemetry: Telemetry | None = PrivateAttr(default=None) _is_subscription: bool = PrivateAttr(default=False) _subscription_credential_store: Any = PrivateAttr(default=None) _subscription_credentials: Any = PrivateAttr(default=None) @@ -781,22 +789,12 @@ def _post_init(self): # OpenRouter ``HTTP-Referer`` / ``X-Title`` headers flow per-call # through ``_openrouter_headers()``. - # Metrics + Telemetry wiring. Guard both: this validator re-runs whenever - # the LLM is passed into another Pydantic model (e.g. RegistryEvent), - # and replacing _telemetry would silently drop any callback callers + # Guarded: this validator re-runs whenever the LLM is passed into + # another Pydantic model (e.g. RegistryEvent), and a new session would + # silently drop the usage recorded so far and any callback callers # have attached via telemetry.set_*_callback(). - if self._metrics is None: - self._metrics = Metrics(model_name=self.model) - - if self._telemetry is None: - self._telemetry = Telemetry( - model_name=self.model, - log_enabled=self.log_completions, - log_dir=self.log_completions_folder if self.log_completions else None, - input_cost_per_token=self.input_cost_per_token, - output_cost_per_token=self.output_cost_per_token, - metrics=self._metrics, - ) + if self._session is None: + self._session = LLMSession(self) # Tokenizer if self.custom_tokenizer: @@ -846,6 +844,14 @@ def model_copy( if route_changed: copied._refresh_litellm_metadata() copied._reset_runtime_metadata_for_key() + if copied.usage_id != self.usage_id: + # Sessions follow the role. A copy made for another role is a + # different user of the model, so it starts its own accounting. + copied.reset_metrics() + elif not deep: + # Same role, same sessions: the shallow copy shares ``_session`` + # already, and stays with the conversation the original is in. + copied._registry_session = self._registered_session() return copied def _openrouter_headers(self) -> dict[str, str]: @@ -908,10 +914,26 @@ def _serialize_secrets(self, v: SecretStr | None, info): # ========================================================================= # Public API # ========================================================================= + @property + def session(self) -> LLMSession: + """The session of this LLM instance itself. + + It counts everything recorded through this object. Once a conversation + registers the LLM this is that conversation's session for the LLM's + role (see :meth:`session_for`). + """ + if self._session is None: + self._session = LLMSession(self) + return self._session + @property def metrics(self) -> Metrics: """Get usage metrics for this LLM instance. + These cover all the usage of this object. A conversation reports its + own usage through ``conversation.conversation_stats``; the two differ + when the same LLM object is used in more than one conversation. + Returns: Metrics object containing token usage, costs, and other statistics. @@ -921,9 +943,7 @@ def metrics(self) -> Metrics: print(f"Total cost: ${cost}") ``` """ - if self._metrics is None: - self._metrics = Metrics(model_name=self.model) - return self._metrics + return self.session.metrics @property def telemetry(self) -> Telemetry: @@ -937,16 +957,66 @@ def telemetry(self) -> Telemetry: llm.telemetry.set_log_completions_callback(my_callback) ``` """ - if self._telemetry is None: - self._telemetry = Telemetry( - model_name=self.model, - log_enabled=self.log_completions, - log_dir=self.log_completions_folder if self.log_completions else None, - input_cost_per_token=self.input_cost_per_token, - output_cost_per_token=self.output_cost_per_token, - metrics=self.metrics, - ) - return self._telemetry + return self.session.telemetry + + def session_for(self, call_context: LLMCallContext | None = None) -> LLMSession: + """Return the session a call made with ``call_context`` records in. + + That is the session the conversation keeps for this LLM's role + (``usage_id``). Without a conversation in the context it is the session + of the registry this object was last added to, and failing that the + object's own. + + The first conversation to register an LLM takes the LLM's own session, + so there ``llm.metrics`` is the conversation's ledger. An object that + records in a second session keeps one of its own apart, and every + session it records in also counts there. + """ + sessions = resolve_llm_call_context(call_context).sessions + session = sessions.get(self.usage_id) if sessions is not None else None + if session is None: + session = self._registered_session() + if session is None: + return self.session + if session is not self._session: + self._keep_own_session_apart() + session.follow(self) + return session + + def _registered_session(self) -> LLMSession | None: + try: + return self._registry_session + except AttributeError: # never added to a registry: the slot is empty + return None + + def _join_registry_session(self, session: LLMSession) -> None: + """Record in ``session``, which a registry keeps for this LLM's role.""" + registered = self._registered_session() + if registered is session: + return + if registered is None: + # Not part of a conversation yet: its session is this object's own + # from now on, as if the usage had always been recorded there. + self._session = session + else: + self._keep_own_session_apart() + self._registry_session = session + session.follow(self) + + def _keep_own_session_apart(self) -> None: + """Give this object a session of its own if a registry holds its current one. + + Needed before it records in another session: usage of one conversation + must not land in the ledger of another. The new session starts from + what the shared one holds, so ``llm.metrics`` keeps its totals. + """ + registered = self._registered_session() + if registered is None or self._session is not registered: + return + own = LLMSession(self) + own.metrics.merge(registered.metrics) + self._session = own + registered.follow(self) @computed_field( return_type=bool, @@ -988,54 +1058,40 @@ def _restore_is_subscription(cls, data, handler): _serialized_is_subscription.reset(token) def restore_metrics(self, metrics: Metrics) -> None: - # Only used by ConversationStats to seed metrics - self._metrics = metrics - # Keep telemetry in sync so post-resume LLM calls record into - # the restored metrics object, not the stale one from __init__. - if self._telemetry is not None: - self._telemetry.metrics = metrics + """Make ``metrics`` the usage metrics of this LLM instance.""" + self.session.restore_metrics(metrics) def reset_metrics(self) -> None: - """Reset metrics and telemetry to fresh instances. - - This is used by the LLMRegistry to ensure each registered LLM has - independent metrics, preventing metrics from being shared between - LLMs that were created via model_copy(). - - When an LLM is copied (e.g., to create a condenser LLM from an agent LLM), - Pydantic's model_copy() does a shallow copy of private attributes by default, - causing the original and copied LLM to share the same Metrics object. - This method allows the registry to fix this by creating fresh metrics and - telemetry immediately, so the copied LLM is ready for the next completion - call even if callers do not access ``metrics``/``telemetry`` first. + """Give this LLM instance a fresh session of its own. + + A copy made with ``model_copy()`` shares the session of the LLM it was + copied from unless it is given another ``usage_id``, so both count into + the same ``llm.metrics``. Call this on a copy that should be counted + apart, such as the LLM handed to a sub-agent. It also detaches the + instance from the session a conversation's registry opened for it. """ - self._metrics = Metrics(model_name=self.model) - self._telemetry = Telemetry( - model_name=self.model, - log_enabled=self.log_completions, - log_dir=self.log_completions_folder if self.log_completions else None, - input_cost_per_token=self.input_cost_per_token, - output_cost_per_token=self.output_cost_per_token, - metrics=self._metrics, - ) + self._session = LLMSession(self) + self._registry_session = None def _handle_error( self, error: Exception, fallback_call_fn: Callable[[LLM], LLMResponse], + session: LLMSession | None = None, ) -> LLMResponse: """Handle an error from completion/responses: try fallback, then map and raise. Must be called from within an except block. Either returns an LLMResponse (fallback succeeded) or re-raises (mapped or original). """ - assert self._telemetry is not None - self._telemetry.on_error(error) + if session is None: + session = self.session + session.telemetry.on_error(error) if self.fallback_strategy and self.fallback_strategy.should_fallback(error): result = self.fallback_strategy.try_fallback( primary_model=self.model, primary_error=error, - primary_metrics=self.metrics, + primary_metrics=session.metrics, call_fn=fallback_call_fn, ) if result is not None: @@ -1049,6 +1105,7 @@ async def _ahandle_error( self, error: Exception, fallback_call_fn: Callable[[LLM], LLMResponse], + session: LLMSession | None = None, ) -> LLMResponse: """Async variant of :meth:`_handle_error`. @@ -1057,8 +1114,9 @@ async def _ahandle_error( offloaded to a thread via :func:`asyncio.loop.run_in_executor` to avoid blocking the event loop. """ - assert self._telemetry is not None - self._telemetry.on_error(error) + if session is None: + session = self.session + session.telemetry.on_error(error) if self.fallback_strategy and self.fallback_strategy.should_fallback(error): loop = asyncio.get_running_loop() result = await loop.run_in_executor( @@ -1066,7 +1124,7 @@ async def _ahandle_error( self.fallback_strategy.try_fallback, self.model, error, - self.metrics, + session.metrics, fallback_call_fn, ) if result is not None: @@ -1226,22 +1284,30 @@ async def _aiter_with_idle_timeout( return yield item - def _build_completion_result(self, resp: ModelResponse) -> LLMResponse: + def _build_completion_result( + self, resp: ModelResponse, session: LLMSession | None = None + ) -> LLMResponse: """Convert a raw :class:`ModelResponse` into an :class:`LLMResponse`.""" + if session is None: + session = self.session first_choice = resp["choices"][0] message = Message.from_llm_chat_message(first_choice["message"]) return LLMResponse( message=message, - metrics=self.metrics.get_snapshot(), + metrics=session.metrics.get_snapshot(), raw_response=resp, ) - def _build_responses_result(self, resp: ResponsesAPIResponse) -> LLMResponse: + def _build_responses_result( + self, resp: ResponsesAPIResponse, session: LLMSession | None = None + ) -> LLMResponse: """Convert a raw :class:`ResponsesAPIResponse` into an :class:`LLMResponse`.""" + if session is None: + session = self.session message = Message.from_llm_responses_output(resp.output) return LLMResponse( message=message, - metrics=self.metrics.get_snapshot(), + metrics=session.metrics.get_snapshot(), raw_response=resp, ) @@ -1330,6 +1396,7 @@ def _finalize_stream_response( self, completed_response: ResponseCompletedEvent | None, collected_output_items: list[Any], + session: LLMSession | None = None, ) -> ResponsesAPIResponse: """Validate and patch the completed response from a Responses stream. @@ -1351,8 +1418,9 @@ def _finalize_stream_response( if not completed_resp.output and collected_output_items: completed_resp.output = collected_output_items - assert self._telemetry is not None - self._telemetry.on_response( + if session is None: + session = self.session + session.telemetry.on_response( completed_resp, provider_info=self._provider_info, ) @@ -1486,7 +1554,7 @@ def _finalize_completion_params( # 4) request context for telemetry (always include context_window for metrics) # Always pass context_window so metrics are tracked even when # logging is disabled. - telemetry = self.telemetry + telemetry = self.session_for(call_context).telemetry telemetry_ctx: dict[str, Any] = { "context_window": self.effective_max_input_tokens or 0 } @@ -1622,7 +1690,7 @@ def _finalize_responses_params( # Request context for telemetry (always include context_window for metrics) # Always pass context_window so metrics are tracked even when # logging is disabled. - telemetry = self.telemetry + telemetry = self.session_for(call_context).telemetry telemetry_ctx: dict[str, Any] = { "context_window": self.effective_max_input_tokens or 0 } @@ -1647,6 +1715,7 @@ def _validate_chat_response( formatted_messages: list[dict[str, Any]], cc_tools: list[ChatCompletionToolParam], add_security_risk_prediction: bool, + session: LLMSession | None = None, ) -> ModelResponse: """Post-process a chat completion response inside the retry boundary. @@ -1669,8 +1738,9 @@ def _validate_chat_response( ) # 6) telemetry - assert self._telemetry is not None - self._telemetry.on_response( + if session is None: + session = self.session + session.telemetry.on_response( resp, raw_resp=raw_resp, provider_info=self._provider_info, @@ -1788,6 +1858,7 @@ def completion( # cached (1h TTL) and negative-cached on failure, and the probe is a # no-op for providers without runtime metadata. self.resolve_runtime_metadata() + session = self.session_for(call_context) _caller_kwargs = kwargs.copy() enable_streaming = bool(kwargs.get("stream", False)) or self.stream @@ -1819,8 +1890,7 @@ def completion( @self._make_retry_decorator() def _one_attempt(**retry_kwargs: Any) -> ModelResponse: - assert self._telemetry is not None - self._telemetry.on_request(telemetry_ctx=telemetry_ctx) + session.telemetry.on_request(telemetry_ctx=telemetry_ctx) final_kwargs = {**call_kwargs, **retry_kwargs} resp = self._transport_call( messages=formatted_messages, @@ -1834,11 +1904,12 @@ def _one_attempt(**retry_kwargs: Any) -> ModelResponse: formatted_messages=formatted_messages, cc_tools=cc_tools, add_security_risk_prediction=add_security_risk_prediction, + session=session, ) return resp try: - return self._build_completion_result(_one_attempt()) + return self._build_completion_result(_one_attempt(), session) except Exception as e: # If the prompt cache content is too small for the provider's # minimum token threshold (e.g., Vertex AI requires ≥4096 tokens), @@ -1854,6 +1925,7 @@ def _one_attempt(**retry_kwargs: Any) -> ModelResponse: tools, add_security_risk_prediction=add_security_risk_prediction, on_token=on_token, + call_context=call_context, **_caller_kwargs, ) # If the credential was rejected (401) and a refresh hook can supply @@ -1881,6 +1953,7 @@ def _one_attempt(**retry_kwargs: Any) -> ModelResponse: call_context=call_context, **_caller_kwargs, ), + session, ) # ========================================================================= @@ -1907,6 +1980,7 @@ async def acompletion( # probe is a no-op for providers without runtime metadata, so repeated # calls are cheap. await self.aresolve_runtime_metadata() + session = self.session_for(call_context) _caller_kwargs = kwargs.copy() enable_streaming = bool(kwargs.get("stream", False)) or self.stream @@ -1939,8 +2013,7 @@ async def acompletion( @self._make_retry_decorator() @self._async_hard_timeout_decorator() async def _one_attempt(**retry_kwargs: Any) -> ModelResponse: - assert self._telemetry is not None - self._telemetry.on_request(telemetry_ctx=telemetry_ctx) + session.telemetry.on_request(telemetry_ctx=telemetry_ctx) final_kwargs = {**call_kwargs, **retry_kwargs} resp = await self._atransport_call( messages=formatted_messages, @@ -1954,11 +2027,12 @@ async def _one_attempt(**retry_kwargs: Any) -> ModelResponse: formatted_messages=formatted_messages, cc_tools=cc_tools, add_security_risk_prediction=add_security_risk_prediction, + session=session, ) return resp try: - return self._build_completion_result(await _one_attempt()) + return self._build_completion_result(await _one_attempt(), session) except Exception as e: # If the prompt cache content is too small for the provider's # minimum token threshold (e.g., Vertex AI requires ≥4096 tokens), @@ -1974,6 +2048,7 @@ async def _one_attempt(**retry_kwargs: Any) -> ModelResponse: tools, add_security_risk_prediction=add_security_risk_prediction, on_token=on_token, + call_context=call_context, **_caller_kwargs, ) # If the credential was rejected (401) and a refresh hook can supply @@ -2007,6 +2082,7 @@ async def _one_attempt(**retry_kwargs: Any) -> ModelResponse: call_context=call_context, **_caller_kwargs, ), + session, ) # ========================================================================= @@ -2046,6 +2122,7 @@ def responses( # Cached (1h TTL) / negative-cached; no-op for providers without runtime # metadata. self.resolve_runtime_metadata() + session = self.session_for(call_context) _caller_kwargs = kwargs.copy() user_enable_streaming = bool(kwargs.get("stream", False)) or self.stream @@ -2081,8 +2158,7 @@ def responses( @self._make_retry_decorator() def _one_attempt(**retry_kwargs: Any) -> ResponsesAPIResponse: - assert self._telemetry is not None - self._telemetry.on_request(telemetry_ctx=telemetry_ctx) + session.telemetry.on_request(telemetry_ctx=telemetry_ctx) final_kwargs = {**call_kwargs, **retry_kwargs} litellm_kwargs = self._build_responses_call_kwargs( input_items, instructions, resp_tools, final_kwargs @@ -2096,7 +2172,7 @@ def _one_attempt(**retry_kwargs: Any) -> ResponsesAPIResponse: "provider returned a non-streaming response; " "no on_token deltas will be emitted." ) - self._telemetry.on_response( + session.telemetry.on_response( ret, provider_info=self._provider_info, ) @@ -2131,13 +2207,13 @@ def _one_attempt(**retry_kwargs: Any) -> ResponsesAPIResponse: completed_response = get_completed_response(ret, completed_response) return self._finalize_stream_response( - completed_response, collected_output_items + completed_response, collected_output_items, session ) raise AssertionError(f"Expected ResponsesAPIResponse, got {type(ret)}") try: - return self._build_responses_result(_one_attempt()) + return self._build_responses_result(_one_attempt(), session) except Exception as e: # If the prompt cache content is too small for the provider's # minimum token threshold (e.g., Vertex AI requires ≥4096 tokens), @@ -2155,6 +2231,7 @@ def _one_attempt(**retry_kwargs: Any) -> ResponsesAPIResponse: store, add_security_risk_prediction=add_security_risk_prediction, on_token=on_token, + call_context=call_context, **_caller_kwargs, ) # If the credential was rejected (401) and a refresh hook can supply @@ -2186,6 +2263,7 @@ def _one_attempt(**retry_kwargs: Any) -> ResponsesAPIResponse: call_context=call_context, **_caller_kwargs, ), + session, ) # ========================================================================= @@ -2211,6 +2289,7 @@ async def aresponses( # the first request so context management reads runtime (not model-level) # limits. No blocking network I/O in-process. await self.aresolve_runtime_metadata() + session = self.session_for(call_context) _caller_kwargs = kwargs.copy() user_enable_streaming = bool(kwargs.get("stream", False)) or self.stream @@ -2249,8 +2328,7 @@ async def aresponses( async def _one_attempt( **retry_kwargs: Any, ) -> ResponsesAPIResponse: - assert self._telemetry is not None - self._telemetry.on_request(telemetry_ctx=telemetry_ctx) + session.telemetry.on_request(telemetry_ctx=telemetry_ctx) final_kwargs = {**call_kwargs, **retry_kwargs} auth_values = await self._aget_litellm_auth_values() litellm_kwargs = self._build_responses_call_kwargs( @@ -2269,7 +2347,7 @@ async def _one_attempt( "provider returned a non-streaming response; " "no on_token deltas will be emitted." ) - self._telemetry.on_response( + session.telemetry.on_response( ret, provider_info=self._provider_info, ) @@ -2323,13 +2401,13 @@ async def _one_attempt( completed_response = get_completed_response(ret, completed_response) return self._finalize_stream_response( - completed_response, collected_output_items + completed_response, collected_output_items, session ) raise AssertionError(f"Expected ResponsesAPIResponse, got {type(ret)}") try: - return self._build_responses_result(await _one_attempt()) + return self._build_responses_result(await _one_attempt(), session) except Exception as e: # If the prompt cache content is too small for the provider's # minimum token threshold (e.g., Vertex AI requires ≥4096 tokens), @@ -2347,6 +2425,7 @@ async def _one_attempt( store, add_security_risk_prediction=add_security_risk_prediction, on_token=on_token, + call_context=call_context, **_caller_kwargs, ) # If the credential was rejected (401) and a refresh hook can supply @@ -2382,6 +2461,7 @@ async def _one_attempt( call_context=call_context, **_caller_kwargs, ), + session, ) # ========================================================================= diff --git a/openhands-sdk/openhands/sdk/llm/llm_registry.py b/openhands-sdk/openhands/sdk/llm/llm_registry.py index 7d1460b85b3..59ae7f991d8 100644 --- a/openhands-sdk/openhands/sdk/llm/llm_registry.py +++ b/openhands-sdk/openhands/sdk/llm/llm_registry.py @@ -3,17 +3,28 @@ from typing import ClassVar from uuid import uuid4 -from pydantic import BaseModel, ConfigDict +from pydantic import BaseModel, ConfigDict, Field from openhands.sdk.llm.llm import LLM +from openhands.sdk.llm.llm_session import LLMSession from openhands.sdk.logger import get_logger logger = get_logger(__name__) +# (log file name, log data, usage ID, model name) +LogCompletionsCallback = Callable[[str, str, str, str], None] + class RegistryEvent(BaseModel): llm: LLM + session: LLMSession | None = Field( + default=None, + description=( + "The session the registry keeps for the LLM's usage ID. Without " + "one the event is about the LLM's own session." + ), + ) model_config: ClassVar[ConfigDict] = ConfigDict( arbitrary_types_allowed=True, @@ -21,15 +32,16 @@ class RegistryEvent(BaseModel): class LLMRegistry: - """A minimal LLM registry for managing LLM instances by usage ID. + """The LLMs of one conversation and their sessions, by role (usage ID). - This registry provides a simple way to manage multiple LLM instances, + The registry maps each usage ID to the LLM that currently fills that role, avoiding the need to recreate LLMs with the same configuration. - The registry also ensures that each registered LLM has independent metrics, - preventing metrics from being shared between LLMs that were created via - model_copy(). This is important for scenarios like creating a condenser LLM - from an agent LLM, where each should track its own usage independently. + It also keeps one :class:`LLMSession` per usage ID. A session holds what the + role accumulates (usage metrics, telemetry callbacks), so it outlives the + LLM object in the slot: replacing or re-adding the LLM for a usage ID keeps + the usage recorded so far. Two LLMs under different usage IDs never share a + session, including when one was created from the other via model_copy(). """ registry_id: str @@ -47,8 +59,9 @@ def __init__( self.registry_id = str(uuid4()) self.retry_listener = retry_listener self._usage_to_llm: dict[str, LLM] = {} - # Track metrics object IDs to detect shared metrics - self._metrics_ids: set[int] = set() + self._sessions: dict[str, LLMSession] = {} + self._stats_update_callback: Callable[[], None] | None = None + self._log_completions_callback: LogCompletionsCallback | None = None self.subscriber: Callable[[RegistryEvent], None] | None = None def subscribe(self, callback: Callable[[RegistryEvent], None]) -> None: @@ -77,41 +90,61 @@ def usage_to_llm(self) -> MappingProxyType[str, LLM]: return MappingProxyType(self._usage_to_llm) - def _ensure_independent_metrics(self, llm: LLM) -> None: - """Ensure the LLM has independent metrics not shared with other LLMs. + @property + def sessions(self) -> MappingProxyType[str, LLMSession]: + """The sessions by usage ID (read-only live view). - When LLMs are created via model_copy(), Pydantic does a shallow copy of - private attributes by default, causing the original and copied LLM to - share the same Metrics object. This method detects such sharing and - resets the metrics to ensure each LLM tracks its own usage independently. + A usage ID keeps its session for the life of the registry, also while + no LLM is registered under it. + """ + return MappingProxyType(self._sessions) - Args: - llm: The LLM instance to check and potentially reset metrics for. + def set_stats_update_callback(self, callback: Callable[[], None] | None) -> None: + """Call ``callback`` whenever any session of this registry records usage. + + It applies to the sessions the registry has now and to those it opens + later, so it does not need setting again when an LLM is switched. + """ + self._stats_update_callback = callback + for session in self._sessions.values(): + session.telemetry.set_stats_update_callback(callback) + + def set_log_completions_callback( + self, callback: LogCompletionsCallback | None + ) -> None: + """Send completion logs to ``callback`` instead of writing them to disk. + + ``callback`` receives the log file name, the log data, the usage ID and + the name of the model that was called. Like + :meth:`set_stats_update_callback`, it covers present and future + sessions; LLMs with ``log_completions`` off produce no logs. """ - # Access the metrics to trigger lazy initialization if needed - metrics = llm.metrics - metrics_id = id(metrics) + self._log_completions_callback = callback + for usage_id, session in self._sessions.items(): + self._apply_log_completions_callback(usage_id, session) - # Check if this metrics object is already tracked by another LLM - if metrics_id in self._metrics_ids: - logger.debug( - f"[LLM registry {self.registry_id}]: Detected shared metrics for " - f"usage '{llm.usage_id}', resetting to independent metrics" - ) - llm.reset_metrics() - # Get the new metrics ID after reset - metrics_id = id(llm.metrics) + def _apply_log_completions_callback( + self, usage_id: str, session: LLMSession + ) -> None: + callback = self._log_completions_callback + telemetry = session.telemetry + if callback is None: + telemetry.set_log_completions_callback(None) + return + + def log_completion(filename: str, log_data: str) -> None: + callback(filename, log_data, usage_id, telemetry.model_name) - # Track this metrics object ID - self._metrics_ids.add(metrics_id) + telemetry.set_log_completions_callback(log_completion) def add(self, llm: LLM) -> None: """Add an LLM instance to the registry. - This method ensures that the LLM has independent metrics before - registering it. If the LLM's metrics are shared with another - registered LLM (e.g., due to model_copy()), fresh metrics will - be created automatically. + The LLM records in the session of its usage ID from then on. The first + LLM added for a usage ID brings its own session along (a new one if it + already serves another conversation). The registry keeps that session + afterwards, so an LLM added later for the same usage ID continues from + the metrics already recorded. Args: llm: The LLM instance to register. @@ -128,11 +161,27 @@ def add(self, llm: LLM) -> None: ) raise ValueError(message) - # Ensure this LLM has independent metrics before registering - self._ensure_independent_metrics(llm) + session = self._sessions.get(usage_id) + if session is None: + session = llm.session + # One session per role: an LLM whose session is spoken for (it + # serves another conversation, or shares it with a registered LLM + # it was copied from) gets a new one here. + if llm._registered_session() is not None or any( + session is other for other in self._sessions.values() + ): + session = LLMSession(llm) + self._sessions[usage_id] = session + # Callbacks the LLM already carries stay unless the registry has + # its own to install. + if self._stats_update_callback is not None: + session.telemetry.set_stats_update_callback(self._stats_update_callback) + if self._log_completions_callback is not None: + self._apply_log_completions_callback(usage_id, session) + llm._join_registry_session(session) self._usage_to_llm[usage_id] = llm - self.notify(RegistryEvent(llm=llm)) + self.notify(RegistryEvent(llm=llm, session=session)) logger.debug( f"[LLM registry {self.registry_id}]: Added LLM for usage {usage_id}" ) @@ -165,18 +214,14 @@ def remove(self, usage_id: str) -> None: Unlike :meth:`add` and :meth:`get`, this is a no-op when ``usage_id`` is absent, so callers can invalidate a lazily-cached slot without - first checking for it. + first checking for it. The session of the usage ID stays, so the usage + it recorded is not lost. Args: usage_id: Usage slot to drop from the registry. """ llm = self._usage_to_llm.pop(usage_id, None) if llm is not None: - # The slot no longer holds this instance, so its metrics are no - # longer shared with anyone here. Keeping the id would make a later - # add() mistake the same instance for a shared-metrics copy and - # reset the usage it has already accumulated. - self._metrics_ids.discard(id(llm.metrics)) logger.debug( f"[LLM registry {self.registry_id}]: Removed LLM for usage {usage_id}" ) diff --git a/openhands-sdk/openhands/sdk/llm/llm_session.py b/openhands-sdk/openhands/sdk/llm/llm_session.py new file mode 100644 index 00000000000..6947cf7f4f6 --- /dev/null +++ b/openhands-sdk/openhands/sdk/llm/llm_session.py @@ -0,0 +1,102 @@ +"""Runtime state of an LLM, kept apart from its configuration.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from openhands.sdk.llm.utils.metrics import Metrics +from openhands.sdk.llm.utils.telemetry import Telemetry + + +if TYPE_CHECKING: + from openhands.sdk.llm.llm import LLM + + +class LLMSession: + """What accumulates while an LLM is used: usage metrics and telemetry. + + An :class:`LLM` describes how to call a model. A session holds what calling + it produces, so one configuration can serve several users without their + usage, callbacks or in-flight request state running together. + + There are two kinds of owner: + + * every ``LLM`` has a session of its own, behind ``llm.metrics`` and + ``llm.telemetry``. It counts all the usage of that object. + * a conversation's :class:`LLMRegistry` keeps one session per role + (``usage_id``). A call made with that conversation's + :class:`LLMCallContext` records there, whichever ``LLM`` object fills the + role at the time. + + The two are the same session while an LLM serves a single conversation: + the registry takes over the session of the first LLM it gets for a role. + They come apart when one LLM object is used by a second conversation, so + that neither conversation sees the other's usage. + """ + + __slots__ = ("_telemetry",) + + def __init__(self, llm: LLM) -> None: + self._telemetry: Telemetry = Telemetry( + model_name=llm.model, + log_enabled=llm.log_completions, + log_dir=_log_dir(llm), + input_cost_per_token=llm.input_cost_per_token, + output_cost_per_token=llm.output_cost_per_token, + metrics=Metrics(model_name=llm.model), + ) + + def __eq__(self, other: object) -> bool: + # By value, as the metrics and telemetry it holds compare: two LLMs + # with equal configuration and equal usage are equal. + if not isinstance(other, LLMSession): + return NotImplemented + return self._telemetry == other._telemetry + + @property + def metrics(self) -> Metrics: + """Usage recorded in this session.""" + return self._telemetry.metrics + + @property + def telemetry(self) -> Telemetry: + """Telemetry handler that records into :attr:`metrics`.""" + return self._telemetry + + def follow(self, llm: LLM) -> None: + """Serve ``llm``, the LLM now calling in this session's role. + + The metrics and the telemetry callbacks belong to the role and stay. + The settings telemetry takes from an LLM follow the caller, and unless + this is the LLM's own session, ``llm.metrics`` also counts what is + recorded here. + """ + telemetry = self._telemetry + telemetry.model_name = llm.model + telemetry.log_enabled = llm.log_completions + telemetry.log_dir = _log_dir(llm) + telemetry.input_cost_per_token = llm.input_cost_per_token + telemetry.output_cost_per_token = llm.output_cost_per_token + + own = llm.session.metrics + self.metrics.roll_up_into(None if own is self.metrics else own) + + def restore_metrics(self, metrics: Metrics) -> None: + """Continue from ``metrics``, e.g. those persisted with a conversation. + + The session records into the given object from now on. What that object + already holds is added once to the ledger this session rolls up into. + """ + current = self.metrics + if metrics is current: + return + target = current.rolls_up_into + current.roll_up_into(None) + if target is not None and target is not metrics: + target.merge(metrics) + metrics.roll_up_into(target) + self._telemetry.metrics = metrics + + +def _log_dir(llm: LLM) -> str | None: + return llm.log_completions_folder if llm.log_completions else None diff --git a/openhands-sdk/openhands/sdk/llm/utils/metrics.py b/openhands-sdk/openhands/sdk/llm/utils/metrics.py index 732af4c6213..720fb27b469 100644 --- a/openhands-sdk/openhands/sdk/llm/utils/metrics.py +++ b/openhands-sdk/openhands/sdk/llm/utils/metrics.py @@ -117,8 +117,15 @@ class Metrics(MetricsSnapshot): - max_budget_per_task (budget limit) - A list of ResponseLatency - A list of TokenUsage (one per call). + + A ledger can roll up into another one (see :meth:`roll_up_into`): what a + conversation records for a role is also counted on the ``LLM`` it used. """ + # The roll-up target is wiring, not data. A slot keeps it out of what + # pydantic serializes, compares and copies, so a copy is a plain snapshot. + __slots__ = ("_rollup",) + costs: list[Cost] = Field( default_factory=list, description="List of individual costs" ) @@ -162,11 +169,31 @@ def get_snapshot(self) -> MetricsSnapshot: else None, ) + def roll_up_into(self, target: "Metrics | None") -> None: + """Also record into ``target`` whatever is recorded here from now on. + + What this ledger already holds is not copied over. ``None`` detaches. + """ + if target is self: + raise ValueError("Metrics cannot roll up into itself.") + self._rollup = target + + @property + def rolls_up_into(self) -> "Metrics | None": + """The ledger that also records what this one does, if any.""" + try: + return self._rollup + except AttributeError: # never attached: the slot is still empty + return None + def add_cost(self, value: float) -> None: if value < 0: raise ValueError("Added cost cannot be negative.") self.accumulated_cost += value self.costs.append(Cost(cost=value, model=self.model_name)) + target = self.rolls_up_into + if target is not None: + target.add_cost(value) def add_response_latency(self, value: float, response_id: str) -> None: self.response_latencies.append( @@ -174,6 +201,9 @@ def add_response_latency(self, value: float, response_id: str) -> None: latency=max(0.0, value), model=self.model_name, response_id=response_id ) ) + target = self.rolls_up_into + if target is not None: + target.add_response_latency(value, response_id) def add_token_usage( self, @@ -219,8 +249,24 @@ def add_token_usage( else: self.accumulated_token_usage = self.accumulated_token_usage + new_usage + target = self.rolls_up_into + if target is not None: + target.add_token_usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + cache_read_tokens=cache_read_tokens, + cache_write_tokens=cache_write_tokens, + context_window=context_window, + response_id=response_id, + reasoning_tokens=reasoning_tokens, + ) + def merge(self, other: "Metrics") -> None: """Merge 'other' metrics into this one.""" + target = self.rolls_up_into + if target is not None: + target.merge(other) + self.accumulated_cost += other.accumulated_cost # Keep the max_budget_per_task from other if it's set and this one isn't diff --git a/openhands-sdk/openhands/sdk/llm/utils/telemetry.py b/openhands-sdk/openhands/sdk/llm/utils/telemetry.py index 3a0c61eafd0..d2d7c71ce28 100644 --- a/openhands-sdk/openhands/sdk/llm/utils/telemetry.py +++ b/openhands-sdk/openhands/sdk/llm/utils/telemetry.py @@ -199,6 +199,15 @@ def set_stats_update_callback(self, callback: Callable[[], None] | None) -> None """ self._stats_update_callback = callback + def notify_stats_update(self) -> None: + """Tell the stats-update callback, if any, that the metrics changed.""" + if self._stats_update_callback is None: + return + try: + self._stats_update_callback() + except Exception: + logger.exception("Stats update callback failed", exc_info=True) + def on_request(self, telemetry_ctx: dict | None) -> None: self._req_start = time.time() self._req_ctx = _redact_request_credentials(telemetry_ctx or {}) @@ -243,11 +252,7 @@ def on_response( self._close_span() # 6) notify about stats update - if self._stats_update_callback is not None: - try: - self._stats_update_callback() - except Exception: - logger.exception("Stats update callback failed", exc_info=True) + self.notify_stats_update() return self.metrics.deep_copy() diff --git a/openhands-sdk/openhands/sdk/settings/model.py b/openhands-sdk/openhands/sdk/settings/model.py index 6fe64c17d4a..1c2de970d46 100644 --- a/openhands-sdk/openhands/sdk/settings/model.py +++ b/openhands-sdk/openhands/sdk/settings/model.py @@ -318,7 +318,6 @@ def build_condenser(self, llm: LLM) -> LLMSummarizingCondenser | None: from openhands.sdk.context.condenser import LLMSummarizingCondenser condenser_llm = llm.model_copy(update={"usage_id": "condenser"}) - condenser_llm.reset_metrics() condenser_kwargs = self.model_dump( exclude={"enabled", "condenser_kind"}, exclude_none=True, diff --git a/openhands-sdk/openhands/sdk/subagent/registry.py b/openhands-sdk/openhands/sdk/subagent/registry.py index 0f0c99fa2d1..350a06fd65c 100644 --- a/openhands-sdk/openhands/sdk/subagent/registry.py +++ b/openhands-sdk/openhands/sdk/subagent/registry.py @@ -256,14 +256,14 @@ def _factory(llm: "LLM") -> "Agent": # Sub-agents get a summarizing condenser by default (parity with the # top-level agent) so deep runs auto-compact instead of erroring on context - # overflow. The condenser LLM needs a distinct usage_id or its tokens get - # deduped out of conversation stats. A NoOpCondenser disables condensation. + # overflow. The condenser LLM needs a distinct usage_id or its tokens are + # reported together with the agent's. A NoOpCondenser disables condensation. if agent_def.condenser is not None: condenser = agent_def.condenser if isinstance(condenser, LLMSummarizingCondenser): # Freshen per spawn: a distinct LLM with its own metrics (so # sub-agents don't share/double-count condenser cost) and a - # usage_id distinct from the agent's, else its tokens get deduped. + # usage_id distinct from the agent's, so its tokens show apart. cond_llm = condenser.llm usage_id = ( cond_llm.usage_id diff --git a/tests/agent_server/test_event_service.py b/tests/agent_server/test_event_service.py index 44feacddb3e..19102700de0 100644 --- a/tests/agent_server/test_event_service.py +++ b/tests/agent_server/test_event_service.py @@ -3585,15 +3585,8 @@ def _make_service_with_callback(self): service._emit_event_from_thread = MagicMock(name="_emit_event_from_thread") callbacks: list = [] - service._setup_stats_streaming( - MagicMock( - get_all_llms=lambda: [ - MagicMock( - telemetry=MagicMock(set_stats_update_callback=callbacks.append) - ) - ] - ) - ) + conversation.llm_registry.set_stats_update_callback = callbacks.append + service._setup_stats_streaming(conversation) assert len(callbacks) == 1, "stats_callback must be registered exactly once" return service, state, callbacks[0] @@ -3908,8 +3901,8 @@ def test_llm_log_callback_swallows_emit_failures( event_service: EventService, caplog ) -> None: callbacks = [] - llm = MagicMock(log_completions=True, usage_id="test-usage", model="gpt-4o") - llm.telemetry.set_log_completions_callback.side_effect = callbacks.append + conversation = MagicMock() + conversation.llm_registry.set_log_completions_callback = callbacks.append object.__setattr__( event_service, @@ -3918,8 +3911,8 @@ def test_llm_log_callback_swallows_emit_failures( ) with caplog.at_level("ERROR"): - event_service._setup_llm_log_streaming(MagicMock(get_all_llms=lambda: [llm])) - callbacks[0]("completion.json", "{}") + event_service._setup_llm_log_streaming(conversation) + callbacks[0]("completion.json", "{}", "test-usage", "gpt-4o") emit_mock = cast(MagicMock, event_service._emit_event_from_thread) emit_mock.assert_called_once() diff --git a/tests/sdk/context/condenser/test_llm_summarizing_condenser.py b/tests/sdk/context/condenser/test_llm_summarizing_condenser.py index 06099406b30..ecaace6e400 100644 --- a/tests/sdk/context/condenser/test_llm_summarizing_condenser.py +++ b/tests/sdk/context/condenser/test_llm_summarizing_condenser.py @@ -93,8 +93,7 @@ def create_completion_result(content: str) -> LLMResponse: mock_llm.input_cost_per_token = None mock_llm.output_cost_per_token = None - mock_llm._metrics = None - mock_llm._telemetry = None + mock_llm._session = None # Helper method to set mock response content def set_mock_response_content(content: str): diff --git a/tests/sdk/conversation/local/test_llm_session_isolation.py b/tests/sdk/conversation/local/test_llm_session_isolation.py new file mode 100644 index 00000000000..7f851d66789 --- /dev/null +++ b/tests/sdk/conversation/local/test_llm_session_isolation.py @@ -0,0 +1,186 @@ +"""A conversation's LLM usage and callbacks belong to it, per usage_id. + +The usage metrics and telemetry callbacks used to sit on the ``LLM`` object, so +they went wherever that object went: into a second conversation that reused +the LLM, and away from the conversation when the LLM was switched. +""" + +from collections.abc import Iterator +from pathlib import Path +from unittest.mock import patch + +import pytest +from litellm.types.utils import Choices, Message as LiteLLMMessage, ModelResponse, Usage +from pydantic import SecretStr + +from openhands.sdk.agent import Agent +from openhands.sdk.context.condenser import LLMSummarizingCondenser +from openhands.sdk.conversation.impl.local_conversation import ( + ASK_AGENT_LLM_USAGE_ID, + LocalConversation, +) +from openhands.sdk.llm import LLM, Message, TextContent + + +PROMPT_TOKENS = 100 + + +def _llm(model: str = "gpt-4o") -> LLM: + return LLM(model=model, api_key=SecretStr("k"), usage_id="agent") + + +def _conversation(tmp_path: Path, llm: LLM, name: str = "conv") -> LocalConversation: + condenser = LLMSummarizingCondenser( + llm=llm.model_copy(update={"usage_id": "condenser"}), max_size=80 + ) + return LocalConversation( + agent=Agent(llm=llm, tools=[], condenser=condenser), + workspace=tmp_path / name, + visualizer=None, + ) + + +def _run(conversation: LocalConversation, text: str = "hi") -> None: + conversation.send_message(text) + conversation.run() + + +def _prompt_tokens(conversation: LocalConversation, usage_id: str = "agent") -> int: + usage = conversation.conversation_stats.usage_to_metrics[ + usage_id + ].accumulated_token_usage + assert usage is not None + return usage.prompt_tokens + + +@pytest.fixture(autouse=True) +def fake_completion() -> Iterator[None]: + def respond(*args, **kwargs) -> ModelResponse: + return ModelResponse( + id="resp", + choices=[ + Choices( + index=0, + finish_reason="stop", + message=LiteLLMMessage(role="assistant", content="done"), + ) + ], + model="gpt-4o", + usage=Usage( + prompt_tokens=PROMPT_TOKENS, completion_tokens=1, total_tokens=101 + ), + ) + + with patch("openhands.sdk.llm.llm.litellm_completion", side_effect=respond): + yield + + +def test_conversations_sharing_an_llm_keep_their_usage_apart(tmp_path): + llm = _llm() + first = _conversation(tmp_path, llm, "first") + second = _conversation(tmp_path, llm, "second") + + _run(first) + _run(second) + _run(second) + + assert _prompt_tokens(first) == PROMPT_TOKENS + assert _prompt_tokens(second) == 2 * PROMPT_TOKENS + # The LLM object still reports everything that went through it. + assert llm.metrics.accumulated_token_usage is not None + assert llm.metrics.accumulated_token_usage.prompt_tokens == 3 * PROMPT_TOKENS + + +def test_conversations_sharing_an_llm_keep_their_callbacks_apart(tmp_path): + llm = _llm() + first = _conversation(tmp_path, llm, "first") + second = _conversation(tmp_path, llm, "second") + fired: list[str] = [] + first.llm_registry.set_stats_update_callback(lambda: fired.append("first")) + second.llm_registry.set_stats_update_callback(lambda: fired.append("second")) + + _run(first) + _run(second) + + assert fired == ["first", "second"] + + +def test_single_conversation_reports_through_the_llm_object(tmp_path): + """The common case is unchanged: ``llm.metrics`` is the conversation's ledger.""" + llm = _llm() + conversation = _conversation(tmp_path, llm) + + _run(conversation) + + assert conversation.conversation_stats.usage_to_metrics["agent"] is llm.metrics + assert _prompt_tokens(conversation) == PROMPT_TOKENS + + +def test_stats_callback_keeps_firing_after_switch_llm(tmp_path): + conversation = _conversation(tmp_path, _llm()) + fired: list[str] = [] + conversation.llm_registry.set_stats_update_callback(lambda: fired.append("stats")) + _run(conversation) + + conversation.switch_llm(_llm("gpt-4o-mini")) + _run(conversation, "after the switch") + + assert fired == ["stats", "stats"] + assert _prompt_tokens(conversation) == 2 * PROMPT_TOKENS + + +def test_condenser_usage_is_counted_after_switch_llm(tmp_path): + conversation = _conversation(tmp_path, _llm()) + _run(conversation) + assert _prompt_tokens(conversation, "condenser") == 0 + + conversation.switch_llm(_llm("gpt-4o-mini")) + condenser = conversation.agent.condenser + assert isinstance(condenser, LLMSummarizingCondenser) + assert condenser.llm.model == "gpt-4o-mini" + condenser.llm.completion( + [Message(role="user", content=[TextContent(text="summarise")])], + call_context=conversation.get_llm_call_context(), + ) + + assert _prompt_tokens(conversation, "condenser") == PROMPT_TOKENS + + +def test_ask_agent_usage_is_counted_after_every_switch(tmp_path): + conversation = _conversation(tmp_path, _llm()) + _run(conversation) + + counted = [] + for model in ("gpt-4o", "gpt-4o-mini", "gpt-4o"): + conversation.switch_llm(_llm(model)) + conversation.ask_agent("what are you doing?") + counted.append(_prompt_tokens(conversation, ASK_AGENT_LLM_USAGE_ID)) + + assert counted == [PROMPT_TOKENS, 2 * PROMPT_TOKENS, 3 * PROMPT_TOKENS] + + +def test_resumed_conversation_continues_from_its_persisted_usage(tmp_path): + persistence = tmp_path / "persist" + workspace = tmp_path / "ws" + first = LocalConversation( + agent=Agent(llm=_llm(), tools=[]), + workspace=workspace, + persistence_dir=persistence, + visualizer=None, + ) + _run(first) + conversation_id = first.id + first.close() + + llm = _llm() + resumed = LocalConversation( + agent=Agent(llm=llm, tools=[]), + workspace=workspace, + persistence_dir=persistence, + conversation_id=conversation_id, + visualizer=None, + ) + _run(resumed, "again") + + assert _prompt_tokens(resumed) == 2 * PROMPT_TOKENS + assert llm.metrics is resumed.conversation_stats.usage_to_metrics["agent"] diff --git a/tests/sdk/conversation/test_switch_model.py b/tests/sdk/conversation/test_switch_model.py index bbe1b7d807c..b450a94ec45 100644 --- a/tests/sdk/conversation/test_switch_model.py +++ b/tests/sdk/conversation/test_switch_model.py @@ -473,7 +473,7 @@ def test_switch_llm_refreshes_llm_condenser_credentials( assert state_condenser_llm.model == condenser_llm.model assert state_condenser_llm.api_key == condenser_llm.api_key assert condenser_llm.metrics is not conv.agent.llm.metrics - assert condenser_llm._telemetry is not None + assert condenser_llm.telemetry.metrics is condenser_llm.metrics async def _fake_acompletion(**kwargs): return create_mock_litellm_response( diff --git a/tests/sdk/llm/test_llm.py b/tests/sdk/llm/test_llm.py index 6279575d2e8..b3888786963 100644 --- a/tests/sdk/llm/test_llm.py +++ b/tests/sdk/llm/test_llm.py @@ -1799,8 +1799,7 @@ def test_llm_reset_metrics(): llm.reset_metrics() # Verify new metrics are created - assert llm._metrics is not None - assert llm._telemetry is not None + assert llm.telemetry.metrics is llm.metrics assert llm.metrics is not original_metrics assert llm.telemetry is not original_telemetry assert llm.metrics.accumulated_cost == 0.0 @@ -1867,14 +1866,14 @@ def test_stale_metrics_not_updated(llm): assert llm.metrics.accumulated_cost == 2.75 -def test_restore_metrics_telemetry_none(): - """restore_metrics() must not crash when telemetry has not been initialized.""" +def test_restore_metrics_session_none(): + """restore_metrics() must not crash when the session has not been created.""" llm = LLM( model="openai/gpt-4o", api_key=SecretStr("test-key"), usage_id="test-service", ) - llm._telemetry = None + llm._session = None restored = Metrics(model_name="openai/gpt-4o") restored.add_cost(1.00) @@ -1926,13 +1925,12 @@ def test_telemetry_callback_preserved_across_revalidation(): ) fired: list[bool] = [] llm.telemetry.set_stats_update_callback(lambda: fired.append(True)) - telemetry_before = llm._telemetry + telemetry_before = llm.telemetry RegistryEvent(llm=llm) - assert llm._telemetry is telemetry_before - assert llm.telemetry._stats_update_callback is not None - llm.telemetry._stats_update_callback() + assert llm.telemetry is telemetry_before + llm.telemetry.notify_stats_update() assert fired == [True] diff --git a/tests/sdk/llm/test_llm_metrics.py b/tests/sdk/llm/test_llm_metrics.py index 95bb6192d36..ecbb4936307 100644 --- a/tests/sdk/llm/test_llm_metrics.py +++ b/tests/sdk/llm/test_llm_metrics.py @@ -642,3 +642,53 @@ def test_cache_hit_rate_is_not_serialized(): snapshot = Metrics().get_snapshot() assert "cache_hit_rate" not in snapshot.model_dump() assert "cache_hit_rate" not in MetricsSnapshot.model_json_schema()["properties"] + + +def test_metrics_roll_up_into_records_in_both_ledgers(): + """What a ledger records after it is attached is also added to its target.""" + total = Metrics(model_name="gpt-4o") + total.add_cost(1.0) + ledger = Metrics(model_name="gpt-4o") + ledger.add_cost(0.5) # recorded before attaching: stays local + + ledger.roll_up_into(total) + ledger.add_cost(2.0) + ledger.add_token_usage(100, 50, 10, 5, 4096, "resp-1") + ledger.add_response_latency(1.5, "resp-1") + other = Metrics(model_name="gpt-4o") + other.add_cost(4.0) + ledger.merge(other) + + assert ledger.rolls_up_into is total + assert ledger.accumulated_cost == 6.5 + assert total.accumulated_cost == 7.0 + assert total.accumulated_token_usage is not None + assert total.accumulated_token_usage.prompt_tokens == 100 + assert [latency.response_id for latency in total.response_latencies] == ["resp-1"] + + # One direction only, and it can be detached again. + total.add_cost(1.0) + assert ledger.accumulated_cost == 6.5 + ledger.roll_up_into(None) + ledger.add_cost(1.0) + assert total.accumulated_cost == 8.0 + + +def test_metrics_roll_up_target_is_wiring_not_data(): + """The target is not serialized, compared or carried over to copies.""" + total = Metrics(model_name="gpt-4o") + ledger = Metrics(model_name="gpt-4o") + ledger.roll_up_into(total) + ledger.add_cost(1.0) + + same_data = Metrics.model_validate(ledger.model_dump()) + assert same_data.rolls_up_into is None + assert ledger == same_data + + for copied in (ledger.deep_copy(), ledger.model_copy()): + assert copied.rolls_up_into is None + copied.add_cost(5.0) + assert total.accumulated_cost == 1.0 + + with pytest.raises(ValueError, match="roll up into itself"): + ledger.roll_up_into(ledger) diff --git a/tests/sdk/llm/test_llm_registry.py b/tests/sdk/llm/test_llm_registry.py index 39fc1388cc6..b359f94c307 100644 --- a/tests/sdk/llm/test_llm_registry.py +++ b/tests/sdk/llm/test_llm_registry.py @@ -1,12 +1,16 @@ from __future__ import annotations import unittest -from unittest.mock import MagicMock, Mock, patch +from unittest.mock import MagicMock, patch from openhands.sdk.llm.llm import LLM from openhands.sdk.llm.llm_registry import LLMRegistry, RegistryEvent +def _llm(usage_id: str) -> LLM: + return LLM(model="gpt-4o", usage_id=usage_id) + + class TestLLMRegistry(unittest.TestCase): def setUp(self): """Set up test environment before each test.""" @@ -23,19 +27,17 @@ def callback(event: RegistryEvent): # Subscribe to events self.registry.subscribe(callback) - # Create a mock LLM and add it to trigger notification - mock_llm = Mock(spec=LLM) - mock_llm.usage_id = "notify-service" + # Add an LLM to trigger notification + llm = _llm("notify-service") + self.registry.add(llm) - # Mock the RegistryEvent to avoid LLM attribute access - with patch( - "openhands.sdk.llm.llm_registry.RegistryEvent" - ) as mock_registry_event: - mock_registry_event.return_value = Mock() - self.registry.add(mock_llm) - - # Should receive notification for the newly added LLM + # Should receive notification for the newly added LLM, with the + # session the registry keeps for its usage ID self.assertEqual(len(events_received), 1) + self.assertIs(events_received[0].llm, llm) + self.assertIs( + events_received[0].session, self.registry.sessions["notify-service"] + ) # Test that the subscriber is set correctly self.assertIsNotNone(self.registry.subscriber) @@ -67,7 +69,7 @@ def failing_subscriber(event): # Mock the logger to capture warning messages with patch("openhands.sdk.llm.llm_registry.logger") as mock_logger: # Create a mock event - mock_event = Mock() + mock_event = MagicMock() # This should handle the exception and log a warning (lines 146-147) registry.notify(mock_event) @@ -82,26 +84,16 @@ def test_llm_registry_list_usage_ids(): registry = LLMRegistry() - # Create mock LLM objects - mock_llm1 = Mock(spec=LLM) - mock_llm1.usage_id = "service1" - mock_llm2 = Mock(spec=LLM) - mock_llm2.usage_id = "service2" - - # Mock the RegistryEvent to avoid LLM attribute access - with patch("openhands.sdk.llm.llm_registry.RegistryEvent") as mock_registry_event: - mock_registry_event.return_value = Mock() + # Add some LLMs using the new API + registry.add(_llm("service1")) + registry.add(_llm("service2")) - # Add some LLMs using the new API - registry.add(mock_llm1) - registry.add(mock_llm2) + # Test list_usage_ids + usage_ids = registry.list_usage_ids() - # Test list_usage_ids - usage_ids = registry.list_usage_ids() - - assert "service1" in usage_ids - assert "service2" in usage_ids - assert len(usage_ids) == 2 + assert "service1" in usage_ids + assert "service2" in usage_ids + assert len(usage_ids) == 2 def test_llm_registry_remove_method(): @@ -109,12 +101,7 @@ def test_llm_registry_remove_method(): registry = LLMRegistry() - mock_llm = Mock(spec=LLM) - mock_llm.usage_id = "service1" - - with patch("openhands.sdk.llm.llm_registry.RegistryEvent") as mock_registry_event: - mock_registry_event.return_value = Mock() - registry.add(mock_llm) + registry.add(_llm("service1")) assert "service1" in registry.list_usage_ids() @@ -144,28 +131,19 @@ def test_llm_registry_add_method(): """Test the new add() method for LLMRegistry.""" registry = LLMRegistry() - # Create a mock LLM - mock_llm = Mock(spec=LLM) - mock_llm.usage_id = "test-service" - service_id = mock_llm.usage_id - - # Mock the RegistryEvent to avoid LLM attribute access - with patch("openhands.sdk.llm.llm_registry.RegistryEvent") as mock_registry_event: - mock_registry_event.return_value = Mock() + llm = _llm("test-service") + service_id = llm.usage_id - # Test adding an LLM - registry.add(mock_llm) - - # Verify the LLM was added - assert service_id in registry.usage_to_llm - assert registry.usage_to_llm[service_id] is mock_llm + # Test adding an LLM + registry.add(llm) - # Verify RegistryEvent was called - mock_registry_event.assert_called_once_with(llm=mock_llm) + # Verify the LLM was added + assert service_id in registry.usage_to_llm + assert registry.usage_to_llm[service_id] is llm # Test that adding the same usage_id raises ValueError with unittest.TestCase().assertRaises(ValueError) as context: - registry.add(mock_llm) + registry.add(llm) assert "already exists in registry" in str(context.exception) @@ -174,21 +152,15 @@ def test_llm_registry_get_method(): """Test the new get() method for LLMRegistry.""" registry = LLMRegistry() - # Create a mock LLM - mock_llm = Mock(spec=LLM) - mock_llm.usage_id = "test-service" - service_id = mock_llm.usage_id + llm = _llm("test-service") + service_id = llm.usage_id - # Mock the RegistryEvent to avoid LLM attribute access - with patch("openhands.sdk.llm.llm_registry.RegistryEvent") as mock_registry_event: - mock_registry_event.return_value = Mock() - - # Add the LLM first - registry.add(mock_llm) + # Add the LLM first + registry.add(llm) - # Test getting the LLM - retrieved_llm = registry.get(service_id) - assert retrieved_llm is mock_llm + # Test getting the LLM + retrieved_llm = registry.get(service_id) + assert retrieved_llm is llm # Test getting non-existent service raises KeyError with unittest.TestCase().assertRaises(KeyError) as context: @@ -201,33 +173,26 @@ def test_llm_registry_add_get_workflow(): """Test the complete add/get workflow.""" registry = LLMRegistry() - # Create mock LLMs - llm1 = Mock(spec=LLM) - llm1.usage_id = "service1" - llm2 = Mock(spec=LLM) - llm2.usage_id = "service2" + llm1 = _llm("service1") + llm2 = _llm("service2") - # Mock the RegistryEvent to avoid LLM attribute access - with patch("openhands.sdk.llm.llm_registry.RegistryEvent") as mock_registry_event: - mock_registry_event.return_value = Mock() - - # Add multiple LLMs - registry.add(llm1) - registry.add(llm2) + # Add multiple LLMs + registry.add(llm1) + registry.add(llm2) - # Verify we can retrieve them - assert registry.get("service1") is llm1 - assert registry.get("service2") is llm2 + # Verify we can retrieve them + assert registry.get("service1") is llm1 + assert registry.get("service2") is llm2 - # Verify list_usage_ids works - usage_ids = registry.list_usage_ids() - assert "service1" in usage_ids - assert "service2" in usage_ids - assert len(usage_ids) == 2 + # Verify list_usage_ids works + usage_ids = registry.list_usage_ids() + assert "service1" in usage_ids + assert "service2" in usage_ids + assert len(usage_ids) == 2 - # Verify usage_id is set correctly - assert llm1.usage_id == "service1" - assert llm2.usage_id == "service2" + # Verify usage_id is set correctly + assert llm1.usage_id == "service1" + assert llm2.usage_id == "service2" def test_llm_registry_ensures_independent_metrics_for_copied_llms(): @@ -251,21 +216,18 @@ def test_llm_registry_ensures_independent_metrics_for_copied_llms(): usage_id="original-llm", ) - # Copy with updated usage_id (simulates creating a condenser LLM) - # Note: model_copy() does a shallow copy of private attributes by default, - # so the copied LLM shares the same metrics object as the original + # Copy with updated usage_id (simulates creating a condenser LLM). A copy + # made for another usage ID starts its own session, so it never shares the + # original's metrics, registered or not. copied = original.model_copy(update={"usage_id": "copied-llm"}) + assert original.metrics is not copied.metrics - # Before registering, they share the same metrics (this is the bug we're fixing) - assert original.metrics is copied.metrics - - # Register both LLMs - the registry should detect and fix shared metrics registry.add(original) registry.add(copied) - # After registering, they should have different metrics objects + # They still have different metrics objects after registering assert original.metrics is not copied.metrics - assert id(original.metrics) != id(copied.metrics) + assert registry.sessions["original-llm"] is not registry.sessions["copied-llm"] # Verify metrics are independent - changes to one don't affect the other original.metrics.add_cost(1.0) @@ -294,19 +256,37 @@ def test_llm_registry_ensures_independent_telemetry_for_copied_llms(): usage_id="original-llm", ) - # Copy with updated usage_id + # Copy with updated usage_id: its own session means its own telemetry copied = original.model_copy(update={"usage_id": "copied-llm"}) - - # Before registering, they share the same telemetry - assert original.telemetry is copied.telemetry + assert original.telemetry is not copied.telemetry + assert copied.telemetry.metrics is copied.metrics # Register both LLMs registry.add(original) registry.add(copied) - # After registering, they should have different telemetry objects + # They still have different telemetry objects after registering assert original.telemetry is not copied.telemetry - assert id(original.telemetry) != id(copied.telemetry) + + +def test_llm_registry_separates_llms_that_share_a_session(): + """Two usage IDs never get one session, however the LLMs came to share it. + + A plain copy keeps the session of its source. Renaming it afterwards is + the one way left to bring a shared session to the registry under two IDs. + """ + registry = LLMRegistry() + original = _llm("original-llm") + renamed = original.model_copy() + renamed.usage_id = "renamed-llm" + assert renamed.metrics is original.metrics + + registry.add(original) + registry.add(renamed) + + assert registry.sessions["original-llm"] is not registry.sessions["renamed-llm"] + original.metrics.add_cost(1.0) + assert renamed.metrics.accumulated_cost == 0.0 def test_llm_registry_does_not_reset_metrics_for_independent_llms(): diff --git a/tests/sdk/llm/test_llm_serialization.py b/tests/sdk/llm/test_llm_serialization.py index a5fd013a1e0..b5ebab4846c 100644 --- a/tests/sdk/llm/test_llm_serialization.py +++ b/tests/sdk/llm/test_llm_serialization.py @@ -115,7 +115,7 @@ def test_llm_private_attributes_not_serialized() -> None: # Private attributes should not be present assert "_model_info" not in llm_dict assert "_tokenizer" not in llm_dict - assert "_telemetry" not in llm_dict + assert "_session" not in llm_dict # Serialize to JSON and deserialize llm_json = llm.model_dump_json() @@ -127,8 +127,8 @@ def test_llm_private_attributes_not_serialized() -> None: assert deserialized_llm._tokenizer is None assert deserialized_llm.native_tool_calling is True assert ( - deserialized_llm._telemetry is not None - ) # LLM creates telemetry automatically + deserialized_llm._session is not None + ) # LLM creates its session (metrics + telemetry) automatically assert deserialized_llm.model_dump() == llm.model_dump() diff --git a/tests/sdk/llm/test_llm_session.py b/tests/sdk/llm/test_llm_session.py new file mode 100644 index 00000000000..0a506dccd6a --- /dev/null +++ b/tests/sdk/llm/test_llm_session.py @@ -0,0 +1,284 @@ +"""Tests for LLMSession: where an LLM's usage metrics and telemetry live.""" + +from collections.abc import Iterator +from unittest.mock import patch + +import pytest +from litellm.types.utils import Choices, Message as LiteLLMMessage, ModelResponse, Usage +from pydantic import SecretStr + +from openhands.sdk.llm import ( + LLM, + LLMCallContext, + LLMRegistry, + LLMSession, + Message, + TextContent, +) +from openhands.sdk.llm.call_context import llm_call_context_scope + + +PROMPT_TOKENS = 100 + + +def _llm(usage_id: str = "agent", model: str = "gpt-4o", **kwargs) -> LLM: + return LLM(model=model, api_key=SecretStr("k"), usage_id=usage_id, **kwargs) + + +def _context(registry: LLMRegistry) -> LLMCallContext: + return LLMCallContext().for_conversation("conv", sessions=registry.sessions) + + +def _call(llm: LLM, call_context: LLMCallContext | None = None) -> None: + llm.completion( + [Message(role="user", content=[TextContent(text="hi")])], + call_context=call_context, + ) + + +def _prompt_tokens(session: LLMSession) -> int: + usage = session.metrics.accumulated_token_usage + assert usage is not None + return usage.prompt_tokens + + +@pytest.fixture(autouse=True) +def fake_completion() -> Iterator[None]: + def respond(*args, **kwargs) -> ModelResponse: + return ModelResponse( + id="resp", + choices=[ + Choices( + index=0, + finish_reason="stop", + message=LiteLLMMessage(role="assistant", content="ok"), + ) + ], + model="gpt-4o", + usage=Usage( + prompt_tokens=PROMPT_TOKENS, completion_tokens=1, total_tokens=101 + ), + ) + + with patch("openhands.sdk.llm.llm.litellm_completion", side_effect=respond): + yield + + +# ── The LLM's own session ────────────────────────────────────────────── + + +def test_llm_metrics_and_telemetry_are_its_session(): + llm = _llm() + + assert llm.metrics is llm.session.metrics + assert llm.telemetry is llm.session.telemetry + assert llm.session_for() is llm.session + + _call(llm) + assert _prompt_tokens(llm.session) == PROMPT_TOKENS + + +def test_copy_keeps_the_session_unless_the_usage_id_changes(): + llm = _llm() + + assert llm.model_copy(update={"stream": False}).session is llm.session + + other_role = llm.model_copy(update={"usage_id": "condenser"}) + assert other_role.session is not llm.session + _call(other_role) + assert _prompt_tokens(other_role.session) == PROMPT_TOKENS + assert _prompt_tokens(llm.session) == 0 + + +def test_equal_llms_stay_equal(): + """Sessions compare by value, so LLM equality is unchanged.""" + assert _llm() == _llm() + used = _llm() + used.metrics.add_cost(1.0) + assert used != _llm() + + +# ── Sessions of a registry ───────────────────────────────────────────── + + +def test_registry_takes_over_the_session_of_the_first_llm_of_a_role(): + registry = LLMRegistry() + llm = _llm() + llm.metrics.add_cost(1.0) # usage from before the conversation is kept + + registry.add(llm) + + assert registry.sessions["agent"] is llm.session + assert llm.metrics.accumulated_cost == 1.0 + _call(llm, _context(registry)) + assert _prompt_tokens(registry.sessions["agent"]) == PROMPT_TOKENS + + +def test_session_stays_with_the_role_when_the_llm_is_replaced(): + registry = LLMRegistry() + first = _llm() + registry.add(first) + session = registry.sessions["agent"] + _call(first, _context(registry)) + + registry.remove("agent") + assert registry.sessions["agent"] is session # kept while the slot is empty + second = _llm(model="gpt-4o-mini") + registry.add(second) + + assert registry.sessions["agent"] is session + assert second.metrics is session.metrics + assert session.telemetry.model_name == "gpt-4o-mini" + _call(second, _context(registry)) + assert _prompt_tokens(session) == 2 * PROMPT_TOKENS + + +def test_call_records_in_the_session_of_its_role_whichever_object_calls(): + """A copy that never went through ``add()`` still counts for its role.""" + registry = LLMRegistry() + registry.add(_llm(usage_id="condenser")) + unregistered = _llm(model="gpt-4o-mini").model_copy( + update={"usage_id": "condenser"} + ) + + _call(unregistered, _context(registry)) + + session = registry.sessions["condenser"] + assert _prompt_tokens(session) == PROMPT_TOKENS + assert session.telemetry.model_name == "gpt-4o-mini" + # ...and on the object that made the call. + assert _prompt_tokens(unregistered.session) == PROMPT_TOKENS + + +def test_call_without_a_registered_role_records_on_the_llm_itself(): + registry = LLMRegistry() + registry.add(_llm()) + helper = _llm(usage_id="helper") + + _call(helper, _context(registry)) + + assert "helper" not in registry.sessions + assert _prompt_tokens(helper.session) == PROMPT_TOKENS + assert _prompt_tokens(registry.sessions["agent"]) == 0 + + +def test_registered_llm_records_in_its_conversation_without_a_context(): + registry = LLMRegistry() + llm = _llm() + registry.add(llm) + + _call(llm) + + assert _prompt_tokens(registry.sessions["agent"]) == PROMPT_TOKENS + + +def test_scoped_context_selects_the_session_like_an_explicit_one(): + first, second = LLMRegistry(), LLMRegistry() + llm = _llm() + first.add(llm) + second.add(llm) + + with llm_call_context_scope(_context(first)): + _call(llm) + + assert _prompt_tokens(first.sessions["agent"]) == PROMPT_TOKENS + assert _prompt_tokens(second.sessions["agent"]) == 0 + + +def test_one_llm_in_two_registries_gets_a_session_in_each(): + first, second = LLMRegistry(), LLMRegistry() + llm = _llm() + first.add(llm) + _call(llm, _context(first)) + + second.add(llm) + _call(llm, _context(second)) + _call(llm, _context(second)) + _call(llm, _context(first)) + + assert first.sessions["agent"] is not second.sessions["agent"] + assert _prompt_tokens(first.sessions["agent"]) == 2 * PROMPT_TOKENS + assert _prompt_tokens(second.sessions["agent"]) == 2 * PROMPT_TOKENS + # The object itself keeps counting everything it did. + assert _prompt_tokens(llm.session) == 4 * PROMPT_TOKENS + + +def test_child_conversation_context_does_not_inherit_sessions(): + registry = LLMRegistry() + parent = _context(registry) + + assert parent.for_conversation("child").sessions is None + assert parent == LLMCallContext(prompt_cache_key="conv", session_id="conv") + + +# ── Registry-wide callbacks ──────────────────────────────────────────── + + +def test_registry_stats_callback_covers_present_and_future_sessions(): + registry = LLMRegistry() + agent_llm = _llm() + registry.add(agent_llm) + fired: list[str] = [] + registry.set_stats_update_callback(lambda: fired.append("stats")) + + _call(agent_llm, _context(registry)) + late = _llm(usage_id="late") + registry.add(late) + _call(late, _context(registry)) + registry.remove("agent") + replacement = _llm(model="gpt-4o-mini") + registry.add(replacement) + _call(replacement, _context(registry)) + + assert fired == ["stats"] * 3 + + +def test_registry_callbacks_do_not_cross_registries(): + first, second = LLMRegistry(), LLMRegistry() + llm = _llm() + fired: list[str] = [] + first.set_stats_update_callback(lambda: fired.append("first")) + second.set_stats_update_callback(lambda: fired.append("second")) + first.add(llm) + second.add(llm) + + _call(llm, _context(first)) + _call(llm, _context(second)) + + assert fired == ["first", "second"] + + +def test_registry_log_callback_names_the_role_and_the_model_called(tmp_path): + registry = LLMRegistry() + logs: list[tuple[str, str]] = [] + registry.set_log_completions_callback( + lambda filename, data, usage_id, model: logs.append((usage_id, model)) + ) + quiet = _llm() + registry.add(quiet) + _call(quiet, _context(registry)) + assert logs == [] # log_completions is off for this LLM + + registry.remove("agent") + logged = _llm( + model="gpt-4o-mini", + log_completions=True, + log_completions_folder=str(tmp_path), + ) + registry.add(logged) + _call(logged, _context(registry)) + + assert logs == [("agent", "gpt-4o-mini")] + assert list(tmp_path.iterdir()) == [] # sent to the callback, not to disk + + +def test_callback_set_on_the_llm_still_fires_in_a_conversation(): + registry = LLMRegistry() + llm = _llm() + fired: list[str] = [] + llm.telemetry.set_stats_update_callback(lambda: fired.append("llm")) + + registry.add(llm) + _call(llm, _context(registry)) + + assert fired == ["llm"] diff --git a/tests/sdk/llm/test_responses_parsing_and_kwargs.py b/tests/sdk/llm/test_responses_parsing_and_kwargs.py index 35692e08c27..bd209887814 100644 --- a/tests/sdk/llm/test_responses_parsing_and_kwargs.py +++ b/tests/sdk/llm/test_responses_parsing_and_kwargs.py @@ -246,7 +246,7 @@ def test_llm_responses_end_to_end(mock_responses_call): "ok" ] # Telemetry should have recorded usage (one entry) - assert len(llm._telemetry.metrics.token_usages) == 1 # type: ignore[attr-defined] + assert len(llm.telemetry.metrics.token_usages) == 1 @pytest.mark.parametrize( From d7cab95a04742eabfa4434fa67291bc5138b0c72 Mon Sep 17 00:00:00 2001 From: Engel Nyst Date: Thu, 12 Nov 2026 15:20:29 +0100 Subject: [PATCH 2/2] fix(llm): preserve the caller's role and session during fallback (#6784) Co-authored-by: openhands --- .pr/design.html | 147 ++++++++++ .pr/fallback-session-evidence.json | 227 ++++++++++++++ .pr/fallback-session-repro.py | 165 +++++++++++ examples/01_standalone_sdk/39_llm_fallback.py | 2 +- .../openhands/sdk/llm/fallback_strategy.py | 36 ++- openhands-sdk/openhands/sdk/llm/llm.py | 10 +- .../openhands/sdk/llm/utils/metrics.py | 21 +- .../openhands/sdk/llm/utils/telemetry.py | 7 +- tests/sdk/llm/test_llm_fallback_sessions.py | 277 ++++++++++++++++++ 9 files changed, 864 insertions(+), 28 deletions(-) create mode 100644 .pr/design.html create mode 100644 .pr/fallback-session-evidence.json create mode 100644 .pr/fallback-session-repro.py create mode 100644 tests/sdk/llm/test_llm_fallback_sessions.py diff --git a/.pr/design.html b/.pr/design.html new file mode 100644 index 00000000000..8f0437cb972 --- /dev/null +++ b/.pr/design.html @@ -0,0 +1,147 @@ + + + + + +Fallbacks keep the caller's accounting session + + + +
+ +
+
+
Stacked fix for SDK PR #6690
+

Fallbacks keep the caller's accounting session.

+

A fallback changes the model serving a request. Its tokens, cost and telemetry still belong to the agent, condenser or ask-agent role that made the request.

+

The session refactor already records a fallback response in a conversation ledger. Its old post-call merge records the same usage again, and a profile's usage_id="default" can route auxiliary calls into the agent ledger. This fix gives a temporary fallback LLM the caller's existing session references and removes the second accounting path for SDK calls.

+

Before: fallback_strategy.py:99 · After: fallback_strategy.py:106

+
★ One successful provider request writes once to the caller's ledger. The individual records and completion logs identify the model that actually answered.
+
+
+

1. Bind the request before calling the fallback

+
+ +An ask-agent fallback before and after the fix +Before, the fallback profile selects the default role and its metrics are merged into ask-agent, counting one response twice. After, a temporary fallback object is explicitly bound to ask-agent's session, which records one hundred tokens once. + + + + + + + + + +Before · profile role wins + + +try_fallback() → call_fn(fb) +Saved fallback keeps usage_id="default" +fallback_strategy.py:99 + + + +session_for() → sessions["default"] +Agent ledger receives +100 tokens +llm.py:975 + + + +primary_metrics.merge(metrics_diff) +Ask-agent ledger receives another +100 +fallback_strategy.py:110 +100 provider tokens → 200 recorded +Same-role fallback also doubles its single ledger. + + + +After · caller role stays + + +fb.model_copy(usage_id=caller.role) +Temporary copy disables nested fallbacks +fallback_strategy.py:106 + + + +Reuse _session + _registry_session +Existing role and aggregate ownership +fallback_strategy.py:117 + + + +session_for() → caller's session +Ask-agent records +100; its callbacks fire +llm.py:975 +100 provider tokens → 100 recorded +SDK calls skip the post-call metrics merge. + + +
Fig. 1 · The fallback profile supplies model configuration; the caller supplies accounting ownership. Orange arrows mark the changed path.
+
+
+

Context survives async fallback

asyncio.to_thread() carries the scoped call context into async fallback work. Explicit contexts already travel with the callback, so both paths preserve the conversation identity and prompt-cache key.

llm.py:1122 · _ahandle_error()
+

Records keep the answering model

Telemetry passes its current model into token, cost and latency entries. Session totals stay with the role while individual entries identify the fallback model.

telemetry.py:227 · on_response()
+
+
+
+

2. Preserve existing call signatures

+

The original positional arguments remain valid. The fallback reuses existing session references; serialized LLM and profile shapes are unchanged. fallback_strategy.py:117

+
+

Before

try_fallback(model, error, metrics, call_fn)
+metrics.add_cost(value)
+metrics.add_response_latency(value, response_id)
+metrics.add_token_usage(...)
+

After · optional keyword additions

try_fallback(model, error, metrics, call_fn,
+             *, primary=None)
+metrics.add_cost(value, *, model_name=None)
+metrics.add_response_latency(..., model_name=None)
+metrics.add_token_usage(..., model_name=None)
+
+

fallback_strategy.py:73 · metrics.py:189

+

The original four-argument try_fallback() API keeps its metrics-diff merge. SDK calls pass primary=self, reuse the caller's sessions and callbacks, and skip that merge. The fallback example's accounting description is updated to match. fallback_strategy.py:117

+
+
+

3. Five operations, five corrected ledgers

+

The same script ran on PR #6690's 0a68ab94 and on this fix. Each operation sent one request that received HTTP 503, then one successful fallback request reporting 100 prompt tokens.

+ + + + + + + + + +
Public operationBefore: token increaseAfter: token increase
run(), first turndefault +200default +100
run(), second turndefault +200default +100
run(), third turndefault +200default +100
ask_agent()default +100; ask-agent-llm +100ask-agent-llm +100
condense()default +100; condenser +100condenser +100
+

Evidence: fallback-session-evidence.json · fallback-session-repro.py

+

macOS · Python 3.13.11 · real settings construction, profile save/load, conversation lifecycle and HTTP transport. The provider runs on loopback and supplies deterministic responses. These observations establish token totals and role attribution; they do not measure callback counts, actual provider billing or remote-service behavior.

+
Reproduce with the checkout's dependencies
OH_PERSISTENCE_DIR=/tmp/fallback-repro-state \
+OPENHANDS_SUPPRESS_BANNER=1 LITELLM_LOCAL_MODEL_COST_MAP=True \
+uv run python .pr/fallback-session-repro.py --assert-fixed

The final flag makes incorrect accounting fail with exit code 1. The script starts and shuts down its own local HTTP server and uses temporary profile and conversation directories.

+
+
+

4. Regression checks cover the other boundaries

+

1,791 tests passed; 2 expected failures. The LLM, conversation-session, model-switch and agent-server event-service checks passed. Two credential-path checks ran separately without the temporary persistence-directory override. The two fallback regression files account for 58 passing cases.

+

The focused regression file checks sync and async Chat Completions and Responses calls, explicit and scoped contexts, standalone use, a colliding default role, and one LLM shared across registries. Two fallback calls followed by a recovered primary call verify cumulative tokens, cost, callback counts, returned metrics and actual-model labels. A separate case exercises the original direct try_fallback() API.

+

test_fallback_preserves_caller_accounting:41 · test_direct_fallback_preserves_supplied_metrics:230

+
Focused regression command
uv run pytest tests/sdk/llm/test_llm_fallback_sessions.py \
+  tests/sdk/llm/test_llm_fallback.py tests/sdk/llm/test_llm_session.py -q

These regression tests replace provider responses at the LiteLLM boundary. The loopback reproduction above separately exercises real HTTP transport.

+
+ +
+
+ + diff --git a/.pr/fallback-session-evidence.json b/.pr/fallback-session-evidence.json new file mode 100644 index 00000000000..65bffa51193 --- /dev/null +++ b/.pr/fallback-session-evidence.json @@ -0,0 +1,227 @@ +{ + "base_commit": "0a68ab94cb3f447254bdc68ce778048ce517be22", + "fixed_source_sha256": { + "openhands-sdk/openhands/sdk/llm/fallback_strategy.py": "c7f268068411b524ab403d53326f41ec68ae1f17b6be39f95ef2e7085845c48c", + "openhands-sdk/openhands/sdk/llm/llm.py": "a16bc975d4b126839dede49c84e0401f9236a48014e27c7c3ee3aa4639c82c05", + "openhands-sdk/openhands/sdk/llm/utils/metrics.py": "c8b6977296c1274fdd8f63a2e4556d9be2303469e9b559065048536e8996f93b", + "openhands-sdk/openhands/sdk/llm/utils/telemetry.py": "bed4636f931f6dd95f3275ac63cca3d77e533e2d4f1648d4d56a864eee5e73b4" + }, + "runtime": "macOS, Python 3.13.11, PR checkout editable dependencies", + "entry_points": [ + "OpenHandsAgentSettings.create_agent", + "LLMProfileStore.save/load", + "Conversation.run", + "Conversation.ask_agent", + "Conversation.condense" + ], + "provider": "Loopback HTTP provider: gpt-4o returns 503; gpt-4o-mini returns 200 with 100 prompt tokens and 5 completion tokens.", + "limitations": "Provider responses are simulated; SDK execution, profile persistence, HTTP transport and accounting are real. This artifact checks token totals and role attribution. It does not measure costs or callback invocation counts.", + "reproduce": "OH_PERSISTENCE_DIR=/tmp/fallback-repro-state OPENHANDS_SUPPRESS_BANNER=1 LITELLM_LOCAL_MODEL_COST_MAP=True uv run python .pr/fallback-session-repro.py --assert-fixed", + "before": [ + { + "operation": "run-1", + "http_requests": [ + { + "model": "gpt-4o", + "status": 503 + }, + { + "model": "gpt-4o-mini", + "status": 200 + } + ], + "expected_prompt_token_deltas": { + "default": 100 + }, + "actual_prompt_token_deltas": { + "default": 200 + }, + "correct": false + }, + { + "operation": "run-2", + "http_requests": [ + { + "model": "gpt-4o", + "status": 503 + }, + { + "model": "gpt-4o-mini", + "status": 200 + } + ], + "expected_prompt_token_deltas": { + "default": 100 + }, + "actual_prompt_token_deltas": { + "default": 200 + }, + "correct": false + }, + { + "operation": "run-3", + "http_requests": [ + { + "model": "gpt-4o", + "status": 503 + }, + { + "model": "gpt-4o-mini", + "status": 200 + } + ], + "expected_prompt_token_deltas": { + "default": 100 + }, + "actual_prompt_token_deltas": { + "default": 200 + }, + "correct": false + }, + { + "operation": "ask_agent", + "http_requests": [ + { + "model": "gpt-4o", + "status": 503 + }, + { + "model": "gpt-4o-mini", + "status": 200 + } + ], + "expected_prompt_token_deltas": { + "ask-agent-llm": 100 + }, + "actual_prompt_token_deltas": { + "default": 100, + "ask-agent-llm": 100 + }, + "correct": false + }, + { + "operation": "condense", + "http_requests": [ + { + "model": "gpt-4o", + "status": 503 + }, + { + "model": "gpt-4o-mini", + "status": 200 + } + ], + "expected_prompt_token_deltas": { + "condenser": 100 + }, + "actual_prompt_token_deltas": { + "default": 100, + "condenser": 100 + }, + "correct": false + } + ], + "after": [ + { + "operation": "run-1", + "http_requests": [ + { + "model": "gpt-4o", + "status": 503 + }, + { + "model": "gpt-4o-mini", + "status": 200 + } + ], + "expected_prompt_token_deltas": { + "default": 100 + }, + "actual_prompt_token_deltas": { + "default": 100 + }, + "correct": true + }, + { + "operation": "run-2", + "http_requests": [ + { + "model": "gpt-4o", + "status": 503 + }, + { + "model": "gpt-4o-mini", + "status": 200 + } + ], + "expected_prompt_token_deltas": { + "default": 100 + }, + "actual_prompt_token_deltas": { + "default": 100 + }, + "correct": true + }, + { + "operation": "run-3", + "http_requests": [ + { + "model": "gpt-4o", + "status": 503 + }, + { + "model": "gpt-4o-mini", + "status": 200 + } + ], + "expected_prompt_token_deltas": { + "default": 100 + }, + "actual_prompt_token_deltas": { + "default": 100 + }, + "correct": true + }, + { + "operation": "ask_agent", + "http_requests": [ + { + "model": "gpt-4o", + "status": 503 + }, + { + "model": "gpt-4o-mini", + "status": 200 + } + ], + "expected_prompt_token_deltas": { + "ask-agent-llm": 100 + }, + "actual_prompt_token_deltas": { + "ask-agent-llm": 100 + }, + "correct": true + }, + { + "operation": "condense", + "http_requests": [ + { + "model": "gpt-4o", + "status": 503 + }, + { + "model": "gpt-4o-mini", + "status": 200 + } + ], + "expected_prompt_token_deltas": { + "condenser": 100 + }, + "actual_prompt_token_deltas": { + "condenser": 100 + }, + "correct": true + } + ], + "fixed_commit": "6bfdc18917a3ec2847029f3b22cf1861bf8b99ec" +} diff --git a/.pr/fallback-session-repro.py b/.pr/fallback-session-repro.py new file mode 100644 index 00000000000..88c67297f2e --- /dev/null +++ b/.pr/fallback-session-repro.py @@ -0,0 +1,165 @@ +"""Exercise fallback accounting through real SDK conversations and loopback HTTP. + +Run with an isolated OH_PERSISTENCE_DIR and the checkout's Python environment: + OH_PERSISTENCE_DIR=/tmp/fallback-repro-state uv run python \ + .pr/fallback-session-repro.py + +The provider is simulated: primary requests receive HTTP 503, while successful +fallback requests report exactly 100 input and 5 output tokens. SDK execution, +settings, profile loading, HTTP transport, telemetry and accounting are real. +""" + +import argparse +import json +import os +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Any + +from openhands.sdk import LLM, Conversation +from openhands.sdk.llm.fallback_strategy import FallbackStrategy +from openhands.sdk.llm.llm_profile_store import LLMProfileStore +from openhands.sdk.settings import OpenHandsAgentSettings + + +def token_totals(conversation: Conversation) -> dict[str, int]: + return { + role: metrics.accumulated_token_usage.prompt_tokens + if metrics.accumulated_token_usage is not None + else 0 + for role, metrics in conversation.conversation_stats.usage_to_metrics.items() + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--assert-fixed", action="store_true") + args = parser.parse_args() + if not os.environ.get("OH_PERSISTENCE_DIR"): + parser.error("Set OH_PERSISTENCE_DIR to an isolated temporary directory") + + requests: list[dict[str, Any]] = [] + + class Provider(BaseHTTPRequestHandler): + def do_POST(self) -> None: + body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + model = body["model"] + failed = model == "gpt-4o" + status = 503 if failed else 200 + requests.append({"model": model, "status": status}) + if failed: + payload = {"error": {"message": "Simulated outage", "type": "server"}} + else: + payload = { + "id": f"chatcmpl-{len(requests)}", + "object": "chat.completion", + "created": 1, + "model": model, + "choices": [ + { + "index": 0, + "finish_reason": "stop", + "message": {"role": "assistant", "content": "Done."}, + } + ], + "usage": { + "prompt_tokens": 100, + "completion_tokens": 5, + "total_tokens": 105, + }, + } + data = json.dumps(payload).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def log_message(self, format: str, *args: Any) -> None: + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), Provider) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + evidence: list[dict[str, Any]] = [] + try: + with TemporaryDirectory(prefix="fallback-repro-") as directory: + root = Path(directory) + common = { + "base_url": f"http://127.0.0.1:{server.server_port}/v1", + "api_key": "loopback-only", + "num_retries": 1, + "stream": False, + "input_cost_per_token": 0.001, + "output_cost_per_token": 0.002, + } + profiles = root / "profiles" + LLMProfileStore(profiles).save( + "fallback", + LLM(model="openai/gpt-4o-mini", **common), + include_secrets=True, + ) + primary = LLM( + model="openai/gpt-4o", + fallback_strategy=FallbackStrategy( + fallback_llms=["fallback"], profile_store_dir=profiles + ), + **common, + ) + settings = OpenHandsAgentSettings(llm=primary, tools=[]) + conversation = Conversation( + agent=settings.create_agent(), + workspace=root / "workspace", + persistence_dir=root / "conversations", + visualizer=None, + ) + try: + for operation, expected_role in [ + ("run-1", "default"), + ("run-2", "default"), + ("run-3", "default"), + ("ask_agent", "ask-agent-llm"), + ("condense", "condenser"), + ]: + before = token_totals(conversation) + start = len(requests) + if operation.startswith("run"): + conversation.send_message(f"Reply briefly for {operation}.") + conversation.run() + elif operation == "ask_agent": + conversation.ask_agent("What are you doing?") + else: + conversation.condense() + after = token_totals(conversation) + deltas = { + role: total - before.get(role, 0) + for role, total in after.items() + if total != before.get(role, 0) + } + receipts = requests[start:] + successes = sum(item["status"] == 200 for item in receipts) + expected = {expected_role: 100 * successes} + evidence.append( + { + "operation": operation, + "http_requests": receipts, + "expected_prompt_token_deltas": expected, + "actual_prompt_token_deltas": deltas, + "correct": successes == 1 and deltas == expected, + } + ) + finally: + conversation.close() + finally: + server.shutdown() + server.server_close() + worker.join() + print(json.dumps(evidence, indent=2)) + if args.assert_fixed and not all(item["correct"] for item in evidence): + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/examples/01_standalone_sdk/39_llm_fallback.py b/examples/01_standalone_sdk/39_llm_fallback.py index ebd891c8f37..566cc464ea1 100644 --- a/examples/01_standalone_sdk/39_llm_fallback.py +++ b/examples/01_standalone_sdk/39_llm_fallback.py @@ -3,7 +3,7 @@ When the primary LLM fails with a transient error (rate limit, timeout, etc.), FallbackStrategy automatically tries alternate LLMs in order. Fallback is per-call: each new request starts with the primary model. Token usage and -cost from fallback calls are merged into the primary LLM's metrics. +cost from fallback calls are recorded in the caller's usage session. This example: 1. Saves two fallback LLM profiles to a temporary store. diff --git a/openhands-sdk/openhands/sdk/llm/fallback_strategy.py b/openhands-sdk/openhands/sdk/llm/fallback_strategy.py index faa2f43a4fc..c9ca0c9794c 100644 --- a/openhands-sdk/openhands/sdk/llm/fallback_strategy.py +++ b/openhands-sdk/openhands/sdk/llm/fallback_strategy.py @@ -25,6 +25,7 @@ if TYPE_CHECKING: + from openhands.sdk.llm.llm import LLM from openhands.sdk.llm.llm_response import LLMResponse from openhands.sdk.llm.utils.metrics import Metrics @@ -75,14 +76,17 @@ def try_fallback( primary_error: Exception, primary_metrics: Metrics, call_fn: Callable[[Any], LLMResponse], + *, + primary: LLM | None = None, ) -> LLMResponse | None: - """Try fallback LLMs in order. Merges metrics into primary on success. + """Try fallback models in the caller's role and session. Args: primary_model: The primary model name (for logging). primary_error: The error from the primary model. - primary_metrics: The primary LLM's Metrics to merge fallback costs into. + primary_metrics: The primary LLM's metrics for legacy direct callers. call_fn: A callable that takes an LLM instance and returns an LLMResponse. + primary: The caller whose usage ID and aggregate accounting are kept. Returns: LLMResponse from the first successful fallback, or None if all fail. @@ -99,17 +103,23 @@ def try_fallback( f"{remaining} fallback(s) remaining" ) try: - # Disable nested fallbacks to prevent recursive chains - saved_strategy = fb.fallback_strategy - fb.fallback_strategy = None - metrics_before = fb.metrics.deep_copy() - try: - result = call_fn(fb) - finally: - fb.fallback_strategy = saved_strategy - # Merge fallback metrics (cost + tokens) into primary - metrics_diff = fb.metrics.diff(metrics_before) - primary_metrics.merge(metrics_diff) + fallback = fb.model_copy( + update={ + "usage_id": primary.usage_id + if primary is not None + else fb.usage_id, + "fallback_strategy": None, + } + ) + metrics_before = ( + fallback.metrics.deep_copy() if primary is None else None + ) + if primary is not None: + fallback._session = primary.session + fallback._registry_session = primary._registered_session() + result = call_fn(fallback) + if metrics_before is not None: + primary_metrics.merge(fallback.metrics.diff(metrics_before)) logger.info(f"[Fallback Strategy] Fallback LLM ({fb.model}) succeeded") return result except Exception as fb_error: diff --git a/openhands-sdk/openhands/sdk/llm/llm.py b/openhands-sdk/openhands/sdk/llm/llm.py index 9e77e97a103..6a65bcef11c 100644 --- a/openhands-sdk/openhands/sdk/llm/llm.py +++ b/openhands-sdk/openhands/sdk/llm/llm.py @@ -977,7 +977,7 @@ def session_for(self, call_context: LLMCallContext | None = None) -> LLMSession: if session is None: session = self._registered_session() if session is None: - return self.session + session = self.session if session is not self._session: self._keep_own_session_apart() session.follow(self) @@ -1093,6 +1093,7 @@ def _handle_error( primary_error=error, primary_metrics=session.metrics, call_fn=fallback_call_fn, + primary=self, ) if result is not None: return result @@ -1111,21 +1112,20 @@ async def _ahandle_error( The *fallback_call_fn* is synchronous (it calls the fallback LLM's sync ``completion``/``responses``), so the fallback attempt is - offloaded to a thread via :func:`asyncio.loop.run_in_executor` to + offloaded to a thread via :func:`asyncio.to_thread` to avoid blocking the event loop. """ if session is None: session = self.session session.telemetry.on_error(error) if self.fallback_strategy and self.fallback_strategy.should_fallback(error): - loop = asyncio.get_running_loop() - result = await loop.run_in_executor( - None, + result = await asyncio.to_thread( self.fallback_strategy.try_fallback, self.model, error, session.metrics, fallback_call_fn, + primary=self, ) if result is not None: return result diff --git a/openhands-sdk/openhands/sdk/llm/utils/metrics.py b/openhands-sdk/openhands/sdk/llm/utils/metrics.py index 720fb27b469..e851c7d9386 100644 --- a/openhands-sdk/openhands/sdk/llm/utils/metrics.py +++ b/openhands-sdk/openhands/sdk/llm/utils/metrics.py @@ -186,24 +186,28 @@ def rolls_up_into(self) -> "Metrics | None": except AttributeError: # never attached: the slot is still empty return None - def add_cost(self, value: float) -> None: + def add_cost(self, value: float, *, model_name: str | None = None) -> None: if value < 0: raise ValueError("Added cost cannot be negative.") self.accumulated_cost += value - self.costs.append(Cost(cost=value, model=self.model_name)) + self.costs.append(Cost(cost=value, model=model_name or self.model_name)) target = self.rolls_up_into if target is not None: - target.add_cost(value) + target.add_cost(value, model_name=model_name) - def add_response_latency(self, value: float, response_id: str) -> None: + def add_response_latency( + self, value: float, response_id: str, *, model_name: str | None = None + ) -> None: self.response_latencies.append( ResponseLatency( - latency=max(0.0, value), model=self.model_name, response_id=response_id + latency=max(0.0, value), + model=model_name or self.model_name, + response_id=response_id, ) ) target = self.rolls_up_into if target is not None: - target.add_response_latency(value, response_id) + target.add_response_latency(value, response_id, model_name=model_name) def add_token_usage( self, @@ -214,13 +218,15 @@ def add_token_usage( context_window: int, response_id: str, reasoning_tokens: int = 0, + *, + model_name: str | None = None, ) -> None: """Add a single usage record.""" # Token each turn for calculating context usage. per_turn_token = prompt_tokens + completion_tokens usage = TokenUsage( - model=self.model_name, + model=model_name or self.model_name, prompt_tokens=prompt_tokens, completion_tokens=completion_tokens, cache_read_tokens=cache_read_tokens, @@ -259,6 +265,7 @@ def add_token_usage( context_window=context_window, response_id=response_id, reasoning_tokens=reasoning_tokens, + model_name=model_name, ) def merge(self, other: "Metrics") -> None: diff --git a/openhands-sdk/openhands/sdk/llm/utils/telemetry.py b/openhands-sdk/openhands/sdk/llm/utils/telemetry.py index d2d7c71ce28..a1616e2beef 100644 --- a/openhands-sdk/openhands/sdk/llm/utils/telemetry.py +++ b/openhands-sdk/openhands/sdk/llm/utils/telemetry.py @@ -227,13 +227,15 @@ def on_response( # 1) latency self._last_latency = time.time() - (self._req_start or time.time()) response_id = resp.id - self.metrics.add_response_latency(self._last_latency, response_id) + self.metrics.add_response_latency( + self._last_latency, response_id, model_name=self.model_name + ) # 2) cost cost = self._compute_cost(resp, provider_info=provider_info) # A confirmed zero charge is different from unavailable pricing. if cost is not None: - self.metrics.add_cost(cost) + self.metrics.add_cost(cost, model_name=self.model_name) # 3) tokens - normalize the provider's usage shape exactly once usage = normalize_usage(_response_usage(resp)) @@ -315,6 +317,7 @@ def _record_usage( reasoning_tokens=usage.reasoning_tokens, context_window=context_window, response_id=response_id, + model_name=self.model_name, ) # ---------- Observability span ---------- diff --git a/tests/sdk/llm/test_llm_fallback_sessions.py b/tests/sdk/llm/test_llm_fallback_sessions.py new file mode 100644 index 00000000000..b2b713571f9 --- /dev/null +++ b/tests/sdk/llm/test_llm_fallback_sessions.py @@ -0,0 +1,277 @@ +"""Fallback requests keep the caller's role and accounting session.""" + +import json +from contextlib import nullcontext +from pathlib import Path +from typing import Any, Literal +from unittest.mock import AsyncMock + +import pytest +from litellm.exceptions import APIConnectionError +from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse +from litellm.types.utils import Choices, Message as LiteLLMMessage, ModelResponse, Usage +from pydantic import SecretStr + +from openhands.sdk.llm import ( + LLM, + FallbackStrategy, + LLMCallContext, + LLMRegistry, + Message, + TextContent, +) +from openhands.sdk.llm.call_context import llm_call_context_scope +from openhands.sdk.llm.llm_profile_store import LLMProfileStore + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "method,role,context_mode,collision,shared", + [ + ("completion", "default", "explicit", False, False), + ("acompletion", "default", "scoped", False, True), + ("responses", "condenser", "explicit", True, False), + ("aresponses", "condenser", "scoped", True, False), + ("completion", "condenser", "explicit", False, False), + ("acompletion", "condenser", "scoped", False, False), + ("completion", "condenser", "standalone", False, False), + ("aresponses", "default", "standalone", False, False), + ], +) +async def test_fallback_preserves_caller_accounting( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + method: Literal["completion", "acompletion", "responses", "aresponses"], + role: str, + context_mode: Literal["explicit", "scoped", "standalone"], + collision: bool, + shared: bool, +): + fallback = LLM( + model="gpt-4o-mini", + api_key=SecretStr("test-key"), + num_retries=0, + input_cost_per_token=0.01, + output_cost_per_token=0.02, + log_completions=True, + log_completions_folder=str(tmp_path / "logs"), + ) + LLMProfileStore(base_dir=tmp_path / "profiles").save( + "backup", fallback, include_secrets=True + ) + primary = LLM( + model="gpt-4o", + api_key=SecretStr("test-key"), + usage_id=role, + num_retries=0, + input_cost_per_token=0.1, + output_cost_per_token=0.2, + log_completions=True, + log_completions_folder=str(tmp_path / "logs"), + fallback_strategy=FallbackStrategy( + fallback_llms=["backup"], profile_store_dir=tmp_path / "profiles" + ), + ) + registry = LLMRegistry() + if context_mode != "standalone": + registry.add(primary) + if collision: + registry.add(LLM(model="gpt-4o-mini")) + other_registry = LLMRegistry() + if shared: + other_registry.add(primary) + metrics = ( + primary.metrics + if context_mode == "standalone" + else registry.sessions[role].metrics + ) + notifications: list[tuple[int, float]] = [] + logs: list[tuple[str, str, dict[str, Any]]] = [] + + def on_stats() -> None: + usage = metrics.accumulated_token_usage + assert usage is not None + notifications.append((usage.prompt_tokens, metrics.accumulated_cost)) + + def on_log(filename: str, data: str, usage_id: str, model: str) -> None: + logs.append((usage_id, model, json.loads(data))) + + if context_mode == "standalone": + primary.telemetry.set_stats_update_callback(on_stats) + primary.telemetry.set_log_completions_callback( + lambda filename, data: on_log( + filename, data, role, primary.telemetry.model_name + ) + ) + else: + registry.set_stats_update_callback(on_stats) + registry.set_log_completions_callback(on_log) + + context = LLMCallContext( + prompt_cache_key="caller-cache", + session_id="caller-conversation", + sessions=registry.sessions if context_mode != "standalone" else None, + ) + requests: list[dict[str, Any]] = [] + primary_attempts = 0 + + def respond(**kwargs: Any) -> ModelResponse | ResponsesAPIResponse: + nonlocal primary_attempts + requests.append(kwargs) + model = kwargs["model"] + if model == primary.model: + primary_attempts += 1 + if primary_attempts <= 2: + raise APIConnectionError( + message="primary unavailable", llm_provider="openai", model=model + ) + if method in ("responses", "aresponses"): + return ResponsesAPIResponse( + id=f"response-{len(requests)}", + created_at=1, + model=model, + object="response", + output=[ + { + "type": "message", + "id": "message", + "role": "assistant", + "status": "completed", + "content": [ + {"type": "output_text", "text": "ok", "annotations": []} + ], + } + ], + parallel_tool_calls=False, + tool_choice="auto", + tools=[], + usage=ResponseAPIUsage( + input_tokens=10, output_tokens=5, total_tokens=15 + ), + ) + return ModelResponse( + id=f"response-{len(requests)}", + model=model, + choices=[ + Choices( + index=0, + finish_reason="stop", + message=LiteLLMMessage(role="assistant", content="ok"), + ) + ], + usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15), + ) + + monkeypatch.setattr("openhands.sdk.llm.llm.litellm_completion", respond) + monkeypatch.setattr("openhands.sdk.llm.llm.litellm_responses", respond) + monkeypatch.setattr( + "openhands.sdk.llm.llm.litellm_acompletion", AsyncMock(side_effect=respond) + ) + monkeypatch.setattr( + "openhands.sdk.llm.llm.litellm_aresponses", AsyncMock(side_effect=respond) + ) + messages = [ + Message(role="system", content=[TextContent(text="Reply briefly.")]), + Message(role="user", content=[TextContent(text="Hello.")]), + ] + supplied_context = context if context_mode == "explicit" else None + scope = ( + llm_call_context_scope(context) if context_mode == "scoped" else nullcontext() + ) + with scope: + for count, cost in enumerate((0.2, 0.4, 2.4), start=1): + if method == "completion": + result = primary.completion(messages, call_context=supplied_context) + elif method == "acompletion": + result = await primary.acompletion( + messages, call_context=supplied_context + ) + elif method == "responses": + result = primary.responses(messages, call_context=supplied_context) + else: + result = await primary.aresponses( + messages, call_context=supplied_context + ) + assert result.metrics.accumulated_token_usage is not None + assert result.metrics.accumulated_token_usage.prompt_tokens == 10 * count + assert result.metrics.accumulated_cost == pytest.approx(cost) + assert metrics.accumulated_cost == pytest.approx(cost) + assert primary.metrics.accumulated_cost == pytest.approx(cost) + assert [tokens for tokens, _ in notifications] == [ + 10 * n for n in range(1, count + 1) + ] + assert [total for _, total in notifications] == pytest.approx( + (0.2, 0.4, 2.4)[:count] + ) + + assert [record.model for record in metrics.token_usages] == [ + "gpt-4o-mini", + "gpt-4o-mini", + "gpt-4o", + ] + assert [(uid, model) for uid, model, data in logs if "response" in data] == [ + (role, "gpt-4o-mini"), + (role, "gpt-4o-mini"), + (role, "gpt-4o"), + ] + if context_mode != "standalone": + for request in requests: + assert request["prompt_cache_key"] == "caller-cache" + assert ( + request["extra_headers"]["x-litellm-session-id"] + == "caller-conversation" + ) + if collision: + assert registry.sessions["default"].metrics.accumulated_cost == 0 + if shared: + assert other_registry.sessions[role].metrics.accumulated_cost == 0 + + +def test_direct_fallback_preserves_supplied_metrics( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +): + primary = LLM(model="gpt-4o", usage_id="condenser") + registry, other_registry = LLMRegistry(), LLMRegistry() + registry.add(primary) + other_registry.add(primary) + metrics = registry.sessions["condenser"].metrics + LLMProfileStore(base_dir=tmp_path).save( + "backup", LLM(model="gpt-4o-mini", api_key=SecretStr("test-key")) + ) + strategy = FallbackStrategy(fallback_llms=["backup"], profile_store_dir=tmp_path) + response = ModelResponse( + id="fallback-response", + model="gpt-4o-mini", + choices=[ + Choices( + index=0, + finish_reason="stop", + message=LiteLLMMessage(role="assistant", content="ok"), + ) + ], + usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15), + ) + monkeypatch.setattr( + "openhands.sdk.llm.llm.litellm_completion", lambda **kwargs: response + ) + messages = [ + Message(role="system", content=[TextContent(text="Reply briefly.")]), + Message(role="user", content=[TextContent(text="Hello.")]), + ] + + result = strategy.try_fallback( + primary.model, + APIConnectionError( + message="primary unavailable", llm_provider="openai", model=primary.model + ), + metrics, + lambda fallback: fallback.completion(messages), + ) + + assert result is not None + assert metrics.accumulated_token_usage is not None + assert primary.metrics.accumulated_token_usage is not None + assert metrics.accumulated_token_usage.prompt_tokens == 10 + assert primary.metrics.accumulated_token_usage.prompt_tokens == 10 + assert len(metrics.token_usages) == 1 + assert other_registry.sessions["condenser"].metrics.token_usages == []