diff --git a/openhands-sdk/openhands/sdk/llm/utils/model_info.py b/openhands-sdk/openhands/sdk/llm/utils/model_info.py index 7830bc95f79..332b66cb3a0 100644 --- a/openhands-sdk/openhands/sdk/llm/utils/model_info.py +++ b/openhands-sdk/openhands/sdk/llm/utils/model_info.py @@ -1,5 +1,6 @@ +import threading import time -from collections.abc import Mapping +from collections.abc import Callable, Mapping from functools import lru_cache from logging import getLogger from typing import Any @@ -16,6 +17,69 @@ logger = getLogger(__name__) +# Model-info discovery must never block LLM construction indefinitely. +# +# Some providers (self-hosted / OpenAI-compatible such as ``lemonade``, +# ``ollama`` and ``vllm``) resolve model info by making a live HTTP request to +# the model server rather than reading a static cost map. litellm performs that +# request synchronously with no timeout, so if the endpoint is unreachable -- or +# accidentally points back at the caller (e.g. a reverse proxy that forwards to +# this very process) -- the call hangs forever. Because ``LLM`` construction is +# synchronous and runs this probe in ``_post_init``, an indefinite hang freezes +# any caller, including an async server's event loop. +# +# Bound every probe with this deadline. ``model_info`` is an optional +# enhancement (callers already handle ``None``), so timing out degrades +# gracefully instead of deadlocking. +MODEL_INFO_DISCOVERY_TIMEOUT = 10.0 + + +def _run_with_deadline[T]( + func: Callable[..., T], + *args, + timeout: float | None = None, + **kwargs, +) -> T | None: + """Run a (possibly network-bound, non-cancellable) sync call with a deadline. + + Returns ``None`` if the call does not finish within ``timeout`` so that + model-info discovery never blocks the caller indefinitely. + + The call runs on a dedicated daemon thread. A timed-out probe cannot be + cancelled, but because the thread is a daemon it is never joined at + interpreter shutdown -- so an endpoint that is reachable-but-silent (which + only litellm's much larger socket timeout would bound) delays neither the + caller nor process exit. A fresh thread per call also means a stuck probe + cannot saturate a shared worker pool and starve later discovery. + """ + if timeout is None: + timeout = MODEL_INFO_DISCOVERY_TIMEOUT + + result: list[T] = [] + + def _target() -> None: + try: + result.append(func(*args, **kwargs)) + except Exception as e: + logger.debug("Model-info probe raised; ignoring: %s", e) + + thread = threading.Thread(target=_target, name="model-info-discovery", daemon=True) + try: + thread.start() + except RuntimeError: + # Interpreter is shutting down and will not start new threads. Model + # info is optional, so skip discovery rather than probing inline (which + # could re-introduce the very hang this guard exists to bound). + return None + thread.join(timeout) + if thread.is_alive(): + logger.warning( + "Timed out after %ss while fetching model info; continuing without it.", + timeout, + ) + return None + return result[0] if result else None + def _merge_raw_model_metadata(model_info: Mapping[str, Any]) -> dict[str, Any]: """Preserve raw LiteLLM capability fields.""" @@ -114,7 +178,11 @@ def _get_model_info_from_litellm_proxy( if secret_api_key: headers["Authorization"] = f"Bearer {secret_api_key}" - response = httpx.get(f"{base_url}/v1/model/info", headers=headers) + response = httpx.get( + f"{base_url}/v1/model/info", + headers=headers, + timeout=MODEL_INFO_DISCOVERY_TIMEOUT, + ) data = response.json().get("data", []) # Match against either the public alias (`model_name`) or the # underlying provider/model_name form (`litellm_params.model`). The proxy itself @@ -154,8 +222,14 @@ def _get_model_info_from_litellm_proxy( underlying_model = current.get("litellm_params", {}).get("model") underlying_model_info = None if isinstance(underlying_model, str) and underlying_model != stripped: + # Bound this probe too: like the discovery calls below, it is a + # synchronous, network-bound litellm lookup with no timeout, and + # it runs inside ``LLM._post_init`` -- so an unreachable + # underlying endpoint here would reintroduce the same deadlock. try: - underlying_model_info = get_model_info(underlying_model) + underlying_model_info = _run_with_deadline( + get_model_info, underlying_model + ) except Exception as e: logger.debug( f"get_model_info(underlying={underlying_model}) failed: {e}" @@ -185,7 +259,7 @@ def get_litellm_model_info( # Try to get model info via openrouter or litellm proxy first try: if model.startswith("openrouter"): - model_info = get_model_info(model) + model_info = _run_with_deadline(get_model_info, model) if model_info: return _merge_raw_model_metadata(model_info) except Exception as e: @@ -206,13 +280,13 @@ def get_litellm_model_info( # Fallbacks: try base name variants try: - model_info = get_model_info(model.split(":")[0]) + model_info = _run_with_deadline(get_model_info, model.split(":")[0]) if model_info: return _merge_raw_model_metadata(model_info) except Exception: pass try: - model_info = get_model_info(model.split("/")[-1]) + model_info = _run_with_deadline(get_model_info, model.split("/")[-1]) if model_info: return _merge_raw_model_metadata(model_info) except Exception: diff --git a/tests/sdk/llm/test_llm.py b/tests/sdk/llm/test_llm.py index 472ba6a27da..09c5d73481c 100644 --- a/tests/sdk/llm/test_llm.py +++ b/tests/sdk/llm/test_llm.py @@ -16,6 +16,7 @@ from openhands.sdk.llm.exceptions import LLMNoResponseError from openhands.sdk.llm.options.responses_options import select_responses_options from openhands.sdk.llm.utils.metrics import Metrics, TokenUsage +from openhands.sdk.llm.utils.model_info import MODEL_INFO_DISCOVERY_TIMEOUT from openhands.sdk.llm.utils.telemetry import Telemetry from openhands.sdk.tool.builtins.finish import FinishTool @@ -133,6 +134,7 @@ def test_base_url_for_openhands_provider(mock_get): mock_get.assert_called_once_with( "https://llm-proxy.app.all-hands.dev/v1/model/info", headers={"Authorization": "Bearer test-key"}, + timeout=MODEL_INFO_DISCOVERY_TIMEOUT, ) diff --git a/tests/sdk/llm/test_model_info_discovery_timeout.py b/tests/sdk/llm/test_model_info_discovery_timeout.py new file mode 100644 index 00000000000..07115148a34 --- /dev/null +++ b/tests/sdk/llm/test_model_info_discovery_timeout.py @@ -0,0 +1,117 @@ +"""Model-info discovery must never block the caller indefinitely. + +Some providers (self-hosted / OpenAI-compatible such as ``lemonade``, +``ollama`` and ``vllm``) resolve model info via a live HTTP request to the +model server. litellm makes that request synchronously with no timeout, so an +unreachable endpoint -- or one that loops back to the caller through a reverse +proxy -- hangs forever. Because ``LLM`` construction runs this probe in +``_post_init``, that previously froze any caller, including an async server's +event loop. + +These tests pin the bounded behavior: discovery returns ``None`` within the +deadline instead of hanging. +""" + +import time +from types import SimpleNamespace + +from openhands.sdk.llm.utils import model_info +from openhands.sdk.llm.utils.model_info import ( + _run_with_deadline, + get_litellm_model_info, +) + + +def test_run_with_deadline_returns_none_on_timeout(): + def _hang(*_a, **_kw): + time.sleep(30) + return "never" + + started = time.time() + result = _run_with_deadline(_hang, timeout=0.2) + elapsed = time.time() - started + + assert result is None + # Returned promptly rather than waiting for the (abandoned) call. + assert elapsed < 5 + + +def test_run_with_deadline_returns_value_when_fast(): + assert _run_with_deadline(lambda: 42, timeout=5) == 42 + + +def test_get_litellm_model_info_does_not_hang_on_slow_probe(monkeypatch): + """A self-hosted model whose model-info probe hangs must not block. + + Regression test for the deadlock where ``litellm.get_model_info`` for an + unreachable self-hosted endpoint blocked ``LLM`` construction (and any + async caller) forever. + """ + + def _hang(*_a, **_kw): + time.sleep(30) + raise AssertionError("should have been abandoned by the deadline") + + monkeypatch.setattr(model_info, "get_model_info", _hang) + monkeypatch.setattr(model_info, "MODEL_INFO_DISCOVERY_TIMEOUT", 0.2) + + started = time.time() + info = get_litellm_model_info( + secret_api_key=None, + base_url="http://localhost:9", + model="lemonade/some-self-hosted-model", + ) + elapsed = time.time() - started + + assert info is None + # Two bounded fallback probes at ~0.2s each must still return promptly. + assert elapsed < 5 + + +def test_litellm_proxy_underlying_probe_does_not_hang(monkeypatch): + """The proxy alias-pricing probe must also be bounded. + + ``_get_model_info_from_litellm_proxy`` resolves a proxy alias, then probes + the *underlying* model via ``get_model_info`` to make the alias priceable. + That nested probe is a synchronous, network-bound litellm call with no + timeout and runs inside ``LLM._post_init``; an unreachable underlying + endpoint there would reintroduce the deadlock this PR targets. + """ + + def _fake_proxy_response(*_a, **_kw): + # Advertise an alias whose underlying model differs from the request, + # so the underlying-model probe branch runs. + return SimpleNamespace( + json=lambda: { + "data": [ + { + "model_name": "my-alias", + "litellm_params": {"model": "some/underlying-model"}, + "model_info": {"max_input_tokens": 100}, + } + ] + } + ) + + def _hang(*_a, **_kw): + time.sleep(30) + raise AssertionError("underlying probe should have been abandoned") + + monkeypatch.setattr(model_info.httpx, "get", _fake_proxy_response) + monkeypatch.setattr(model_info, "get_model_info", _hang) + monkeypatch.setattr(model_info, "MODEL_INFO_DISCOVERY_TIMEOUT", 0.2) + # Bypass the hourly lru_cache so this probe actually runs. + model_info._get_model_info_from_litellm_proxy.cache_clear() + + started = time.time() + info = get_litellm_model_info( + secret_api_key=None, + base_url="http://localhost:9", + model="litellm_proxy/my-alias", + ) + elapsed = time.time() - started + + # The alias's own model_info is still returned; only the underlying probe + # was abandoned, and it did so without blocking construction. + assert info == {"max_input_tokens": 100, "_capability_source": "proxy"} + assert elapsed < 5