From 461a8488f0165e392c2bb9e15a3e0db514e9cc34 Mon Sep 17 00:00:00 2001 From: bconsolvo Date: Mon, 29 Jun 2026 15:07:11 -0700 Subject: [PATCH 1/6] fix(llm): bound model-info discovery with a timeout to prevent deadlocks Model-info discovery for self-hosted / OpenAI-compatible providers (lemonade, ollama, vllm) issues a synchronous HTTP request with no timeout. When the endpoint is unreachable -- or loops back to the caller via a reverse proxy -- the call hangs forever. Because LLM construction runs this probe in _post_init, the hang freezes any caller, including the agent-server event loop (every request times out). Bound every probe with a deadline (returns None on timeout; callers already treat model_info=None as valid) and run the blocking profile load off the event loop in the agent-server profile endpoints. Co-authored-by: Cursor --- .../openhands/agent_server/profiles_router.py | 9 ++- .../openhands/sdk/llm/utils/model_info.py | 60 +++++++++++++++-- .../llm/test_model_info_discovery_timeout.py | 67 +++++++++++++++++++ 3 files changed, 130 insertions(+), 6 deletions(-) create mode 100644 tests/sdk/llm/test_model_info_discovery_timeout.py diff --git a/openhands-agent-server/openhands/agent_server/profiles_router.py b/openhands-agent-server/openhands/agent_server/profiles_router.py index 90b0466979d..bb428b9dd58 100644 --- a/openhands-agent-server/openhands/agent_server/profiles_router.py +++ b/openhands-agent-server/openhands/agent_server/profiles_router.py @@ -5,6 +5,7 @@ from typing import Annotated, Any from fastapi import APIRouter, HTTPException, Path, Request, status +from fastapi.concurrency import run_in_threadpool from pydantic import BaseModel, Field, SecretStr from openhands.agent_server._secrets_exposure import ( @@ -165,7 +166,10 @@ async def get_profile(request: Request, name: ProfileName) -> ProfileDetailRespo store = get_llm_profile_store() try: with _store_errors(): - llm = store.load(name, cipher=cipher) + # Loading constructs an ``LLM``, which may perform a synchronous, + # network-bound model-info probe. Run it in a worker thread so a + # slow or unreachable endpoint can never block the event loop. + llm = await run_in_threadpool(store.load, name, cipher=cipher) except FileNotFoundError: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, @@ -326,7 +330,8 @@ async def activate_profile( profile_store = get_llm_profile_store() try: with _store_errors(): - llm = profile_store.load(name, cipher=cipher) + # See ``get_profile``: keep the network-bound load off the loop. + llm = await run_in_threadpool(profile_store.load, name, cipher=cipher) except FileNotFoundError: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, diff --git a/openhands-sdk/openhands/sdk/llm/utils/model_info.py b/openhands-sdk/openhands/sdk/llm/utils/model_info.py index f19f417196e..79910a750df 100644 --- a/openhands-sdk/openhands/sdk/llm/utils/model_info.py +++ b/openhands-sdk/openhands/sdk/llm/utils/model_info.py @@ -1,4 +1,6 @@ import time +from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor, TimeoutError as FutureTimeoutError from functools import lru_cache from logging import getLogger @@ -12,6 +14,52 @@ logger = getLogger(__name__) +# Model-info discovery must never block LLM construction indefinitely. +# +# Some providers (self-hosted / OpenAI-compatible such as ``lemonade``, +# ``ollama`` and ``vllm``) resolve model info by making a live HTTP request to +# the model server rather than reading a static cost map. litellm performs that +# request synchronously with no timeout, so if the endpoint is unreachable -- or +# accidentally points back at the caller (e.g. a reverse proxy that forwards to +# this very process) -- the call hangs forever. Because ``LLM`` construction is +# synchronous and runs this probe in ``_post_init``, an indefinite hang freezes +# any caller, including an async server's event loop. +# +# Bound every probe with this deadline. ``model_info`` is an optional +# enhancement (callers already handle ``None``), so timing out degrades +# gracefully instead of deadlocking. +MODEL_INFO_DISCOVERY_TIMEOUT = 10.0 + +_discovery_executor = ThreadPoolExecutor( + max_workers=4, thread_name_prefix="model-info-discovery" +) + + +def _run_with_deadline[T]( + func: Callable[..., T], + *args, + timeout: float | None = None, + **kwargs, +) -> T | None: + """Run a (possibly network-bound, non-cancellable) sync call with a deadline. + + Returns ``None`` if the call does not finish within ``timeout`` so that + model-info discovery never blocks the caller indefinitely. A timed-out + worker thread is abandoned (it cannot be cancelled) but is bounded by + litellm's own socket timeout and does not block the caller. + """ + if timeout is None: + timeout = MODEL_INFO_DISCOVERY_TIMEOUT + future = _discovery_executor.submit(func, *args, **kwargs) + try: + return future.result(timeout=timeout) + except FutureTimeoutError: + logger.warning( + "Timed out after %ss while fetching model info; continuing without it.", + timeout, + ) + return None + @lru_cache def _get_model_info_from_litellm_proxy( @@ -28,7 +76,11 @@ def _get_model_info_from_litellm_proxy( if secret_api_key: headers["Authorization"] = f"Bearer {secret_api_key}" - response = httpx.get(f"{base_url}/v1/model/info", headers=headers) + response = httpx.get( + f"{base_url}/v1/model/info", + headers=headers, + timeout=MODEL_INFO_DISCOVERY_TIMEOUT, + ) data = response.json().get("data", []) # Match against either the public alias (`model_name`) or the # underlying provider/model_name form (`litellm_params.model`). The proxy itself @@ -69,7 +121,7 @@ def get_litellm_model_info( # Try to get model info via openrouter or litellm proxy first try: if model.startswith("openrouter"): - model_info = get_model_info(model) + model_info = _run_with_deadline(get_model_info, model) if model_info: return model_info except Exception as e: @@ -90,13 +142,13 @@ def get_litellm_model_info( # Fallbacks: try base name variants try: - model_info = get_model_info(model.split(":")[0]) + model_info = _run_with_deadline(get_model_info, model.split(":")[0]) if model_info: return model_info except Exception: pass try: - model_info = get_model_info(model.split("/")[-1]) + model_info = _run_with_deadline(get_model_info, model.split("/")[-1]) if model_info: return model_info except Exception: diff --git a/tests/sdk/llm/test_model_info_discovery_timeout.py b/tests/sdk/llm/test_model_info_discovery_timeout.py new file mode 100644 index 00000000000..464b609ee50 --- /dev/null +++ b/tests/sdk/llm/test_model_info_discovery_timeout.py @@ -0,0 +1,67 @@ +"""Model-info discovery must never block the caller indefinitely. + +Some providers (self-hosted / OpenAI-compatible such as ``lemonade``, +``ollama`` and ``vllm``) resolve model info via a live HTTP request to the +model server. litellm makes that request synchronously with no timeout, so an +unreachable endpoint -- or one that loops back to the caller through a reverse +proxy -- hangs forever. Because ``LLM`` construction runs this probe in +``_post_init``, that previously froze any caller, including an async server's +event loop. + +These tests pin the bounded behavior: discovery returns ``None`` within the +deadline instead of hanging. +""" + +import time + +from openhands.sdk.llm.utils import model_info +from openhands.sdk.llm.utils.model_info import ( + _run_with_deadline, + get_litellm_model_info, +) + + +def test_run_with_deadline_returns_none_on_timeout(): + def _hang(*_a, **_kw): + time.sleep(30) + return "never" + + started = time.time() + result = _run_with_deadline(_hang, timeout=0.2) + elapsed = time.time() - started + + assert result is None + # Returned promptly rather than waiting for the (abandoned) call. + assert elapsed < 5 + + +def test_run_with_deadline_returns_value_when_fast(): + assert _run_with_deadline(lambda: 42, timeout=5) == 42 + + +def test_get_litellm_model_info_does_not_hang_on_slow_probe(monkeypatch): + """A self-hosted model whose model-info probe hangs must not block. + + Regression test for the deadlock where ``litellm.get_model_info`` for an + unreachable self-hosted endpoint blocked ``LLM`` construction (and any + async caller) forever. + """ + + def _hang(*_a, **_kw): + time.sleep(30) + raise AssertionError("should have been abandoned by the deadline") + + monkeypatch.setattr(model_info, "get_model_info", _hang) + monkeypatch.setattr(model_info, "MODEL_INFO_DISCOVERY_TIMEOUT", 0.2) + + started = time.time() + info = get_litellm_model_info( + secret_api_key=None, + base_url="http://localhost:9", + model="lemonade/some-self-hosted-model", + ) + elapsed = time.time() - started + + assert info is None + # Two bounded fallback probes at ~0.2s each must still return promptly. + assert elapsed < 5 From 5173128d654899f0a9530c86c62751d0cd25a542 Mon Sep 17 00:00:00 2001 From: Consolvo Date: Thu, 1 Oct 2026 09:43:40 -0400 Subject: [PATCH 2/6] =?UTF-8?q?fix(llm):=20address=20review=20=E2=80=94=20?= =?UTF-8?q?fix=20broken=20test=20assertion=20and=20shutdown=20hang?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - tests/sdk/llm/test_llm.py: the model-info probe now passes an explicit timeout to httpx.get, so test_base_url_for_openhands_provider must expect timeout=MODEL_INFO_DISCOVERY_TIMEOUT in the asserted call signature. - model_info.py: run the bounded probe on a dedicated daemon thread instead of a module-level ThreadPoolExecutor. The pool's atexit handler joined abandoned (uncancellable) workers, so a reachable-but-silent endpoint turned the request-path deadlock into a ~40s shutdown hang and 4 stuck probes could saturate the pool. A daemon thread is never joined at exit and each probe gets its own thread. Co-Authored-By: Claude --- .../openhands/sdk/llm/utils/model_info.py | 37 +++++++++++++------ tests/sdk/llm/test_llm.py | 2 + 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/openhands-sdk/openhands/sdk/llm/utils/model_info.py b/openhands-sdk/openhands/sdk/llm/utils/model_info.py index 8146987260f..7c10ca7b26b 100644 --- a/openhands-sdk/openhands/sdk/llm/utils/model_info.py +++ b/openhands-sdk/openhands/sdk/llm/utils/model_info.py @@ -1,6 +1,6 @@ +import threading import time from collections.abc import Callable, Mapping -from concurrent.futures import ThreadPoolExecutor, TimeoutError as FutureTimeoutError from functools import lru_cache from logging import getLogger from typing import Any @@ -32,10 +32,6 @@ # gracefully instead of deadlocking. MODEL_INFO_DISCOVERY_TIMEOUT = 10.0 -_discovery_executor = ThreadPoolExecutor( - max_workers=4, thread_name_prefix="model-info-discovery" -) - def _run_with_deadline[T]( func: Callable[..., T], @@ -46,21 +42,38 @@ def _run_with_deadline[T]( """Run a (possibly network-bound, non-cancellable) sync call with a deadline. Returns ``None`` if the call does not finish within ``timeout`` so that - model-info discovery never blocks the caller indefinitely. A timed-out - worker thread is abandoned (it cannot be cancelled) but is bounded by - litellm's own socket timeout and does not block the caller. + model-info discovery never blocks the caller indefinitely. + + The call runs on a dedicated daemon thread. A timed-out probe cannot be + cancelled, but because the thread is a daemon it is never joined at + interpreter shutdown -- so an endpoint that is reachable-but-silent (which + only litellm's much larger socket timeout would bound) delays neither the + caller nor process exit. A fresh thread per call also means a stuck probe + cannot saturate a shared worker pool and starve later discovery. """ if timeout is None: timeout = MODEL_INFO_DISCOVERY_TIMEOUT - future = _discovery_executor.submit(func, *args, **kwargs) - try: - return future.result(timeout=timeout) - except FutureTimeoutError: + + result: list[T] = [] + + def _target() -> None: + try: + result.append(func(*args, **kwargs)) + except Exception as e: + logger.debug("Model-info probe raised; ignoring: %s", e) + + thread = threading.Thread( + target=_target, name="model-info-discovery", daemon=True + ) + thread.start() + thread.join(timeout) + if thread.is_alive(): logger.warning( "Timed out after %ss while fetching model info; continuing without it.", timeout, ) return None + return result[0] if result else None def _merge_raw_model_metadata(model_info: Mapping[str, Any]) -> dict[str, Any]: diff --git a/tests/sdk/llm/test_llm.py b/tests/sdk/llm/test_llm.py index fba46fb80df..d3524e7ad6c 100644 --- a/tests/sdk/llm/test_llm.py +++ b/tests/sdk/llm/test_llm.py @@ -14,6 +14,7 @@ from openhands.sdk.llm.exceptions import LLMNoResponseError from openhands.sdk.llm.options.responses_options import select_responses_options from openhands.sdk.llm.utils.metrics import Metrics, TokenUsage +from openhands.sdk.llm.utils.model_info import MODEL_INFO_DISCOVERY_TIMEOUT from openhands.sdk.llm.utils.telemetry import Telemetry from openhands.sdk.tool.builtins.finish import FinishTool @@ -131,6 +132,7 @@ def test_base_url_for_openhands_provider(mock_get): mock_get.assert_called_once_with( "https://llm-proxy.app.all-hands.dev/v1/model/info", headers={"Authorization": "Bearer test-key"}, + timeout=MODEL_INFO_DISCOVERY_TIMEOUT, ) From 85ce41728137e5f1654f5646a06ee6d460012424 Mon Sep 17 00:00:00 2001 From: Consolvo Date: Thu, 1 Oct 2026 13:54:10 -0400 Subject: [PATCH 3/6] fix(llm): guard probe thread start during interpreter shutdown Match the daemon-thread idiom in utils/async_executor.py: Thread.start() can raise RuntimeError when the interpreter is shutting down and will not start new threads. Model info is optional, so return None in that case rather than probing inline (which could re-introduce the hang the deadline bounds). Co-Authored-By: Claude --- openhands-sdk/openhands/sdk/llm/utils/model_info.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/openhands-sdk/openhands/sdk/llm/utils/model_info.py b/openhands-sdk/openhands/sdk/llm/utils/model_info.py index 7c10ca7b26b..f973d7a0111 100644 --- a/openhands-sdk/openhands/sdk/llm/utils/model_info.py +++ b/openhands-sdk/openhands/sdk/llm/utils/model_info.py @@ -65,7 +65,13 @@ def _target() -> None: thread = threading.Thread( target=_target, name="model-info-discovery", daemon=True ) - thread.start() + try: + thread.start() + except RuntimeError: + # Interpreter is shutting down and will not start new threads. Model + # info is optional, so skip discovery rather than probing inline (which + # could re-introduce the very hang this guard exists to bound). + return None thread.join(timeout) if thread.is_alive(): logger.warning( From 3074209548c7471b2fb8aeb25e53a59a23e2c6a0 Mon Sep 17 00:00:00 2001 From: Consolvo Date: Mon, 5 Oct 2026 17:44:45 -0700 Subject: [PATCH 4/6] style(llm): apply ruff format to model-info-discovery thread call The precommit CI hook runs `ruff format`; the multi-line threading.Thread(...) call fits on one 88-col line, so ruff collapses it. Pre-apply to keep the hook green. Co-Authored-By: Claude --- openhands-sdk/openhands/sdk/llm/utils/model_info.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/openhands-sdk/openhands/sdk/llm/utils/model_info.py b/openhands-sdk/openhands/sdk/llm/utils/model_info.py index f973d7a0111..32e9209b87f 100644 --- a/openhands-sdk/openhands/sdk/llm/utils/model_info.py +++ b/openhands-sdk/openhands/sdk/llm/utils/model_info.py @@ -62,9 +62,7 @@ def _target() -> None: except Exception as e: logger.debug("Model-info probe raised; ignoring: %s", e) - thread = threading.Thread( - target=_target, name="model-info-discovery", daemon=True - ) + thread = threading.Thread(target=_target, name="model-info-discovery", daemon=True) try: thread.start() except RuntimeError: From 98db909d85967ebe141906a8231915ec99046eb9 Mon Sep 17 00:00:00 2001 From: Consolvo Date: Mon, 5 Oct 2026 17:48:01 -0700 Subject: [PATCH 5/6] fix(llm): bound the proxy underlying-model probe too _get_model_info_from_litellm_proxy resolves a proxy alias, then probes the underlying model via get_model_info() to make the alias priceable (#4816). That nested call was still unbounded -- a synchronous, network-bound litellm lookup with no timeout -- and it runs inside LLM._post_init, so an unreachable underlying endpoint on the litellm_proxy/* route could still freeze LLM construction and the agent-server event loop, the exact deadlock this PR targets. Wrap it with _run_with_deadline like the other probes. Adds a regression test that stubs the proxy to advertise a distinct underlying alias and makes get_model_info hang: without the wrap the call blocks ~30s, with it the alias model_info returns promptly and the underlying probe is abandoned. Co-Authored-By: Claude --- .../openhands/sdk/llm/utils/model_info.py | 8 ++- .../llm/test_model_info_discovery_timeout.py | 50 +++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/openhands-sdk/openhands/sdk/llm/utils/model_info.py b/openhands-sdk/openhands/sdk/llm/utils/model_info.py index 32e9209b87f..1676d9b7779 100644 --- a/openhands-sdk/openhands/sdk/llm/utils/model_info.py +++ b/openhands-sdk/openhands/sdk/llm/utils/model_info.py @@ -196,8 +196,14 @@ def _get_model_info_from_litellm_proxy( underlying_model = current.get("litellm_params", {}).get("model") underlying_model_info = None if isinstance(underlying_model, str) and underlying_model != stripped: + # Bound this probe too: like the discovery calls below, it is a + # synchronous, network-bound litellm lookup with no timeout, and + # it runs inside ``LLM._post_init`` -- so an unreachable + # underlying endpoint here would reintroduce the same deadlock. try: - underlying_model_info = get_model_info(underlying_model) + underlying_model_info = _run_with_deadline( + get_model_info, underlying_model + ) except Exception as e: logger.debug( f"get_model_info(underlying={underlying_model}) failed: {e}" diff --git a/tests/sdk/llm/test_model_info_discovery_timeout.py b/tests/sdk/llm/test_model_info_discovery_timeout.py index 464b609ee50..6db5c71097f 100644 --- a/tests/sdk/llm/test_model_info_discovery_timeout.py +++ b/tests/sdk/llm/test_model_info_discovery_timeout.py @@ -13,6 +13,7 @@ """ import time +from types import SimpleNamespace from openhands.sdk.llm.utils import model_info from openhands.sdk.llm.utils.model_info import ( @@ -65,3 +66,52 @@ def _hang(*_a, **_kw): assert info is None # Two bounded fallback probes at ~0.2s each must still return promptly. assert elapsed < 5 + + +def test_litellm_proxy_underlying_probe_does_not_hang(monkeypatch): + """The proxy alias-pricing probe must also be bounded. + + ``_get_model_info_from_litellm_proxy`` resolves a proxy alias, then probes + the *underlying* model via ``get_model_info`` to make the alias priceable. + That nested probe is a synchronous, network-bound litellm call with no + timeout and runs inside ``LLM._post_init``; an unreachable underlying + endpoint there would reintroduce the deadlock this PR targets. + """ + + def _fake_proxy_response(*_a, **_kw): + # Advertise an alias whose underlying model differs from the request, + # so the underlying-model probe branch runs. + return SimpleNamespace( + json=lambda: { + "data": [ + { + "model_name": "my-alias", + "litellm_params": {"model": "some/underlying-model"}, + "model_info": {"max_input_tokens": 100}, + } + ] + } + ) + + def _hang(*_a, **_kw): + time.sleep(30) + raise AssertionError("underlying probe should have been abandoned") + + monkeypatch.setattr(model_info.httpx, "get", _fake_proxy_response) + monkeypatch.setattr(model_info, "get_model_info", _hang) + monkeypatch.setattr(model_info, "MODEL_INFO_DISCOVERY_TIMEOUT", 0.2) + # Bypass the hourly lru_cache so this probe actually runs. + model_info._get_model_info_from_litellm_proxy.cache_clear() + + started = time.time() + info = get_litellm_model_info( + secret_api_key=None, + base_url="http://localhost:9", + model="litellm_proxy/my-alias", + ) + elapsed = time.time() - started + + # The alias's own model_info is still returned; only the underlying probe + # was abandoned, and it did so without blocking construction. + assert info == {"max_input_tokens": 100} + assert elapsed < 5 From b6cb261a188af7b3728ef4213fffc012bba2648c Mon Sep 17 00:00:00 2001 From: Consolvo Date: Wed, 18 Nov 2026 10:47:47 -0800 Subject: [PATCH 6/6] test(llm): expect proxy capability source in underlying-probe timeout test main now tags proxy-sourced model info with `_capability_source: "proxy"`, so the exact-dict assertion in the underlying-probe regression test must include it. The deadline behavior under test is unchanged. Co-Authored-By: Claude --- tests/sdk/llm/test_model_info_discovery_timeout.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/sdk/llm/test_model_info_discovery_timeout.py b/tests/sdk/llm/test_model_info_discovery_timeout.py index 6db5c71097f..07115148a34 100644 --- a/tests/sdk/llm/test_model_info_discovery_timeout.py +++ b/tests/sdk/llm/test_model_info_discovery_timeout.py @@ -113,5 +113,5 @@ def _hang(*_a, **_kw): # The alias's own model_info is still returned; only the underlying probe # was abandoned, and it did so without blocking construction. - assert info == {"max_input_tokens": 100} + assert info == {"max_input_tokens": 100, "_capability_source": "proxy"} assert elapsed < 5