Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 80 additions & 6 deletions openhands-sdk/openhands/sdk/llm/utils/model_info.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import threading
import time
from collections.abc import Mapping
from collections.abc import Callable, Mapping
from functools import lru_cache
from logging import getLogger
from typing import Any
Expand All @@ -16,6 +17,69 @@

logger = getLogger(__name__)

# Model-info discovery must never block LLM construction indefinitely.
#
# Some providers (self-hosted / OpenAI-compatible such as ``lemonade``,
# ``ollama`` and ``vllm``) resolve model info by making a live HTTP request to
# the model server rather than reading a static cost map. litellm performs that
# request synchronously with no timeout, so if the endpoint is unreachable -- or
# accidentally points back at the caller (e.g. a reverse proxy that forwards to
# this very process) -- the call hangs forever. Because ``LLM`` construction is
# synchronous and runs this probe in ``_post_init``, an indefinite hang freezes
# any caller, including an async server's event loop.
#
# Bound every probe with this deadline. ``model_info`` is an optional
# enhancement (callers already handle ``None``), so timing out degrades
# gracefully instead of deadlocking.
MODEL_INFO_DISCOVERY_TIMEOUT = 10.0


def _run_with_deadline[T](
func: Callable[..., T],
*args,
timeout: float | None = None,
**kwargs,
) -> T | None:
"""Run a (possibly network-bound, non-cancellable) sync call with a deadline.

Returns ``None`` if the call does not finish within ``timeout`` so that
model-info discovery never blocks the caller indefinitely.

The call runs on a dedicated daemon thread. A timed-out probe cannot be
cancelled, but because the thread is a daemon it is never joined at
interpreter shutdown -- so an endpoint that is reachable-but-silent (which
only litellm's much larger socket timeout would bound) delays neither the
caller nor process exit. A fresh thread per call also means a stuck probe
cannot saturate a shared worker pool and starve later discovery.
"""
if timeout is None:
timeout = MODEL_INFO_DISCOVERY_TIMEOUT

result: list[T] = []

def _target() -> None:
try:
result.append(func(*args, **kwargs))
except Exception as e:
logger.debug("Model-info probe raised; ignoring: %s", e)

thread = threading.Thread(target=_target, name="model-info-discovery", daemon=True)
try:
thread.start()
except RuntimeError:
# Interpreter is shutting down and will not start new threads. Model
# info is optional, so skip discovery rather than probing inline (which
# could re-introduce the very hang this guard exists to bound).
return None
thread.join(timeout)
if thread.is_alive():
logger.warning(
"Timed out after %ss while fetching model info; continuing without it.",
timeout,
)
return None
return result[0] if result else None


def _merge_raw_model_metadata(model_info: Mapping[str, Any]) -> dict[str, Any]:
"""Preserve raw LiteLLM capability fields."""
Expand Down Expand Up @@ -114,7 +178,11 @@ def _get_model_info_from_litellm_proxy(
if secret_api_key:
headers["Authorization"] = f"Bearer {secret_api_key}"

response = httpx.get(f"{base_url}/v1/model/info", headers=headers)
response = httpx.get(
f"{base_url}/v1/model/info",
headers=headers,
timeout=MODEL_INFO_DISCOVERY_TIMEOUT,
Comment thread
bconsolvo marked this conversation as resolved.
Resolved
Comment thread
bconsolvo marked this conversation as resolved.
Resolved
)
data = response.json().get("data", [])
# Match against either the public alias (`model_name`) or the
# underlying provider/model_name form (`litellm_params.model`). The proxy itself
Expand Down Expand Up @@ -154,8 +222,14 @@ def _get_model_info_from_litellm_proxy(
underlying_model = current.get("litellm_params", {}).get("model")
underlying_model_info = None
if isinstance(underlying_model, str) and underlying_model != stripped:
# Bound this probe too: like the discovery calls below, it is a
# synchronous, network-bound litellm lookup with no timeout, and
# it runs inside ``LLM._post_init`` -- so an unreachable
# underlying endpoint here would reintroduce the same deadlock.
try:
underlying_model_info = get_model_info(underlying_model)
underlying_model_info = _run_with_deadline(
get_model_info, underlying_model
)
except Exception as e:
logger.debug(
f"get_model_info(underlying={underlying_model}) failed: {e}"
Expand Down Expand Up @@ -185,7 +259,7 @@ def get_litellm_model_info(
# Try to get model info via openrouter or litellm proxy first
try:
if model.startswith("openrouter"):
model_info = get_model_info(model)
model_info = _run_with_deadline(get_model_info, model)
if model_info:
return _merge_raw_model_metadata(model_info)
except Exception as e:
Expand All @@ -206,13 +280,13 @@ def get_litellm_model_info(

# Fallbacks: try base name variants
try:
model_info = get_model_info(model.split(":")[0])
model_info = _run_with_deadline(get_model_info, model.split(":")[0])
if model_info:
return _merge_raw_model_metadata(model_info)
except Exception:
pass
try:
model_info = get_model_info(model.split("/")[-1])
model_info = _run_with_deadline(get_model_info, model.split("/")[-1])
if model_info:
return _merge_raw_model_metadata(model_info)
except Exception:
Expand Down
2 changes: 2 additions & 0 deletions tests/sdk/llm/test_llm.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
from openhands.sdk.llm.exceptions import LLMNoResponseError
from openhands.sdk.llm.options.responses_options import select_responses_options
from openhands.sdk.llm.utils.metrics import Metrics, TokenUsage
from openhands.sdk.llm.utils.model_info import MODEL_INFO_DISCOVERY_TIMEOUT
from openhands.sdk.llm.utils.telemetry import Telemetry
from openhands.sdk.tool.builtins.finish import FinishTool

Expand Down Expand Up @@ -133,6 +134,7 @@ def test_base_url_for_openhands_provider(mock_get):
mock_get.assert_called_once_with(
"https://llm-proxy.app.all-hands.dev/v1/model/info",
headers={"Authorization": "Bearer test-key"},
timeout=MODEL_INFO_DISCOVERY_TIMEOUT,
)


Expand Down
117 changes: 117 additions & 0 deletions tests/sdk/llm/test_model_info_discovery_timeout.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
"""Model-info discovery must never block the caller indefinitely.

Some providers (self-hosted / OpenAI-compatible such as ``lemonade``,
``ollama`` and ``vllm``) resolve model info via a live HTTP request to the
model server. litellm makes that request synchronously with no timeout, so an
unreachable endpoint -- or one that loops back to the caller through a reverse
proxy -- hangs forever. Because ``LLM`` construction runs this probe in
``_post_init``, that previously froze any caller, including an async server's
event loop.

These tests pin the bounded behavior: discovery returns ``None`` within the
deadline instead of hanging.
"""

import time
from types import SimpleNamespace

from openhands.sdk.llm.utils import model_info
from openhands.sdk.llm.utils.model_info import (
_run_with_deadline,
get_litellm_model_info,
)


def test_run_with_deadline_returns_none_on_timeout():
def _hang(*_a, **_kw):
time.sleep(30)
return "never"

started = time.time()
result = _run_with_deadline(_hang, timeout=0.2)
elapsed = time.time() - started

assert result is None
# Returned promptly rather than waiting for the (abandoned) call.
assert elapsed < 5


def test_run_with_deadline_returns_value_when_fast():
assert _run_with_deadline(lambda: 42, timeout=5) == 42


def test_get_litellm_model_info_does_not_hang_on_slow_probe(monkeypatch):
"""A self-hosted model whose model-info probe hangs must not block.

Regression test for the deadlock where ``litellm.get_model_info`` for an
unreachable self-hosted endpoint blocked ``LLM`` construction (and any
async caller) forever.
"""

def _hang(*_a, **_kw):
time.sleep(30)
raise AssertionError("should have been abandoned by the deadline")

monkeypatch.setattr(model_info, "get_model_info", _hang)
monkeypatch.setattr(model_info, "MODEL_INFO_DISCOVERY_TIMEOUT", 0.2)

started = time.time()
info = get_litellm_model_info(
secret_api_key=None,
base_url="http://localhost:9",
model="lemonade/some-self-hosted-model",
)
elapsed = time.time() - started

assert info is None
# Two bounded fallback probes at ~0.2s each must still return promptly.
assert elapsed < 5


def test_litellm_proxy_underlying_probe_does_not_hang(monkeypatch):
"""The proxy alias-pricing probe must also be bounded.

``_get_model_info_from_litellm_proxy`` resolves a proxy alias, then probes
the *underlying* model via ``get_model_info`` to make the alias priceable.
That nested probe is a synchronous, network-bound litellm call with no
timeout and runs inside ``LLM._post_init``; an unreachable underlying
endpoint there would reintroduce the deadlock this PR targets.
"""

def _fake_proxy_response(*_a, **_kw):
# Advertise an alias whose underlying model differs from the request,
# so the underlying-model probe branch runs.
return SimpleNamespace(
json=lambda: {
"data": [
{
"model_name": "my-alias",
"litellm_params": {"model": "some/underlying-model"},
"model_info": {"max_input_tokens": 100},
}
]
}
)

def _hang(*_a, **_kw):
time.sleep(30)
raise AssertionError("underlying probe should have been abandoned")

monkeypatch.setattr(model_info.httpx, "get", _fake_proxy_response)
monkeypatch.setattr(model_info, "get_model_info", _hang)
monkeypatch.setattr(model_info, "MODEL_INFO_DISCOVERY_TIMEOUT", 0.2)
# Bypass the hourly lru_cache so this probe actually runs.
model_info._get_model_info_from_litellm_proxy.cache_clear()

started = time.time()
info = get_litellm_model_info(
secret_api_key=None,
base_url="http://localhost:9",
model="litellm_proxy/my-alias",
)
elapsed = time.time() - started

# The alias's own model_info is still returned; only the underlying probe
# was abandoned, and it did so without blocking construction.
assert info == {"max_input_tokens": 100, "_capability_source": "proxy"}
assert elapsed < 5
Loading