diff --git a/.pr/agent-server-perf/README.md b/.pr/agent-server-perf/README.md new file mode 100644 index 00000000000..227f5766f31 --- /dev/null +++ b/.pr/agent-server-perf/README.md @@ -0,0 +1,19 @@ +# Agent-server lock-starvation / cold-load repro + +Runs a real `python -m openhands.agent_server` plus a mock OpenAI-compatible LLM +(`mock_llm.py`), driven over HTTP/WebSocket. `SDK_REPO=` picks the tree whose +`.venv` runs the server (default: this repo). + +```bash +# deadlock: cpu_count=2 (6-thread default executor), 5 WS clients, 3 s tools, 2 tools/step +.venv/bin/python .pr/agent-server-perf/bench_lock_starvation.py 2 5 3 label 2 + +# cold load: build a 300-step fixture, then measure /alive during the load (3 reps) +.venv/bin/python .pr/agent-server-perf/bench_cold_load.py build 300 500 +.venv/bin/python .pr/agent-server-perf/bench_cold_load.py measure 300 500 3 label +``` + +`results_*.jsonl` are the runs quoted in the PR (main = 8dc712f, fix12 = this branch). +`results_starvation_patch.jsonl` sends 6 tag PATCHes instead of WebSocket connects +(`... bench_lock_starvation.py 2 6 3 label 2 patch`); `patch-main-cs` is this branch +with main's `conversation_service.py`. diff --git a/.pr/agent-server-perf/bench_cold_load.py b/.pr/agent-server-perf/bench_cold_load.py new file mode 100644 index 00000000000..c1b07597f0c --- /dev/null +++ b/.pr/agent-server-perf/bench_cold_load.py @@ -0,0 +1,112 @@ +"""Scenario B: cold-loading a long conversation stalls the whole event loop. + +build: bench_cold_load.py build -> fixtures/conv--/ +measure: bench_cold_load.py measure [label] + +measure starts a fresh server on a copy of the fixture, probes /alive every +5 ms, then issues one events/search request that forces EventService.start() +(LocalConversation construction -> full event replay). Reports load latency +and the worst /alive latency seen during the load (= loop stall). +""" + +import asyncio +import json +import shutil +import statistics +import sys +import tempfile +import time +from pathlib import Path + +import httpx + +from harness import HERE, Stack, log + +FIX = HERE / "fixtures" + + +async def build(steps: int, nbytes: int): + async with Stack() as st: + cid = await st.create(f"LOOP {steps} BYTES {nbytes}") + t0 = time.monotonic() + s = await st.wait_status(cid, {"finished", "error"}, 3600) + log("built", cid, s, round(time.monotonic() - t0, 1), "s") + r = await st.client.get(f"/api/conversations/{cid}/events/count") + n_events = r.json() + dst = FIX / f"conv-{steps}-{nbytes}" + if dst.exists(): + shutil.rmtree(dst) + shutil.copytree(st.tmp / "work", dst, symlinks=True) + (dst / "meta.bench.json").write_text(json.dumps({"id": cid, "events": n_events})) + print(json.dumps({"built": steps, "events": n_events, "id": cid})) + + +async def measure_once(steps: int, nbytes: int) -> dict: + src = FIX / f"conv-{steps}-{nbytes}" + meta = json.loads((src / "meta.bench.json").read_text()) + tmp = Path(tempfile.mkdtemp(prefix="ohbench-cold-")) + shutil.copytree(src, tmp / "work", symlinks=True) + async with Stack(tmp=tmp) as st: + await asyncio.sleep(1.0) + samples: list[float] = [] + stop = asyncio.Event() + + async def probe(): + async with httpx.AsyncClient(base_url=st.base) as c: + while not stop.is_set(): + t0 = time.perf_counter() + await c.get("/alive", timeout=60) + samples.append(time.perf_counter() - t0) + await asyncio.sleep(0.005) + + pt = asyncio.create_task(probe()) + await asyncio.sleep(0.3) + base_n = len(samples) + t0 = time.perf_counter() + r = await st.client.get( + f"/api/conversations/{meta['id']}/events/search?limit=1", timeout=300 + ) + load = time.perf_counter() - t0 + r.raise_for_status() + await asyncio.sleep(0.2) + stop.set() + await pt + during = samples[base_n:] + # warm request after load, for contrast + t0 = time.perf_counter() + await st.client.get(f"/api/conversations/{meta['id']}/events/search?limit=1") + warm = time.perf_counter() - t0 + shutil.rmtree(tmp, ignore_errors=True) + return { + "events": meta["events"], + "cold_load_s": round(load, 3), + "warm_search_ms": round(warm * 1000, 1), + "alive_max_ms_during_load": round(max(during) * 1000, 1), + "alive_p50_ms_during_load": round(statistics.median(during) * 1000, 1), + } + + +async def main(): + mode = sys.argv[1] + steps = int(sys.argv[2]) + if mode == "build": + await build(steps, int(sys.argv[3])) + return + nbytes = int(sys.argv[3]) + reps = int(sys.argv[4]) + label = sys.argv[5] if len(sys.argv) > 5 else "run" + runs = [await measure_once(steps, nbytes) for _ in range(reps)] + out = { + "label": label, + "steps": steps, + "events": runs[0]["events"], + "cold_load_s_median": statistics.median(r["cold_load_s"] for r in runs), + "alive_max_ms_median": statistics.median(r["alive_max_ms_during_load"] for r in runs), + "warm_search_ms_median": statistics.median(r["warm_search_ms"] for r in runs), + "runs": runs, + } + print(json.dumps(out)) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/.pr/agent-server-perf/bench_lock_starvation.py b/.pr/agent-server-perf/bench_lock_starvation.py new file mode 100644 index 00000000000..664b3ffa7e5 --- /dev/null +++ b/.pr/agent-server-perf/bench_lock_starvation.py @@ -0,0 +1,156 @@ +"""Scenario A: FIFOLock waiters vs. the shared default executor. + +Conversation A runs a step with 2 sequential `terminal` tool calls (`sleep S`). +While the first tool runs (A's arun holds the state FIFOLock on the loop +thread), K WebSocket clients connect to A. Each connect snapshots state in the +default executor and blocks on the FIFOLock. Meanwhile a probe hits a *different* +idle conversation B's event search (lock-free, but also uses the default +executor) and /alive (pure loop). + +Reports: did A finish, how long, probe latencies on B and /alive. + +usage: bench_lock_starvation.py [sleep_s] [label] [ntools] [ws|patch] + +`patch` sends K tag PATCHes to A instead of WebSocket connects. +""" + +import asyncio +import json +import statistics +import sys +import time + +import httpx +import websockets + +from harness import Stack, log + + +async def probe(stack: Stack, path: str, stop: asyncio.Event, out: list, tag: str): + async with httpx.AsyncClient(base_url=stack.base) as c: + while not stop.is_set(): + t0 = time.perf_counter() + try: + r = await c.get(path, timeout=5) + ok = r.status_code == 200 + except Exception: + ok = False + dt = time.perf_counter() - t0 + out.append((time.monotonic(), dt, ok)) + await asyncio.sleep(0.05) + + +async def ws_client(stack: Stack, cid: str, opened: list, stop: asyncio.Event): + t0 = time.perf_counter() + try: + async with websockets.connect( + f"{stack.ws_base}/sockets/events/{cid}", open_timeout=60 + ) as ws: + # First frame = initial state snapshot (needs the FIFOLock). + await asyncio.wait_for(ws.recv(), timeout=60) + opened.append(time.perf_counter() - t0) + while not stop.is_set(): + with contextlib_suppress(): + await asyncio.wait_for(ws.recv(), timeout=0.5) + except Exception as e: # noqa: BLE001 + opened.append(("err", type(e).__name__)) + + +async def patch_client(stack: Stack, cid: str, opened: list, stop: asyncio.Event): + # A tag PATCH updates state and composes the webhook snapshot under the lock. + t0 = time.perf_counter() + try: + async with httpx.AsyncClient(base_url=stack.base) as c: + r = await c.patch( + f"/api/conversations/{cid}", json={"tags": {"bench": "x"}}, timeout=60 + ) + r.raise_for_status() + opened.append(time.perf_counter() - t0) + except Exception as e: # noqa: BLE001 + opened.append(("err", type(e).__name__)) + + +class contextlib_suppress: + def __enter__(self): + return self + + def __exit__(self, *exc): + return exc[0] is not None and issubclass(exc[0], (asyncio.TimeoutError, TimeoutError)) + + +def summarize(samples, t_from, t_to): + w = [dt for (t, dt, ok) in samples if t_from <= t <= t_to] + fails = sum(1 for (t, dt, ok) in samples if t_from <= t <= t_to and not ok) + if not w: + return {"n": 0} + return { + "n": len(w), + "p50_ms": round(statistics.median(w) * 1000, 1), + "max_ms": round(max(w) * 1000, 1), + "fails": fails, + } + + +async def main(): + cpu = int(sys.argv[1]) + K = int(sys.argv[2]) + sleep_s = float(sys.argv[3]) if len(sys.argv) > 3 else 4.0 + label = sys.argv[4] if len(sys.argv) > 4 else "run" + ntools = int(sys.argv[5]) if len(sys.argv) > 5 else 2 + mode = sys.argv[6] if len(sys.argv) > 6 else "ws" + client = {"ws": ws_client, "patch": patch_client}[mode] + async with Stack(cpu_count=cpu) as st: + b = await st.create("SLEEP 0") + assert await st.wait_status(b, {"finished"}, 60), "B did not finish" + a = await st.create(f"SLEEP {sleep_s} x{ntools}") + assert await st.wait_status(a, {"running"}, 30), "A did not start" + await asyncio.sleep(1.0) # first tool is now running; A holds the lock + + stop = asyncio.Event() + b_samples: list = [] + alive_samples: list = [] + probes = [ + asyncio.create_task(probe(st, f"/api/conversations/{b}/events/search?limit=5", stop, b_samples, "B")), + asyncio.create_task(probe(st, "/alive", stop, alive_samples, "alive")), + ] + t_start = time.monotonic() + opened: list = [] + clients = [asyncio.create_task(client(st, a, opened, stop)) for _ in range(K)] + + timeout = sleep_s * 2 + 25 + done = await st.wait_status(a, {"finished", "error"}, timeout) + t_end = time.monotonic() + a_secs = round(t_end - t_start, 2) + recheck = None + if not done: + await asyncio.sleep(60) + recheck = await st.status(a) + # let ws clients get their snapshots if unblocked + await asyncio.sleep(1.0) + stop.set() + for t in clients + probes: + t.cancel() + await asyncio.gather(*clients, *probes, return_exceptions=True) + + ok_open = [x for x in opened if isinstance(x, float)] + result = { + "label": label, + "cpu_count": cpu, + "default_executor_workers": min(32, cpu + 4), + "client_mode": mode, + "K_ws_clients": K, + "tool_sleep_s": sleep_s, + "A_final_status": done or "STUCK (timeout)", + "A_secs_from_storm": a_secs if done else f">{timeout}", + "A_status_60s_later": recheck, + "tools_in_step": ntools, + "ws_snapshots_received": len(ok_open), + "ws_snapshot_p50_s": round(statistics.median(ok_open), 2) if ok_open else None, + "probe_B_event_search": summarize(b_samples, t_start, t_end), + "probe_alive": summarize(alive_samples, t_start, t_end), + } + print(json.dumps(result)) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/.pr/agent-server-perf/harness.py b/.pr/agent-server-perf/harness.py new file mode 100644 index 00000000000..dbc61fa1804 --- /dev/null +++ b/.pr/agent-server-perf/harness.py @@ -0,0 +1,150 @@ +"""Spawn a real agent-server + mock LLM as subprocesses and drive them over HTTP/WS.""" + +import asyncio +import contextlib +import os +import socket +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +import httpx + +HERE = Path(__file__).parent +# Checkout whose .venv runs the server; defaults to the repo containing .pr/. +REPO = Path(os.environ.get("SDK_REPO", Path(__file__).resolve().parents[2])) +PY = str(REPO / ".venv/bin/python") + + +def free_port() -> int: + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +async def wait_http(url: str, timeout: float = 90) -> None: + deadline = time.monotonic() + timeout + async with httpx.AsyncClient() as c: + while time.monotonic() < deadline: + with contextlib.suppress(Exception): + r = await c.get(url, timeout=2) + if r.status_code < 500: + return + await asyncio.sleep(0.25) + raise TimeoutError(url) + + +class Stack: + def __init__(self, cpu_count: int | None = None, extra_env: dict | None = None, + stream: bool = False, tmp: Path | None = None): + self.cpu_count = cpu_count + self.extra_env = extra_env or {} + self.stream = stream + self.tmp = tmp or Path(tempfile.mkdtemp(prefix="ohbench-")) + self.llm_port = free_port() + self.port = free_port() + self.procs: list[subprocess.Popen] = [] + + @property + def base(self) -> str: + return f"http://127.0.0.1:{self.port}" + + @property + def ws_base(self) -> str: + return f"ws://127.0.0.1:{self.port}" + + async def __aenter__(self): + env = {**os.environ, "OPENHANDS_SUPPRESS_BANNER": "1", **self.extra_env} + env.pop("OH_SESSION_API_KEYS_0", None) + env.pop("SESSION_API_KEY", None) + llm_log = open(self.tmp / "llm.log", "w") + self.procs.append(subprocess.Popen( + [PY, str(HERE / "mock_llm.py"), str(self.llm_port)], + stdout=llm_log, stderr=subprocess.STDOUT, env=env, + )) + srv_env = dict(env) + if self.cpu_count: + srv_env["PYTHON_CPU_COUNT"] = str(self.cpu_count) + srv_env.setdefault("LOG_LEVEL", "WARNING") + srv_log = open(self.tmp / "server.log", "w") + (self.tmp / "work").mkdir(exist_ok=True) + self.procs.append(subprocess.Popen( + [PY, "-m", "openhands.agent_server", "--host", "127.0.0.1", + "--port", str(self.port)], + cwd=self.tmp / "work", stdout=srv_log, stderr=subprocess.STDOUT, + env=srv_env, + )) + await wait_http(f"http://127.0.0.1:{self.llm_port}/stats") + await wait_http(f"{self.base}/ready", timeout=180) + self.client = httpx.AsyncClient(base_url=self.base, timeout=120) + return self + + async def __aexit__(self, *exc): + with contextlib.suppress(Exception): + await self.client.aclose() + for p in self.procs: + p.terminate() + for p in self.procs: + with contextlib.suppress(Exception): + p.wait(timeout=10) + if p.poll() is None: + p.kill() + + def agent_spec(self) -> dict: + return { + "llm": { + "model": "openai/mock", + "base_url": f"http://127.0.0.1:{self.llm_port}/v1", + "api_key": "sk-mock", + "usage_id": "agent", + "stream": self.stream, + "num_retries": 0, + }, + "tools": [{"name": "terminal"}], + } + + async def create(self, text: str | None = None, run: bool = True) -> str: + body = { + "agent": self.agent_spec(), + "workspace": {"working_dir": str(self.tmp / "ws")}, + "autotitle": False, + } + if text is not None: + body["initial_message"] = { + "role": "user", "content": [{"type": "text", "text": text}], + "run": run, + } + r = await self.client.post("/api/conversations", json=body) + r.raise_for_status() + return r.json()["id"] + + async def status(self, cid: str) -> str: + try: + r = await self.client.get(f"/api/conversations/{cid}", timeout=5) + except httpx.TimeoutException: + return "unresponsive" + r.raise_for_status() + return r.json()["execution_status"] + + async def wait_status(self, cid: str, want: set[str], timeout: float) -> str | None: + deadline = time.monotonic() + timeout + last = None + while time.monotonic() < deadline: + last = await self.status(cid) + if last in want: + return last + await asyncio.sleep(0.1) + return None + + async def send(self, cid: str, text: str, run: bool = True) -> None: + r = await self.client.post( + f"/api/conversations/{cid}/events", + json={"role": "user", "content": [{"type": "text", "text": text}], "run": run}, + ) + r.raise_for_status() + + +def log(*a): + print(*a, flush=True, file=sys.stderr) diff --git a/.pr/agent-server-perf/mock_llm.py b/.pr/agent-server-perf/mock_llm.py new file mode 100644 index 00000000000..8423f407d0d --- /dev/null +++ b/.pr/agent-server-perf/mock_llm.py @@ -0,0 +1,156 @@ +"""Minimal OpenAI-compatible chat-completions server for agent-server benchmarks. + +The last *user* message is a tiny script: + + SLEEP [x] [LAT ] [TOKENS ] + +- First LLM call after the user message: return N parallel `terminal` tool calls + running `sleep ` (N defaults to 1). +- Any later call (tool results present): return a `finish` tool call. +- LAT adds artificial LLM latency per call (default 0). +- TOKENS makes the final answer a plain-text message of n words (no tools) so + streaming/large-content paths can be exercised. +""" + +import asyncio +import json +import re +import sys +import time +import uuid + +import uvicorn +from starlette.applications import Starlette +from starlette.requests import Request +from starlette.responses import JSONResponse, StreamingResponse +from starlette.routing import Route + +CALLS = {"n": 0} + + +def _text(content) -> str: + if isinstance(content, str): + return content + if isinstance(content, list): + return " ".join(p.get("text", "") for p in content if isinstance(p, dict)) + return "" + + +def _plan(messages: list[dict]) -> tuple[str, dict]: + last_user_idx = max( + (i for i, m in enumerate(messages) if m.get("role") == "user"), default=-1 + ) + script = _text(messages[last_user_idx]["content"]) if last_user_idx >= 0 else "" + after = messages[last_user_idx + 1 :] + tool_results = sum(1 for m in after if m.get("role") == "tool") + m = re.search(r"SLEEP\s+([\d.]+)(?:\s+x(\d+))?", script) + lat = re.search(r"LAT\s+(\d+)", script) + toks = re.search(r"TOKENS\s+(\d+)", script) + rate = re.search(r"RATE\s+(\d+)", script) + opts = { + "lat": int(lat.group(1)) / 1000 if lat else 0.0, + "tokens": int(toks.group(1)) if toks else 0, + "rate": int(rate.group(1)) if rate else 0, + } + if m and tool_results == 0: + return "tools", {**opts, "sleep": m.group(1), "n": int(m.group(2) or 1)} + loop = re.search(r"LOOP\s+(\d+)(?:\s+BYTES\s+(\d+))?", script) + if loop and tool_results < int(loop.group(1)): + return "loop", {**opts, "bytes": int(loop.group(2) or 200), "i": tool_results} + return "finish", opts + + +def _tool_call(name: str, args: dict) -> dict: + return { + "id": f"call_{uuid.uuid4().hex[:12]}", + "type": "function", + "function": {"name": name, "arguments": json.dumps(args)}, + } + + +async def completions(request: Request): + body = await request.json() + CALLS["n"] += 1 + kind, opts = _plan(body.get("messages", [])) + if opts["lat"]: + await asyncio.sleep(opts["lat"]) + if kind == "tools": + message = { + "role": "assistant", + "content": None, + "tool_calls": [ + _tool_call( + "terminal", + {"command": f"sleep {opts['sleep']}", "security_risk": "LOW"}, + ) + for _ in range(opts["n"]) + ], + } + elif kind == "loop": + message = { + "role": "assistant", + "content": None, + "tool_calls": [_tool_call("terminal", { + "command": f"echo step-{opts['i']}; head -c {opts['bytes']} /dev/zero | tr '\\0' x; echo", + "security_risk": "LOW", + })], + } + elif opts["tokens"]: + message = { + "role": "assistant", + "content": " ".join(f"w{i}" for i in range(opts["tokens"])), + } + else: + message = { + "role": "assistant", + "content": None, + "tool_calls": [_tool_call("finish", {"message": "done"})], + } + resp = { + "id": f"chatcmpl-{uuid.uuid4().hex[:8]}", + "object": "chat.completion", + "created": int(time.time()), + "model": body.get("model", "mock"), + "choices": [{"index": 0, "message": message, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 100, "completion_tokens": 10, "total_tokens": 110}, + } + if not body.get("stream"): + return JSONResponse(resp) + + async def gen(): + # Stream content word-by-word; tool calls in one chunk. + base = {k: resp[k] for k in ("id", "created", "model")} + base["object"] = "chat.completion.chunk" + if message.get("content"): + for w in message["content"].split(" "): + chunk = {**base, "choices": [{"index": 0, "delta": {"content": w + " "}}]} + yield f"data: {json.dumps(chunk)}\n\n" + if opts["rate"]: + await asyncio.sleep(1 / opts["rate"]) + if message.get("tool_calls"): + tcs = [ + {**tc, "index": i} for i, tc in enumerate(message["tool_calls"]) + ] + chunk = {**base, "choices": [{"index": 0, "delta": {"role": "assistant", "tool_calls": tcs}}]} + yield f"data: {json.dumps(chunk)}\n\n" + end = {**base, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], "usage": resp["usage"]} + yield f"data: {json.dumps(end)}\n\n" + yield "data: [DONE]\n\n" + + return StreamingResponse(gen(), media_type="text/event-stream") + + +async def stats(_: Request): + return JSONResponse(CALLS) + + +app = Starlette( + routes=[ + Route("/v1/chat/completions", completions, methods=["POST"]), + Route("/chat/completions", completions, methods=["POST"]), + Route("/stats", stats), + ] +) + +if __name__ == "__main__": + uvicorn.run(app, host="127.0.0.1", port=int(sys.argv[1]), log_level="warning") diff --git a/.pr/agent-server-perf/results_cold_load.jsonl b/.pr/agent-server-perf/results_cold_load.jsonl new file mode 100644 index 00000000000..05d34f9e4cd --- /dev/null +++ b/.pr/agent-server-perf/results_cold_load.jsonl @@ -0,0 +1,6 @@ +{"label": "main", "steps": 300, "events": 908, "cold_load_s_median": 0.226, "alive_max_ms_median": 106.4, "warm_search_ms_median": 3.0, "runs": [{"events": 908, "cold_load_s": 0.25, "warm_search_ms": 3.0, "alive_max_ms_during_load": 123.6, "alive_p50_ms_during_load": 1.8}, {"events": 908, "cold_load_s": 0.216, "warm_search_ms": 3.5, "alive_max_ms_during_load": 103.1, "alive_p50_ms_during_load": 1.7}, {"events": 908, "cold_load_s": 0.226, "warm_search_ms": 2.9, "alive_max_ms_during_load": 106.4, "alive_p50_ms_during_load": 1.7}]} +{"label": "fixed", "steps": 300, "events": 908, "cold_load_s_median": 0.245, "alive_max_ms_median": 8.9, "warm_search_ms_median": 4.0, "runs": [{"events": 908, "cold_load_s": 0.239, "warm_search_ms": 4.5, "alive_max_ms_during_load": 8.7, "alive_p50_ms_during_load": 1.9}, {"events": 908, "cold_load_s": 0.33, "warm_search_ms": 3.1, "alive_max_ms_during_load": 11.4, "alive_p50_ms_during_load": 1.8}, {"events": 908, "cold_load_s": 0.245, "warm_search_ms": 4.0, "alive_max_ms_during_load": 8.9, "alive_p50_ms_during_load": 1.8}]} +{"label": "main", "steps": 1500, "events": 1506, "cold_load_s_median": 0.355, "alive_max_ms_median": 198.9, "warm_search_ms_median": 3.9, "runs": [{"events": 1506, "cold_load_s": 0.617, "warm_search_ms": 4.7, "alive_max_ms_during_load": 306.3, "alive_p50_ms_during_load": 2.9}, {"events": 1506, "cold_load_s": 0.355, "warm_search_ms": 3.3, "alive_max_ms_during_load": 198.9, "alive_p50_ms_during_load": 1.7}, {"events": 1506, "cold_load_s": 0.292, "warm_search_ms": 3.9, "alive_max_ms_during_load": 163.4, "alive_p50_ms_during_load": 1.4}]} +{"label": "fixed", "steps": 1500, "events": 1506, "cold_load_s_median": 0.357, "alive_max_ms_median": 8.6, "warm_search_ms_median": 3.4, "runs": [{"events": 1506, "cold_load_s": 0.368, "warm_search_ms": 3.6, "alive_max_ms_during_load": 8.9, "alive_p50_ms_during_load": 1.8}, {"events": 1506, "cold_load_s": 0.338, "warm_search_ms": 3.2, "alive_max_ms_during_load": 6.9, "alive_p50_ms_during_load": 1.5}, {"events": 1506, "cold_load_s": 0.357, "warm_search_ms": 3.4, "alive_max_ms_during_load": 8.6, "alive_p50_ms_during_load": 1.7}]} +{"label": "main", "steps": 400, "events": 1208, "cold_load_s_median": 0.286, "alive_max_ms_median": 152.4, "warm_search_ms_median": 3.3, "runs": [{"events": 1208, "cold_load_s": 0.368, "warm_search_ms": 5.8, "alive_max_ms_during_load": 174.8, "alive_p50_ms_during_load": 2.4}, {"events": 1208, "cold_load_s": 0.268, "warm_search_ms": 3.0, "alive_max_ms_during_load": 139.8, "alive_p50_ms_during_load": 1.6}, {"events": 1208, "cold_load_s": 0.286, "warm_search_ms": 3.3, "alive_max_ms_during_load": 152.4, "alive_p50_ms_during_load": 1.5}]} +{"label": "fixed", "steps": 400, "events": 1208, "cold_load_s_median": 0.31, "alive_max_ms_median": 9.0, "warm_search_ms_median": 3.7, "runs": [{"events": 1208, "cold_load_s": 0.324, "warm_search_ms": 3.8, "alive_max_ms_during_load": 11.9, "alive_p50_ms_during_load": 1.8}, {"events": 1208, "cold_load_s": 0.297, "warm_search_ms": 3.7, "alive_max_ms_during_load": 8.1, "alive_p50_ms_during_load": 1.6}, {"events": 1208, "cold_load_s": 0.31, "warm_search_ms": 3.6, "alive_max_ms_during_load": 9.0, "alive_p50_ms_during_load": 1.7}]} diff --git a/.pr/agent-server-perf/results_starvation_fix12.jsonl b/.pr/agent-server-perf/results_starvation_fix12.jsonl new file mode 100644 index 00000000000..fbafec8b979 --- /dev/null +++ b/.pr/agent-server-perf/results_starvation_fix12.jsonl @@ -0,0 +1,6 @@ +{"label": "fix12", "cpu_count": 2, "default_executor_workers": 6, "K_ws_clients": 4, "tool_sleep_s": 3.0, "A_final_status": "finished", "A_secs_from_storm": 6.83, "A_status_60s_later": null, "tools_in_step": 2, "ws_snapshots_received": 4, "ws_snapshot_p50_s": 6.7, "probe_B_event_search": {"n": 118, "p50_ms": 4.8, "max_ms": 54.5, "fails": 0}, "probe_alive": {"n": 125, "p50_ms": 2.5, "max_ms": 34.8, "fails": 0}} +{"label": "fix12", "cpu_count": 2, "default_executor_workers": 6, "K_ws_clients": 5, "tool_sleep_s": 3.0, "A_final_status": "finished", "A_secs_from_storm": 6.65, "A_status_60s_later": null, "tools_in_step": 2, "ws_snapshots_received": 5, "ws_snapshot_p50_s": 6.47, "probe_B_event_search": {"n": 118, "p50_ms": 4.1, "max_ms": 46.7, "fails": 0}, "probe_alive": {"n": 124, "p50_ms": 2.3, "max_ms": 27.2, "fails": 0}} +{"label": "fix12", "cpu_count": 2, "default_executor_workers": 6, "K_ws_clients": 6, "tool_sleep_s": 3.0, "A_final_status": "finished", "A_secs_from_storm": 7.33, "A_status_60s_later": null, "tools_in_step": 2, "ws_snapshots_received": 6, "ws_snapshot_p50_s": 7.16, "probe_B_event_search": {"n": 133, "p50_ms": 3.7, "max_ms": 45.0, "fails": 0}, "probe_alive": {"n": 137, "p50_ms": 2.1, "max_ms": 28.9, "fails": 0}} +{"label": "fix12", "cpu_count": 2, "default_executor_workers": 6, "K_ws_clients": 6, "tool_sleep_s": 3.0, "A_final_status": "finished", "A_secs_from_storm": 3.03, "A_status_60s_later": null, "tools_in_step": 1, "ws_snapshots_received": 6, "ws_snapshot_p50_s": 2.96, "probe_B_event_search": {"n": 54, "p50_ms": 3.8, "max_ms": 51.6, "fails": 0}, "probe_alive": {"n": 57, "p50_ms": 2.0, "max_ms": 31.7, "fails": 0}} +{"label": "fix12", "cpu_count": 2, "default_executor_workers": 6, "K_ws_clients": 12, "tool_sleep_s": 3.0, "A_final_status": "finished", "A_secs_from_storm": 6.51, "A_status_60s_later": null, "tools_in_step": 2, "ws_snapshots_received": 12, "ws_snapshot_p50_s": 6.42, "probe_B_event_search": {"n": 116, "p50_ms": 4.1, "max_ms": 58.0, "fails": 0}, "probe_alive": {"n": 120, "p50_ms": 2.3, "max_ms": 43.9, "fails": 0}} +{"label": "fix12", "cpu_count": 2, "default_executor_workers": 6, "K_ws_clients": 32, "tool_sleep_s": 3.0, "A_final_status": "finished", "A_secs_from_storm": 6.9, "A_status_60s_later": null, "tools_in_step": 2, "ws_snapshots_received": 32, "ws_snapshot_p50_s": 6.82, "probe_B_event_search": {"n": 121, "p50_ms": 4.1, "max_ms": 85.8, "fails": 0}, "probe_alive": {"n": 127, "p50_ms": 2.3, "max_ms": 63.0, "fails": 0}} diff --git a/.pr/agent-server-perf/results_starvation_main.jsonl b/.pr/agent-server-perf/results_starvation_main.jsonl new file mode 100644 index 00000000000..1347626789c --- /dev/null +++ b/.pr/agent-server-perf/results_starvation_main.jsonl @@ -0,0 +1,5 @@ +{"label": "main", "cpu_count": 2, "default_executor_workers": 6, "K_ws_clients": 4, "tool_sleep_s": 3.0, "A_final_status": "finished", "A_secs_from_storm": 6.65, "A_status_60s_later": null, "tools_in_step": 2, "ws_snapshots_received": 4, "ws_snapshot_p50_s": 6.45, "probe_B_event_search": {"n": 118, "p50_ms": 4.0, "max_ms": 62.2, "fails": 0}, "probe_alive": {"n": 122, "p50_ms": 2.2, "max_ms": 38.0, "fails": 0}} +{"label": "main", "cpu_count": 2, "default_executor_workers": 6, "K_ws_clients": 5, "tool_sleep_s": 3.0, "A_final_status": "STUCK (timeout)", "A_secs_from_storm": ">31.0", "A_status_60s_later": "running", "tools_in_step": 2, "ws_snapshots_received": 0, "ws_snapshot_p50_s": null, "probe_B_event_search": {"n": 6, "p50_ms": 5003.7, "max_ms": 5035.1, "fails": 6}, "probe_alive": {"n": 581, "p50_ms": 2.3, "max_ms": 49.8, "fails": 0}} +{"label": "main", "cpu_count": 2, "default_executor_workers": 6, "K_ws_clients": 6, "tool_sleep_s": 3.0, "A_final_status": "STUCK (timeout)", "A_secs_from_storm": ">31.0", "A_status_60s_later": "unresponsive", "tools_in_step": 2, "ws_snapshots_received": 0, "ws_snapshot_p50_s": null, "probe_B_event_search": {"n": 6, "p50_ms": 5015.0, "max_ms": 5354.3, "fails": 6}, "probe_alive": {"n": 482, "p50_ms": 4.7, "max_ms": 374.7, "fails": 0}} +{"label": "main", "cpu_count": 2, "default_executor_workers": 6, "K_ws_clients": 6, "tool_sleep_s": 3.0, "A_final_status": "finished", "A_secs_from_storm": 3.14, "A_status_60s_later": null, "tools_in_step": 1, "ws_snapshots_received": 6, "ws_snapshot_p50_s": 3.05, "probe_B_event_search": {"n": 1, "p50_ms": 3090.2, "max_ms": 3090.2, "fails": 0}, "probe_alive": {"n": 56, "p50_ms": 2.7, "max_ms": 53.2, "fails": 0}} +{"label": "main", "cpu_count": 2, "default_executor_workers": 6, "K_ws_clients": 12, "tool_sleep_s": 3.0, "A_final_status": "STUCK (timeout)", "A_secs_from_storm": ">31.0", "A_status_60s_later": "running", "tools_in_step": 2, "ws_snapshots_received": 0, "ws_snapshot_p50_s": null, "probe_B_event_search": {"n": 6, "p50_ms": 5006.2, "max_ms": 5035.2, "fails": 6}, "probe_alive": {"n": 557, "p50_ms": 2.8, "max_ms": 40.3, "fails": 0}} diff --git a/.pr/agent-server-perf/results_starvation_patch.jsonl b/.pr/agent-server-perf/results_starvation_patch.jsonl new file mode 100644 index 00000000000..06f692fa353 --- /dev/null +++ b/.pr/agent-server-perf/results_starvation_patch.jsonl @@ -0,0 +1,2 @@ +{"label": "patch-fix", "cpu_count": 2, "default_executor_workers": 6, "client_mode": "patch", "K_ws_clients": 6, "tool_sleep_s": 3.0, "A_final_status": "finished", "A_secs_from_storm": 6.01, "A_status_60s_later": null, "tools_in_step": 2, "ws_snapshots_received": 6, "ws_snapshot_p50_s": 5.91, "probe_B_event_search": {"n": 110, "p50_ms": 2.7, "max_ms": 64.7, "fails": 0}, "probe_alive": {"n": 115, "p50_ms": 1.3, "max_ms": 34.7, "fails": 0}} +{"label": "patch-main-cs", "cpu_count": 2, "default_executor_workers": 6, "client_mode": "patch", "K_ws_clients": 6, "tool_sleep_s": 3.0, "A_final_status": "finished", "A_secs_from_storm": 6.55, "A_status_60s_later": null, "tools_in_step": 2, "ws_snapshots_received": 6, "ws_snapshot_p50_s": 6.42, "probe_B_event_search": {"n": 4, "p50_ms": 673.4, "max_ms": 5036.8, "fails": 1}, "probe_alive": {"n": 124, "p50_ms": 1.5, "max_ms": 34.1, "fails": 0}} diff --git a/openhands-agent-server/AGENTS.md b/openhands-agent-server/AGENTS.md index bba83b57e8e..c6b152d18df 100644 --- a/openhands-agent-server/AGENTS.md +++ b/openhands-agent-server/AGENTS.md @@ -123,7 +123,7 @@ feature map (`control-agent-server map coverage`). - `ConversationState` uses a synchronous `FIFOLock`. In async agent-server code, never do `with conversation._state` directly on the event loop when the conversation may be running. - WebSocket reconnects call `EventService.subscribe_to_events()` immediately; if initial state snapshot creation blocks on the state lock in async context, the whole FastAPI event loop can stop serving `/ready` and similar probes. - The same rule applies to metadata updates in `ConversationService.update_conversation()`: keep the locked mutation/snapshot semantics, but move the synchronous lock wait into a worker thread first. -- In async routes/services, move state-lock acquisition into `run_in_executor(...)` (or another worker-thread boundary) before awaiting network I/O. +- In async routes/services, move state-lock acquisition into `EventService.run_state_op(...)` before awaiting network I/O. It runs on the conversation's own executor: a run holds the lock for a whole tool step, and waiters on asyncio's default executor (`run_in_executor(None, ...)`, `asyncio.to_thread`) take the threads that event reads of every conversation need. ## REST API compatibility & deprecation policy diff --git a/openhands-agent-server/openhands/agent_server/conversation_service.py b/openhands-agent-server/openhands/agent_server/conversation_service.py index 6942f67ac2c..ee6a501fdb6 100644 --- a/openhands-agent-server/openhands/agent_server/conversation_service.py +++ b/openhands-agent-server/openhands/agent_server/conversation_service.py @@ -1,4 +1,5 @@ import asyncio +import contextvars import importlib import json import logging @@ -7,6 +8,7 @@ from concurrent.futures import ThreadPoolExecutor from contextlib import asynccontextmanager, suppress from dataclasses import dataclass, field +from functools import partial from pathlib import Path from typing import TYPE_CHECKING, Any, cast from uuid import UUID, uuid4 @@ -2000,7 +2002,6 @@ async def update_conversation( if event_service is None: return False - loop = asyncio.get_running_loop() state = await event_service.get_state() update: dict[str, Any] = {"updated_at": utc_now()} if request.title is not None: @@ -2015,8 +2016,8 @@ async def update_conversation( if request.tags is not None: # Keep the persisted ConversationState update under the state lock so # autosave and state-change callbacks observe a consistent mutation. - state = await loop.run_in_executor( - None, _update_state_tags_sync, state, request.tags + state = await event_service.run_state_op( + _update_state_tags_sync, state, request.tags ) if budget_updated: # base_state.json owns the budget; the event service is loaded @@ -2030,8 +2031,8 @@ async def update_conversation( # Notify conversation webhooks about the updated conversation. Compose the # full-state snapshot under the state lock, but do the synchronous wait in a # worker thread so metadata updates cannot block the FastAPI event loop. - conversation_info = await loop.run_in_executor( - None, _compose_webhook_conversation_info_sync, event_service.stored, state + conversation_info = await event_service.run_state_op( + _compose_webhook_conversation_info_sync, event_service.stored, state ) await self._notify_conversation_webhooks(conversation_info) @@ -2146,12 +2147,15 @@ async def fork_conversation( raise ValueError(f"Conversation with id {fork_id} already exists") # fork() deep-copies events and state into ``fork_dir``. - fork_conv = await asyncio.to_thread( - source_conversation.fork, - conversation_id=fork_conv_id, - tags=tags, - reset_metrics=reset_metrics, - from_event_id=from_event_id, + fork_conv = await source_service.run_state_op( + partial( + contextvars.copy_context().run, + source_conversation.fork, + conversation_id=fork_conv_id, + tags=tags, + reset_metrics=reset_metrics, + from_event_id=from_event_id, + ) ) # Extract the persisted data, then discard the temporary conversation. fork_agent = cast(AgentBase, fork_conv.agent) diff --git a/openhands-agent-server/openhands/agent_server/event_service.py b/openhands-agent-server/openhands/agent_server/event_service.py index f1a6ef3c0fb..a877d6ecc7d 100644 --- a/openhands-agent-server/openhands/agent_server/event_service.py +++ b/openhands-agent-server/openhands/agent_server/event_service.py @@ -1,15 +1,16 @@ import asyncio +import contextvars import threading import time import traceback -from collections.abc import Mapping +from collections.abc import Callable, Mapping from concurrent.futures import ThreadPoolExecutor from contextlib import nullcontext, suppress from dataclasses import dataclass, field from datetime import datetime from functools import partial from pathlib import Path -from typing import cast +from typing import Any, cast from uuid import UUID, uuid4 from pydantic import ValidationError @@ -317,6 +318,11 @@ class EventService: _lease_task: asyncio.Task | None = field(default=None, init=False) _external_lease_renewal: bool = field(default=False, init=False) _run_executor: ThreadPoolExecutor | None = field(default=None, init=False) + # Runs calls that may block on this conversation's state lock, which a run + # holds across whole tool steps. On the shared default executor those + # waiters starve lock-free work server-wide (other conversations' event + # reads); here they only queue behind their own conversation. + _state_executor: ThreadPoolExecutor | None = field(default=None, init=False) _run_semaphore: asyncio.Semaphore | None = field(default=None, init=False) # Permit for this conversation's current admission session. Held for the # whole chain of runs (initial run, any re-arm for stranded input, ACP @@ -343,6 +349,11 @@ class EventService: def conversation_dir(self): return self.conversations_dir / self.stored.id.hex + async def run_state_op[R](self, fn: Callable[..., R], *args: Any) -> R: + """Run ``fn``, which acquires the state lock, off the event loop.""" + loop = asyncio.get_running_loop() + return await loop.run_in_executor(self._state_executor, fn, *args) + async def load_meta(self): meta_file = self.conversation_dir / "meta.json" self.stored = StoredConversation.model_validate_json( @@ -528,7 +539,7 @@ def _update() -> None: if isinstance(agent, ACPAgent): agent.restart_for_updated_credentials(secrets) - await asyncio.to_thread(_update) + await self.run_state_op(contextvars.copy_context().run, _update) await self.save_meta( self.stored.model_copy( update={"secrets": {**self.stored.secrets, **secrets}} @@ -834,8 +845,7 @@ def _get_execution_status_sync(self) -> ConversationExecutionStatus: return state.execution_status async def _get_execution_status(self) -> ConversationExecutionStatus: - loop = asyncio.get_running_loop() - return await loop.run_in_executor(None, self._get_execution_status_sync) + return await self.run_state_op(self._get_execution_status_sync) def _mark_error_status_sync(self) -> None: """Force the conversation into ERROR status (idempotent backstop). @@ -894,8 +904,7 @@ def _create_initial_state_sync( return self._create_state_update_event_sync(), len(state.events) async def _create_state_update_event(self) -> ConversationStateUpdateEvent: - loop = asyncio.get_running_loop() - return await loop.run_in_executor(None, self._create_state_update_event_sync) + return await self.run_state_op(self._create_state_update_event_sync) def _event_matches_body(self, event: Event, body: str) -> bool: """Check if event's message content matches body filter (case-insensitive).""" @@ -961,13 +970,12 @@ async def send_message( if not _from_goal_loop: await self.stop_goal_loop() explicit_interrupt_generation = self._explicit_interrupt_generation - loop = asyncio.get_running_loop() send_message = ( partial(self._conversation.send_message, sender="goal") if _from_goal_loop else self._conversation.send_message ) - await loop.run_in_executor(None, send_message, message) + await self.run_state_op(send_message, message) if run: if self._explicit_interrupt_generation != explicit_interrupt_generation: return @@ -1071,10 +1079,7 @@ def _mark_running_acp_prompt_superseded_sync(self) -> tuple[bool, bool]: return (True, False) async def _mark_running_acp_prompt_superseded(self) -> tuple[bool, bool]: - loop = asyncio.get_running_loop() - return await loop.run_in_executor( - None, self._mark_running_acp_prompt_superseded_sync - ) + return await self.run_state_op(self._mark_running_acp_prompt_superseded_sync) async def subscribe_to_events(self, subscriber: Subscriber[Event]) -> UUID: subscriber_id = self._pub_sub.subscribe(subscriber) @@ -1084,9 +1089,8 @@ async def subscribe_to_events(self, subscriber: Subscriber[Event]) -> UUID: # conversation's synchronous FIFOLock cannot block the server event loop. log_length: int | None = None if self._conversation: - loop = asyncio.get_running_loop() - state_update_event, log_length = await loop.run_in_executor( - None, self._create_initial_state_sync + state_update_event, log_length = await self.run_state_op( + self._create_initial_state_sync ) else: state_update_event = ConversationStateUpdateEvent( @@ -1161,7 +1165,7 @@ def locked_on_event(): # needs a done-callback: otherwise it is collected with an unread # exception, so a dropped stats or LLM-log event leaves no signal # beyond a GC-time warning. - future = main_loop.run_in_executor(None, locked_on_event) + future = main_loop.run_in_executor(self._state_executor, locked_on_event) # Attach from the loop thread. This helper is called from foreign # threads, and `add_done_callback` on an already-done future uses # plain `call_soon`, which neither is thread-safe nor wakes an idle @@ -1318,6 +1322,11 @@ def _prepare_workspace_dir(working_dir: Path) -> None: async def start(self): # Store the main event loop for cross-thread communication self._main_loop: asyncio.AbstractEventLoop = asyncio.get_running_loop() + # Created here, not lazily: _emit_event_from_thread submits to it from + # foreign threads. + self._state_executor = ThreadPoolExecutor( + max_workers=4, thread_name_prefix="conversation-state" + ) # self.stored contains an Agent configuration we can instantiate self.conversation_dir.mkdir(parents=True, exist_ok=True) @@ -1429,7 +1438,11 @@ def _token_streaming_callback(chunk: LLMStreamChunk | str) -> None: reasoning_content=reasoning if isinstance(reasoning, str) else None, ) - conversation = LocalConversation( + # Off-loop: resuming replays every persisted event on the active branch + # (read + validate), which scales with conversation length and would + # otherwise freeze the whole server while it runs. + conversation = await asyncio.to_thread( + LocalConversation, agent=agent, workspace=workspace, plugins=self.stored.plugins, @@ -1553,9 +1566,7 @@ def _token_streaming_callback(chunk: LLMStreamChunk | str) -> None: # still active was left by a shutdown or crash mid-loop (close() skips # the status on purpose). Record it as interrupted so it reads as # stopped and stays resumable. - await asyncio.get_running_loop().run_in_executor( - None, self._reconcile_stale_goal_status - ) + await self.run_state_op(self._reconcile_stale_goal_status) # Publish initial state update await self._publish_state_update() @@ -1768,10 +1779,8 @@ async def _run_and_publish(): # ConversationRunError means run()/arun() already emitted its # own event, so skip it there to avoid duplicating the error. if not isinstance(exc, ConversationRunError): - await loop.run_in_executor( - None, self._publish_error_event_sync, exc - ) - await loop.run_in_executor(None, self._mark_error_status_sync) + await self.run_state_op(self._publish_error_event_sync, exc) + await self.run_state_op(self._mark_error_status_sync) finally: settle() # Wait for all pending events to be published via @@ -1934,7 +1943,7 @@ def _require_answered() -> None: if find_pending_ask(conversation._state.active_branch()) is not None: raise ValueError("pending_ask") - await asyncio.get_running_loop().run_in_executor(None, _require_answered) + await self.run_state_op(_require_answered) if self._closing: raise ServiceInactive() # Reserve capacity before spawning the loop so a full server refuses @@ -2162,8 +2171,7 @@ async def resume_goal_loop( """ if not self._conversation or self._closing: raise ServiceInactive() - loop = asyncio.get_running_loop() - last = await loop.run_in_executor(None, self._last_goal_loop_status) + last = await self.run_state_op(self._last_goal_loop_status) if last is None or last.get("status") in ("complete", "capped"): raise ValueError("no_resumable_goal") if judge_llm is None: @@ -2228,10 +2236,8 @@ async def react( if not self._conversation: raise ServiceInactive() conversation = self._conversation - loop = asyncio.get_running_loop() - return await loop.run_in_executor( - None, - partial(conversation.react, emoji, target_event_id, removed=removed), + return await self.run_state_op( + partial(conversation.react, emoji, target_event_id, removed=removed) ) async def answer_ask( @@ -2247,10 +2253,8 @@ async def answer_ask( """ if not self._conversation: raise ServiceInactive() - loop = asyncio.get_running_loop() - answer, message = await loop.run_in_executor( - None, - partial(self._conversation.answer_ask, ask_id, option_index, text), + answer, message = await self.run_state_op( + self._conversation.answer_ask, ask_id, option_index, text ) await self.send_message(message, run=True) return answer @@ -2259,18 +2263,14 @@ async def reject_pending_actions(self, reason: str): """Reject all pending actions and publish updated state.""" if not self._conversation: raise ServiceInactive() - loop = asyncio.get_running_loop() - await loop.run_in_executor( - None, self._conversation.reject_pending_actions, reason - ) + await self.run_state_op(self._conversation.reject_pending_actions, reason) async def pause(self): if self._conversation: self._explicit_interrupt_generation += 1 self._rerun_requested = False self._acp_internal_rerun_requested = False - loop = asyncio.get_running_loop() - await loop.run_in_executor(None, self._conversation.pause) + await self.run_state_op(self._conversation.pause) # Publish state update after pause to ensure stats are updated await self._publish_state_update() @@ -2306,8 +2306,7 @@ async def add_mcp_servers(self, servers: dict[str, MCPServer]) -> None: """Add modern MCP servers to the conversation.""" if not self._conversation: raise ServiceInactive() - loop = asyncio.get_running_loop() - await loop.run_in_executor(None, self._conversation.add_mcp_servers, servers) + await self.run_state_op(self._conversation.add_mcp_servers, servers) await self._publish_state_update() async def update_secrets(self, secrets: dict[str, SecretValue]): @@ -2324,17 +2323,13 @@ async def update_secrets(self, secrets: dict[str, SecretValue]): if CODEX_AUTH_SECRET_NAME in self.credential_bindings: secrets = dict(secrets) secrets.pop(CODEX_AUTH_SECRET_NAME, None) - loop = asyncio.get_running_loop() - await loop.run_in_executor(None, self._conversation.update_secrets, secrets) + await self.run_state_op(self._conversation.update_secrets, secrets) async def set_confirmation_policy(self, policy: ConfirmationPolicyBase): """Set the confirmation policy for the conversation.""" if not self._conversation: raise ServiceInactive() - loop = asyncio.get_running_loop() - await loop.run_in_executor( - None, self._conversation.set_confirmation_policy, policy - ) + await self.run_state_op(self._conversation.set_confirmation_policy, policy) async def set_max_budget_per_run(self, max_budget_per_run: float | None) -> None: """Set the conversation's cost ceiling, taking effect from its next step. @@ -2346,9 +2341,8 @@ async def set_max_budget_per_run(self, max_budget_per_run: float | None) -> None conversation = self._conversation if conversation is None: raise ServiceInactive() - loop = asyncio.get_running_loop() - await loop.run_in_executor( - None, setattr, conversation, "max_budget_per_run", max_budget_per_run + await self.run_state_op( + setattr, conversation, "max_budget_per_run", max_budget_per_run ) async def set_security_analyzer( @@ -2357,17 +2351,15 @@ async def set_security_analyzer( """Set the security analyzer for the conversation.""" if not self._conversation: raise ServiceInactive() - loop = asyncio.get_running_loop() - await loop.run_in_executor( - None, self._conversation.set_security_analyzer, security_analyzer + await self.run_state_op( + self._conversation.set_security_analyzer, security_analyzer ) async def load_plugin(self, plugin_ref: str) -> None: """Load a marketplace plugin into the active conversation.""" if self._conversation is None: raise ServiceInactive() - loop = asyncio.get_running_loop() - await loop.run_in_executor(None, self._conversation.load_plugin, plugin_ref) + await self.run_state_op(self._conversation.load_plugin, plugin_ref) async def switch_acp_model(self, model: str) -> None: """Switch the model on an ACP conversation. @@ -2388,8 +2380,7 @@ async def switch_acp_model(self, model: str) -> None: # a created-but-not-yet-run conversation, so a pre-first-run switch # is a normal 200 deferral, not an error. raise ServiceInactive() - loop = asyncio.get_running_loop() - await loop.run_in_executor(None, self._conversation.switch_acp_model, model) + await self.run_state_op(self._conversation.switch_acp_model, model) async def switch_acp_effort(self, effort: str) -> None: """Switch the reasoning effort on an ACP conversation. @@ -2399,8 +2390,7 @@ async def switch_acp_effort(self, effort: str) -> None: """ if self._conversation is None: raise ServiceInactive() - loop = asyncio.get_running_loop() - await loop.run_in_executor(None, self._conversation.switch_acp_effort, effort) + await self.run_state_op(self._conversation.switch_acp_effort, effort) async def close_connections(self, code: int, reason: str) -> None: """Close the client sockets subscribed to this conversation.""" @@ -2438,9 +2428,8 @@ async def close(self): run_task = self._run_task if run_task is not None and not run_task.done(): if self._conversation is not None: - loop = asyncio.get_running_loop() try: - await loop.run_in_executor(None, self._conversation.pause) + await self.run_state_op(self._conversation.pause) except Exception: logger.warning( "Failed to pause conversation during close", exc_info=True @@ -2465,9 +2454,11 @@ async def close(self): await self._pub_sub.close() await self._stream_pub_sub.close() if self._conversation: - loop = asyncio.get_running_loop() - await loop.run_in_executor(None, self._conversation.close) + await self.run_state_op(self._conversation.close) self._conversation = None + if self._state_executor is not None: + self._state_executor.shutdown(wait=False) + self._state_executor = None self.credential_bindings = {} if self._lease is not None and self._lease_generation is not None: @@ -2509,8 +2500,7 @@ async def ask_agent(self, question: str) -> str: if not self._conversation: raise ServiceInactive() - loop = asyncio.get_running_loop() - return await loop.run_in_executor(None, self._conversation.ask_agent, question) + return await self.run_state_op(self._conversation.ask_agent, question) async def condense(self) -> None: """Force condensation of the conversation history. @@ -2534,8 +2524,7 @@ async def condense(self) -> None: f"{type(condenser).__name__} does not handle condensation requests" ) - loop = asyncio.get_running_loop() - return await loop.run_in_executor(None, self._conversation.condense) + return await self.run_state_op(self._conversation.condense) async def navigate_to(self, event_id: str | None) -> None: """Move the conversation HEAD to an existing event (in-place re-root). @@ -2548,10 +2537,7 @@ async def navigate_to(self, event_id: str | None) -> None: if not self._conversation: raise ServiceInactive() - loop = asyncio.get_running_loop() - return await loop.run_in_executor( - None, self._conversation.navigate_to, event_id - ) + return await self.run_state_op(self._conversation.navigate_to, event_id) def _get_agent_final_response_sync(self) -> str: """Extract the agent's final response from the active branch. diff --git a/openhands-sdk/openhands/sdk/agent/parallel_executor.py b/openhands-sdk/openhands/sdk/agent/parallel_executor.py index c32e4714156..cb510a1d76c 100644 --- a/openhands-sdk/openhands/sdk/agent/parallel_executor.py +++ b/openhands-sdk/openhands/sdk/agent/parallel_executor.py @@ -159,16 +159,27 @@ def _resolve(ae: ActionEvent) -> ToolDefinition | None: return tools.get(ae.tool_name) if tools else None if len(action_events) == 1 or self._max_workers == 1: - return [ - await self._arun_safe( - action, - tool_runner, - _resolve(action), - cancel_token, - span_owner=span_owner, - ) - for action in action_events - ] + # Not the shared default executor: arun() holds the conversation + # state lock across this await, and callers waiting on that lock + # (state snapshots, send_message) park default-executor threads. + # Once they fill it, the tool never starts and the lock is never + # released. Shut down without waiting so an interrupt that cancels + # this await does not block the event loop on the tool thread. + pool = ThreadPoolExecutor(max_workers=1, thread_name_prefix="aexecute_seq") + try: + return [ + await self._arun_safe( + action, + tool_runner, + _resolve(action), + cancel_token, + pool, + span_owner, + ) + for action in action_events + ] + finally: + pool.shutdown(wait=False) with ThreadPoolExecutor( max_workers=self._max_workers, diff --git a/tests/agent_server/test_conversation_service.py b/tests/agent_server/test_conversation_service.py index 38daad6532a..d8f0aae66f0 100644 --- a/tests/agent_server/test_conversation_service.py +++ b/tests/agent_server/test_conversation_service.py @@ -4,7 +4,9 @@ import tempfile import threading import time +from concurrent.futures import ThreadPoolExecutor from datetime import UTC, datetime +from functools import partial from pathlib import Path from typing import Any, cast from unittest.mock import AsyncMock, MagicMock, patch @@ -2972,6 +2974,7 @@ async def test_update_conversation_tags_uses_state_lock( ): """Test that tag updates hold the ConversationState lock.""" mock_service = AsyncMock(spec=EventService) + mock_service.run_state_op.side_effect = asyncio.to_thread mock_service.stored = sample_stored_conversation _adopt_saved_meta(mock_service) mock_state = ConversationState( @@ -3007,6 +3010,7 @@ async def test_update_conversation_tags_wait_does_not_block_event_loop( ): """Waiting on the state lock must not stall unrelated async work.""" mock_service = AsyncMock(spec=EventService) + mock_service.run_state_op.side_effect = asyncio.to_thread mock_service.stored = sample_stored_conversation _adopt_saved_meta(mock_service) state = ConversationState( @@ -3068,6 +3072,63 @@ async def release_after_delay() -> None: "state lock" ) + @pytest.mark.asyncio + async def test_update_conversation_tags_wait_leaves_default_executor_free( + self, conversation_service, sample_stored_conversation + ): + """A run holds the state lock across whole tool steps. A tag PATCH + waiting on it must not park a default-executor thread: event reads of + every conversation need that pool.""" + loop = asyncio.get_running_loop() + loop.set_default_executor(ThreadPoolExecutor(max_workers=1)) + state_executor = ThreadPoolExecutor(max_workers=1) + mock_service = AsyncMock(spec=EventService) + mock_service.run_state_op.side_effect = partial( + loop.run_in_executor, state_executor + ) + mock_service.stored = sample_stored_conversation + _adopt_saved_meta(mock_service) + state = ConversationState( + id=sample_stored_conversation.id, + agent=_sample_agent(), + workspace=sample_stored_conversation.workspace, + execution_status=ConversationExecutionStatus.RUNNING, + confirmation_policy=sample_stored_conversation.confirmation_policy, + ) + mock_service.get_state.return_value = state + conversation_id = sample_stored_conversation.id + conversation_service._event_services[conversation_id] = mock_service + held, release = threading.Event(), threading.Event() + + def hold_state_lock() -> None: + with state: + held.set() + release.wait() + + holder = threading.Thread(target=hold_state_lock) + holder.start() + assert held.wait(timeout=1.0) + with patch.object( + conversation_service, "_notify_conversation_webhooks", new=AsyncMock() + ): + update = asyncio.create_task( + conversation_service.update_conversation( + conversation_id, UpdateConversationRequest(tags={"env": "prod"}) + ) + ) + try: + await asyncio.sleep(0.05) + free = await asyncio.wait_for( + loop.run_in_executor(None, lambda: "free"), timeout=2 + ) + assert free == "free" + finally: + release.set() + holder.join() + assert await asyncio.wait_for(update, timeout=2) + state_executor.shutdown() + assert state.tags == {"env": "prod"} + @pytest.mark.asyncio async def test_update_conversation_not_found(self, conversation_service): """Test updating a non-existent conversation returns False.""" @@ -3095,6 +3156,7 @@ async def test_update_conversation_notifies_webhooks( """Test that updating a conversation triggers webhook notifications.""" # Create mock event service mock_service = AsyncMock(spec=EventService) + mock_service.run_state_op.side_effect = asyncio.to_thread mock_service.stored = sample_stored_conversation _adopt_saved_meta(mock_service) mock_state = ConversationState( @@ -3143,6 +3205,7 @@ async def test_update_acp_conversation_notifies_webhooks_with_acp_shape( updated_at=datetime(2025, 1, 1, 12, 30, 0, tzinfo=UTC), ) mock_service = AsyncMock(spec=EventService) + mock_service.run_state_op.side_effect = asyncio.to_thread mock_service.stored = stored_conversation _adopt_saved_meta(mock_service) mock_state = ConversationState( diff --git a/tests/agent_server/test_event_service.py b/tests/agent_server/test_event_service.py index b01fab373b0..22d306f6c37 100644 --- a/tests/agent_server/test_event_service.py +++ b/tests/agent_server/test_event_service.py @@ -7,6 +7,7 @@ import threading import time import zipfile +from concurrent.futures import ThreadPoolExecutor from contextlib import suppress from datetime import UTC, datetime from pathlib import Path @@ -2812,6 +2813,53 @@ def mock_conversation_with_real_lock(self): conversation._state = state return conversation + @pytest.mark.asyncio + @pytest.mark.parametrize( + "wait_on_state_lock", + [ + pytest.param( + lambda service: service._get_execution_status(), id="execution-status" + ), + pytest.param( + lambda service: service.subscribe_to_events(AsyncMock()), + id="websocket-subscribe", + ), + ], + ) + async def test_state_lock_waiters_leave_default_executor_free( + self, event_service, mock_conversation_with_real_lock, wait_on_state_lock + ): + """A run holds the state lock across whole tool steps. Requests waiting + on it must not park default-executor threads: lock-free work (event + reads of every conversation, tool calls) needs that pool.""" + loop = asyncio.get_running_loop() + loop.set_default_executor(ThreadPoolExecutor(max_workers=1)) + event_service._conversation = mock_conversation_with_real_lock + event_service._state_executor = ThreadPoolExecutor(max_workers=2) + lock = mock_conversation_with_real_lock._state._lock + held, release = threading.Event(), threading.Event() + + def hold_lock(): + with lock: + held.set() + release.wait() + + holder = threading.Thread(target=hold_lock) + holder.start() + held.wait() + waiter = asyncio.create_task(wait_on_state_lock(event_service)) + try: + await asyncio.sleep(0.05) + free = await asyncio.wait_for( + loop.run_in_executor(None, lambda: "free"), timeout=2 + ) + assert free == "free" + finally: + release.set() + holder.join() + await asyncio.wait_for(waiter, timeout=2) + event_service._state_executor.shutdown() + @pytest.mark.asyncio async def test_initial_state_leaves_out_skill_bodies( self, event_service, mock_conversation_with_real_lock diff --git a/tests/sdk/agent/test_parallel_executor.py b/tests/sdk/agent/test_parallel_executor.py index 8436a04c5c8..511ed57001d 100644 --- a/tests/sdk/agent/test_parallel_executor.py +++ b/tests/sdk/agent/test_parallel_executor.py @@ -1,7 +1,9 @@ """Tests for ParallelToolExecutor.""" +import asyncio import threading import time +from concurrent.futures import ThreadPoolExecutor from typing import Any from unittest.mock import MagicMock @@ -238,3 +240,24 @@ def outer_tool_runner(action: Any) -> list: ) assert results == [["inner-a", "inner-b"]] + + +async def test_async_sequential_batch_runs_without_the_default_executor(): + """arun() holds the state lock across this await; callers waiting on that + lock occupy default-executor threads. A tool queued behind them would + never start, so the sequential path must not depend on that pool.""" + loop = asyncio.get_running_loop() + loop.set_default_executor(ThreadPoolExecutor(max_workers=1)) + release = threading.Event() + blocker = loop.run_in_executor(None, release.wait) + action: Any = MagicMock() + event = MagicMock() + try: + results = await asyncio.wait_for( + ParallelToolExecutor().aexecute_batch([action], lambda a: [event]), + timeout=5, + ) + finally: + release.set() + await blocker + assert results == [[event]]