Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions .pr/agent-server-perf/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
# Agent-server lock-starvation / cold-load repro

Runs a real `python -m openhands.agent_server` plus a mock OpenAI-compatible LLM
(`mock_llm.py`), driven over HTTP/WebSocket. `SDK_REPO=<checkout>` picks the tree whose
`.venv` runs the server (default: this repo).

```bash
# deadlock: cpu_count=2 (6-thread default executor), 5 WS clients, 3 s tools, 2 tools/step
.venv/bin/python .pr/agent-server-perf/bench_lock_starvation.py 2 5 3 label 2

# cold load: build a 300-step fixture, then measure /alive during the load (3 reps)
.venv/bin/python .pr/agent-server-perf/bench_cold_load.py build 300 500
.venv/bin/python .pr/agent-server-perf/bench_cold_load.py measure 300 500 3 label
```

`results_*.jsonl` are the runs quoted in the PR (main = 8dc712f, fix12 = this branch).
`results_starvation_patch.jsonl` sends 6 tag PATCHes instead of WebSocket connects
(`... bench_lock_starvation.py 2 6 3 label 2 patch`); `patch-main-cs` is this branch
with main's `conversation_service.py`.
112 changes: 112 additions & 0 deletions .pr/agent-server-perf/bench_cold_load.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
"""Scenario B: cold-loading a long conversation stalls the whole event loop.

build: bench_cold_load.py build <steps> <bytes> -> fixtures/conv-<steps>-<bytes>/
measure: bench_cold_load.py measure <steps> <bytes> <reps> [label]

measure starts a fresh server on a copy of the fixture, probes /alive every
5 ms, then issues one events/search request that forces EventService.start()
(LocalConversation construction -> full event replay). Reports load latency
and the worst /alive latency seen during the load (= loop stall).
"""

import asyncio
import json
import shutil
import statistics
import sys
import tempfile
import time
from pathlib import Path

import httpx

from harness import HERE, Stack, log

FIX = HERE / "fixtures"


async def build(steps: int, nbytes: int):
async with Stack() as st:
cid = await st.create(f"LOOP {steps} BYTES {nbytes}")
t0 = time.monotonic()
s = await st.wait_status(cid, {"finished", "error"}, 3600)
log("built", cid, s, round(time.monotonic() - t0, 1), "s")
r = await st.client.get(f"/api/conversations/{cid}/events/count")
n_events = r.json()
dst = FIX / f"conv-{steps}-{nbytes}"
if dst.exists():
shutil.rmtree(dst)
shutil.copytree(st.tmp / "work", dst, symlinks=True)
(dst / "meta.bench.json").write_text(json.dumps({"id": cid, "events": n_events}))
print(json.dumps({"built": steps, "events": n_events, "id": cid}))


async def measure_once(steps: int, nbytes: int) -> dict:
src = FIX / f"conv-{steps}-{nbytes}"
meta = json.loads((src / "meta.bench.json").read_text())
tmp = Path(tempfile.mkdtemp(prefix="ohbench-cold-"))
shutil.copytree(src, tmp / "work", symlinks=True)
async with Stack(tmp=tmp) as st:
await asyncio.sleep(1.0)
samples: list[float] = []
stop = asyncio.Event()

async def probe():
async with httpx.AsyncClient(base_url=st.base) as c:
while not stop.is_set():
t0 = time.perf_counter()
await c.get("/alive", timeout=60)
samples.append(time.perf_counter() - t0)
await asyncio.sleep(0.005)

pt = asyncio.create_task(probe())
await asyncio.sleep(0.3)
base_n = len(samples)
t0 = time.perf_counter()
r = await st.client.get(
f"/api/conversations/{meta['id']}/events/search?limit=1", timeout=300
)
load = time.perf_counter() - t0
r.raise_for_status()
await asyncio.sleep(0.2)
stop.set()
await pt
during = samples[base_n:]
# warm request after load, for contrast
t0 = time.perf_counter()
await st.client.get(f"/api/conversations/{meta['id']}/events/search?limit=1")
warm = time.perf_counter() - t0
shutil.rmtree(tmp, ignore_errors=True)
return {
"events": meta["events"],
"cold_load_s": round(load, 3),
"warm_search_ms": round(warm * 1000, 1),
"alive_max_ms_during_load": round(max(during) * 1000, 1),
"alive_p50_ms_during_load": round(statistics.median(during) * 1000, 1),
}


async def main():
mode = sys.argv[1]
steps = int(sys.argv[2])
if mode == "build":
await build(steps, int(sys.argv[3]))
return
nbytes = int(sys.argv[3])
reps = int(sys.argv[4])
label = sys.argv[5] if len(sys.argv) > 5 else "run"
runs = [await measure_once(steps, nbytes) for _ in range(reps)]
out = {
"label": label,
"steps": steps,
"events": runs[0]["events"],
"cold_load_s_median": statistics.median(r["cold_load_s"] for r in runs),
"alive_max_ms_median": statistics.median(r["alive_max_ms_during_load"] for r in runs),
"warm_search_ms_median": statistics.median(r["warm_search_ms"] for r in runs),
"runs": runs,
}
print(json.dumps(out))


if __name__ == "__main__":
asyncio.run(main())
156 changes: 156 additions & 0 deletions .pr/agent-server-perf/bench_lock_starvation.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,156 @@
"""Scenario A: FIFOLock waiters vs. the shared default executor.

Conversation A runs a step with 2 sequential `terminal` tool calls (`sleep S`).
While the first tool runs (A's arun holds the state FIFOLock on the loop
thread), K WebSocket clients connect to A. Each connect snapshots state in the
default executor and blocks on the FIFOLock. Meanwhile a probe hits a *different*
idle conversation B's event search (lock-free, but also uses the default
executor) and /alive (pure loop).

Reports: did A finish, how long, probe latencies on B and /alive.

usage: bench_lock_starvation.py <cpu_count> <K> [sleep_s] [label] [ntools] [ws|patch]

`patch` sends K tag PATCHes to A instead of WebSocket connects.
"""

import asyncio
import json
import statistics
import sys
import time

import httpx
import websockets

from harness import Stack, log


async def probe(stack: Stack, path: str, stop: asyncio.Event, out: list, tag: str):
async with httpx.AsyncClient(base_url=stack.base) as c:
while not stop.is_set():
t0 = time.perf_counter()
try:
r = await c.get(path, timeout=5)
ok = r.status_code == 200
except Exception:
ok = False
dt = time.perf_counter() - t0
out.append((time.monotonic(), dt, ok))
await asyncio.sleep(0.05)


async def ws_client(stack: Stack, cid: str, opened: list, stop: asyncio.Event):
t0 = time.perf_counter()
try:
async with websockets.connect(
f"{stack.ws_base}/sockets/events/{cid}", open_timeout=60
) as ws:
# First frame = initial state snapshot (needs the FIFOLock).
await asyncio.wait_for(ws.recv(), timeout=60)
opened.append(time.perf_counter() - t0)
while not stop.is_set():
with contextlib_suppress():
await asyncio.wait_for(ws.recv(), timeout=0.5)
except Exception as e: # noqa: BLE001
opened.append(("err", type(e).__name__))


async def patch_client(stack: Stack, cid: str, opened: list, stop: asyncio.Event):
# A tag PATCH updates state and composes the webhook snapshot under the lock.
t0 = time.perf_counter()
try:
async with httpx.AsyncClient(base_url=stack.base) as c:
r = await c.patch(
f"/api/conversations/{cid}", json={"tags": {"bench": "x"}}, timeout=60
)
r.raise_for_status()
opened.append(time.perf_counter() - t0)
except Exception as e: # noqa: BLE001
opened.append(("err", type(e).__name__))


class contextlib_suppress:
def __enter__(self):
return self

def __exit__(self, *exc):
return exc[0] is not None and issubclass(exc[0], (asyncio.TimeoutError, TimeoutError))


def summarize(samples, t_from, t_to):
w = [dt for (t, dt, ok) in samples if t_from <= t <= t_to]
fails = sum(1 for (t, dt, ok) in samples if t_from <= t <= t_to and not ok)
if not w:
return {"n": 0}
return {
"n": len(w),
"p50_ms": round(statistics.median(w) * 1000, 1),
"max_ms": round(max(w) * 1000, 1),
"fails": fails,
}


async def main():
cpu = int(sys.argv[1])
K = int(sys.argv[2])
sleep_s = float(sys.argv[3]) if len(sys.argv) > 3 else 4.0
label = sys.argv[4] if len(sys.argv) > 4 else "run"
ntools = int(sys.argv[5]) if len(sys.argv) > 5 else 2
mode = sys.argv[6] if len(sys.argv) > 6 else "ws"
client = {"ws": ws_client, "patch": patch_client}[mode]
async with Stack(cpu_count=cpu) as st:
b = await st.create("SLEEP 0")
assert await st.wait_status(b, {"finished"}, 60), "B did not finish"
a = await st.create(f"SLEEP {sleep_s} x{ntools}")
assert await st.wait_status(a, {"running"}, 30), "A did not start"
await asyncio.sleep(1.0) # first tool is now running; A holds the lock

stop = asyncio.Event()
b_samples: list = []
alive_samples: list = []
probes = [
asyncio.create_task(probe(st, f"/api/conversations/{b}/events/search?limit=5", stop, b_samples, "B")),
asyncio.create_task(probe(st, "/alive", stop, alive_samples, "alive")),
]
t_start = time.monotonic()
opened: list = []
clients = [asyncio.create_task(client(st, a, opened, stop)) for _ in range(K)]

timeout = sleep_s * 2 + 25
done = await st.wait_status(a, {"finished", "error"}, timeout)
t_end = time.monotonic()
a_secs = round(t_end - t_start, 2)
recheck = None
if not done:
await asyncio.sleep(60)
recheck = await st.status(a)
# let ws clients get their snapshots if unblocked
await asyncio.sleep(1.0)
stop.set()
for t in clients + probes:
t.cancel()
await asyncio.gather(*clients, *probes, return_exceptions=True)

ok_open = [x for x in opened if isinstance(x, float)]
result = {
"label": label,
"cpu_count": cpu,
"default_executor_workers": min(32, cpu + 4),
"client_mode": mode,
"K_ws_clients": K,
"tool_sleep_s": sleep_s,
"A_final_status": done or "STUCK (timeout)",
"A_secs_from_storm": a_secs if done else f">{timeout}",
"A_status_60s_later": recheck,
"tools_in_step": ntools,
"ws_snapshots_received": len(ok_open),
"ws_snapshot_p50_s": round(statistics.median(ok_open), 2) if ok_open else None,
"probe_B_event_search": summarize(b_samples, t_start, t_end),
"probe_alive": summarize(alive_samples, t_start, t_end),
}
print(json.dumps(result))


if __name__ == "__main__":
asyncio.run(main())
Loading
Loading