From 335003e7cbbe5e9596d9c7004a2ac0375043ddad Mon Sep 17 00:00:00 2001 From: Sebastien Tardif Date: Wed, 2 Sep 2026 10:01:19 -0700 Subject: [PATCH] fix: time out unbounded lane prepare hook Lane prepare used subprocess.run with no timeout. A stuck CLAWBENCH_LANE_PREPARE_CMD blocked lane startup and the EvalWorker. Bound the hook to CLAWBENCH_LANE_PREPARE_TIMEOUT_SECONDS (default 180, matching gateway health) and fail the lane on expiry. Signed-off-by: Sebastien Tardif --- clawbench/worker.py | 8 ++++++- tests/test_worker.py | 57 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/clawbench/worker.py b/clawbench/worker.py index 295c9e2..a0dcfd9 100644 --- a/clawbench/worker.py +++ b/clawbench/worker.py @@ -629,7 +629,13 @@ def _run_lane_prepare_hook(self, lane: ParallelLane) -> None: "CLAWBENCH_LANE_PORT": str(lane.port), } logger.info("Running lane %d prepare hook", lane.index + 1) - subprocess.run([hook], env=hook_env, check=True) + timeout_seconds = int(os.environ.get("CLAWBENCH_LANE_PREPARE_TIMEOUT_SECONDS", "180")) + try: + subprocess.run([hook], env=hook_env, check=True, timeout=timeout_seconds) + except subprocess.TimeoutExpired as exc: + raise RuntimeError( + f"Lane {lane.index + 1} prepare hook timed out after {timeout_seconds}s" + ) from exc def _seed_lane_state_dir(self, target_state_dir: Path) -> None: source_state_dir = Path(os.environ.get("OPENCLAW_STATE_DIR", os.path.expanduser("~/.openclaw"))) diff --git a/tests/test_worker.py b/tests/test_worker.py index 523742f..0b83a2e 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -1,5 +1,9 @@ import asyncio import json +import os +import signal +import subprocess +import threading from pathlib import Path from types import SimpleNamespace @@ -452,3 +456,56 @@ async def fake_update_progress(job_id: str, **kwargs) -> None: }, ) ] + + +def test_run_lane_prepare_hook_kills_hung_hook(tmp_path: Path, monkeypatch): + hook = tmp_path / "hung-hook" + hook.write_text( + '#!/bin/sh\necho $$ > "$HOME/hook.pid"\nexec sleep 1000\n', + encoding="utf-8", + ) + hook.chmod(0o755) + + state_dir = tmp_path / "lane" / "state" + state_dir.mkdir(parents=True) + (state_dir.parent / "home").mkdir(parents=True) + pid_path = state_dir.parent / "home" / "hook.pid" + + monkeypatch.setenv("CLAWBENCH_LANE_PREPARE_CMD", str(hook)) + monkeypatch.setenv("CLAWBENCH_LANE_PREPARE_TIMEOUT_SECONDS", "1") + + worker = EvalWorker(JobQueue()) + lane = ParallelLane(index=0, tasks=[DummyTask("t1", "tier1", "coding")]) + lane.state_dir = state_dir + lane.port = GATEWAY_PORT + + outcome: list[BaseException | None] = [] + + def run() -> None: + try: + worker._run_lane_prepare_hook(lane) + outcome.append(None) + except BaseException as exc: + outcome.append(exc) + + thread = threading.Thread(target=run, daemon=True) + try: + thread.start() + thread.join(5) + assert not thread.is_alive(), "prepare hook hung without timeout" + assert outcome + exc = outcome[0] + assert exc is not None + assert isinstance(exc, (subprocess.TimeoutExpired, RuntimeError)) + if isinstance(exc, RuntimeError): + assert "timed out" in str(exc).lower() + assert pid_path.is_file() + pid = int(pid_path.read_text(encoding="utf-8").strip()) + with pytest.raises(ProcessLookupError): + os.kill(pid, 0) + finally: + if pid_path.is_file(): + try: + os.kill(int(pid_path.read_text(encoding="utf-8").strip()), signal.SIGKILL) + except (ValueError, ProcessLookupError, OSError): + pass