Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion clawbench/worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -629,7 +629,13 @@ def _run_lane_prepare_hook(self, lane: ParallelLane) -> None:
"CLAWBENCH_LANE_PORT": str(lane.port),
}
logger.info("Running lane %d prepare hook", lane.index + 1)
subprocess.run([hook], env=hook_env, check=True)
timeout_seconds = int(os.environ.get("CLAWBENCH_LANE_PREPARE_TIMEOUT_SECONDS", "180"))
try:
subprocess.run([hook], env=hook_env, check=True, timeout=timeout_seconds)
except subprocess.TimeoutExpired as exc:
raise RuntimeError(
f"Lane {lane.index + 1} prepare hook timed out after {timeout_seconds}s"
) from exc

def _seed_lane_state_dir(self, target_state_dir: Path) -> None:
source_state_dir = Path(os.environ.get("OPENCLAW_STATE_DIR", os.path.expanduser("~/.openclaw")))
Expand Down
57 changes: 57 additions & 0 deletions tests/test_worker.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
import asyncio
import json
import os
import signal
import subprocess
import threading
from pathlib import Path
from types import SimpleNamespace

Expand Down Expand Up @@ -452,3 +456,56 @@ async def fake_update_progress(job_id: str, **kwargs) -> None:
},
)
]


def test_run_lane_prepare_hook_kills_hung_hook(tmp_path: Path, monkeypatch):
hook = tmp_path / "hung-hook"
hook.write_text(
'#!/bin/sh\necho $$ > "$HOME/hook.pid"\nexec sleep 1000\n',
encoding="utf-8",
)
hook.chmod(0o755)

state_dir = tmp_path / "lane" / "state"
state_dir.mkdir(parents=True)
(state_dir.parent / "home").mkdir(parents=True)
pid_path = state_dir.parent / "home" / "hook.pid"

monkeypatch.setenv("CLAWBENCH_LANE_PREPARE_CMD", str(hook))
monkeypatch.setenv("CLAWBENCH_LANE_PREPARE_TIMEOUT_SECONDS", "1")

worker = EvalWorker(JobQueue())
lane = ParallelLane(index=0, tasks=[DummyTask("t1", "tier1", "coding")])
lane.state_dir = state_dir
lane.port = GATEWAY_PORT

outcome: list[BaseException | None] = []

def run() -> None:
try:
worker._run_lane_prepare_hook(lane)
outcome.append(None)
except BaseException as exc:
outcome.append(exc)

thread = threading.Thread(target=run, daemon=True)
try:
thread.start()
thread.join(5)
assert not thread.is_alive(), "prepare hook hung without timeout"
assert outcome
exc = outcome[0]
assert exc is not None
assert isinstance(exc, (subprocess.TimeoutExpired, RuntimeError))
if isinstance(exc, RuntimeError):
assert "timed out" in str(exc).lower()
assert pid_path.is_file()
pid = int(pid_path.read_text(encoding="utf-8").strip())
with pytest.raises(ProcessLookupError):
os.kill(pid, 0)
finally:
if pid_path.is_file():
try:
os.kill(int(pid_path.read_text(encoding="utf-8").strip()), signal.SIGKILL)
except (ValueError, ProcessLookupError, OSError):
pass