Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 20 additions & 1 deletion bench/swebench/harness/arms.py
Original file line number Diff line number Diff line change
Expand Up @@ -261,6 +261,22 @@ def arm_openflow(inst, repo, venv, events, tid):
"""


def shim_timeout_s(budget_min: int) -> int:
"""Seconds run_pi may spend inside one openswarm attempt.

Must stay strictly below the openswarm budget (budget_min * 60s):
openswarm kills the backend shim when the budget expires, so a
SHIM_TIMEOUT of budget or more means the shim dies inside run_pi and
never writes executor-*-record.json / executor-*-events.jsonl
(M16 slice 2, #213: 840s budget vs 870s SHIM_TIMEOUT lost the whole
executor events file for sympy__sympy-13878 r1). The 60s margin covers
run_pi's own timeout path (SIGTERM + up to 10s grace, then the record
print) plus the shim copying and catting the events file. The 30s floor
keeps tiny budgets usable instead of collapsing to 0.
"""
return max(30, budget_min * 60 - 60)


def arm_openswarm(inst, repo, venv, events, tid):
rundir = repo.parent
pdir = rundir / "planner"
Expand Down Expand Up @@ -298,8 +314,11 @@ def arm_openswarm(inst, repo, venv, events, tid):
home = events / "openswarm-home"
home.mkdir(exist_ok=True)
shim = Path(__file__).resolve().parent / "pi_shim.sh"
# SHIM_TIMEOUT must stay below the openswarm budget (see shim_timeout_s):
# a timeout of budget or more lets openswarm kill the shim before run_pi
# returns, losing the executor events file (#213).
env = dict(os.environ, **path_env(venv), OPENSWARM_HOME=str(home), OPENSWARM_PI_BIN=str(shim),
SHIM_TASK_ID=tid + "-exec", SHIM_EVENTS=str(events), SHIM_TIMEOUT=str(budget_min * 60 + 30))
SHIM_TASK_ID=tid + "-exec", SHIM_EVENTS=str(events), SHIM_TIMEOUT=str(shim_timeout_s(budget_min)))
# Keep DEEPSEEK_API_KEY: openswarm is forced to --backend pi, so the Pi wrapper run_pi.sh it calls through the shim needs it and cannot route to its own deepseek backend.
for k in ("OPENAI_API_KEY", "ANTHROPIC_API_KEY", "GOOGLE_API_KEY", "GEMINI_API_KEY"):
env.pop(k, None)
Expand Down
73 changes: 73 additions & 0 deletions bench/swebench/test_shim_timeout.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
#!/usr/bin/env python3
"""Tests for the openswarm SHIM_TIMEOUT in harness/arms.py.

Regression test for evalid #213 (M16 slice 2): SHIM_TIMEOUT was
``budget_min * 60 + 30`` while openswarm's own budget is ``budget_min * 60``
seconds, so whenever the executor ran the budget out, openswarm killed the
backend shim (attempt rc=124) before run_pi returned and the shim never
wrote executor-*-record.json / executor-*-events.jsonl. The 840s-budget /
870s-timeout validation cell (sympy__sympy-13878 r1, run
20261117-013602-da5999) lost 843s of executor work: no events file, executor
usage all 0, solution.diff 0 bytes.

Stdlib only, like test_planner_prompt.py: arms.py itself imports swe_env,
which pulls in pandas / swebench, so this test reads shim_timeout_s out of
the source with ast instead of importing the module. That keeps it runnable
by the CI node suite, which shells out here with plain python3.
"""
import ast
import unittest
from pathlib import Path

ARMS = Path(__file__).resolve().parent / "harness" / "arms.py"


def _shim_timeout_s():
tree = ast.parse(ARMS.read_text())
for node in ast.walk(tree):
if isinstance(node, ast.FunctionDef) and node.name == "shim_timeout_s":
src = ast.get_source_segment(ARMS.read_text(), node)
ns = {}
exec(src, ns) # noqa: S102 - test-only, source is this repo's file
return ns["shim_timeout_s"]
raise AssertionError("shim_timeout_s not found in harness/arms.py")


def _arm_openswarm_source() -> str:
tree = ast.parse(ARMS.read_text())
for node in ast.walk(tree):
if isinstance(node, ast.FunctionDef) and node.name == "arm_openswarm":
return ast.get_source_segment(ARMS.read_text(), node)
raise AssertionError("arm_openswarm not found in harness/arms.py")


class TestShimTimeout(unittest.TestCase):
def test_below_openswarm_budget_for_all_budgets(self):
f = _shim_timeout_s()
for budget_min in range(1, 61):
self.assertLess(
f(budget_min), budget_min * 60,
f"SHIM_TIMEOUT must stay below the openswarm budget "
f"({budget_min} min)",
)

def test_validation_cell_budget(self):
# The #213 validation cell: planner 286.2s -> budget_min=14
# (840s budget); the old 870s timeout lost the events file.
f = _shim_timeout_s()
self.assertEqual(f(14), 780)
self.assertLessEqual(f(14), 800)

def test_small_budget_floor(self):
f = _shim_timeout_s()
self.assertEqual(f(1), 30)
self.assertGreater(f(1), 0)

def test_call_site_uses_helper(self):
src = _arm_openswarm_source()
self.assertIn("SHIM_TIMEOUT=str(shim_timeout_s(budget_min))", src)
self.assertNotIn("budget_min * 60 + 30", ARMS.read_text())


if __name__ == "__main__":
unittest.main()