diff --git a/bench/swebench/harness/arms.py b/bench/swebench/harness/arms.py index 40e3b93..cf05995 100644 --- a/bench/swebench/harness/arms.py +++ b/bench/swebench/harness/arms.py @@ -261,6 +261,22 @@ def arm_openflow(inst, repo, venv, events, tid): """ +def shim_timeout_s(budget_min: int) -> int: + """Seconds run_pi may spend inside one openswarm attempt. + + Must stay strictly below the openswarm budget (budget_min * 60s): + openswarm kills the backend shim when the budget expires, so a + SHIM_TIMEOUT of budget or more means the shim dies inside run_pi and + never writes executor-*-record.json / executor-*-events.jsonl + (M16 slice 2, #213: 840s budget vs 870s SHIM_TIMEOUT lost the whole + executor events file for sympy__sympy-13878 r1). The 60s margin covers + run_pi's own timeout path (SIGTERM + up to 10s grace, then the record + print) plus the shim copying and catting the events file. The 30s floor + keeps tiny budgets usable instead of collapsing to 0. + """ + return max(30, budget_min * 60 - 60) + + def arm_openswarm(inst, repo, venv, events, tid): rundir = repo.parent pdir = rundir / "planner" @@ -298,8 +314,11 @@ def arm_openswarm(inst, repo, venv, events, tid): home = events / "openswarm-home" home.mkdir(exist_ok=True) shim = Path(__file__).resolve().parent / "pi_shim.sh" + # SHIM_TIMEOUT must stay below the openswarm budget (see shim_timeout_s): + # a timeout of budget or more lets openswarm kill the shim before run_pi + # returns, losing the executor events file (#213). env = dict(os.environ, **path_env(venv), OPENSWARM_HOME=str(home), OPENSWARM_PI_BIN=str(shim), - SHIM_TASK_ID=tid + "-exec", SHIM_EVENTS=str(events), SHIM_TIMEOUT=str(budget_min * 60 + 30)) + SHIM_TASK_ID=tid + "-exec", SHIM_EVENTS=str(events), SHIM_TIMEOUT=str(shim_timeout_s(budget_min))) # Keep DEEPSEEK_API_KEY: openswarm is forced to --backend pi, so the Pi wrapper run_pi.sh it calls through the shim needs it and cannot route to its own deepseek backend. for k in ("OPENAI_API_KEY", "ANTHROPIC_API_KEY", "GOOGLE_API_KEY", "GEMINI_API_KEY"): env.pop(k, None) diff --git a/bench/swebench/test_shim_timeout.py b/bench/swebench/test_shim_timeout.py new file mode 100644 index 0000000..941b7c0 --- /dev/null +++ b/bench/swebench/test_shim_timeout.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +"""Tests for the openswarm SHIM_TIMEOUT in harness/arms.py. + +Regression test for evalid #213 (M16 slice 2): SHIM_TIMEOUT was +``budget_min * 60 + 30`` while openswarm's own budget is ``budget_min * 60`` +seconds, so whenever the executor ran the budget out, openswarm killed the +backend shim (attempt rc=124) before run_pi returned and the shim never +wrote executor-*-record.json / executor-*-events.jsonl. The 840s-budget / +870s-timeout validation cell (sympy__sympy-13878 r1, run +20261117-013602-da5999) lost 843s of executor work: no events file, executor +usage all 0, solution.diff 0 bytes. + +Stdlib only, like test_planner_prompt.py: arms.py itself imports swe_env, +which pulls in pandas / swebench, so this test reads shim_timeout_s out of +the source with ast instead of importing the module. That keeps it runnable +by the CI node suite, which shells out here with plain python3. +""" +import ast +import unittest +from pathlib import Path + +ARMS = Path(__file__).resolve().parent / "harness" / "arms.py" + + +def _shim_timeout_s(): + tree = ast.parse(ARMS.read_text()) + for node in ast.walk(tree): + if isinstance(node, ast.FunctionDef) and node.name == "shim_timeout_s": + src = ast.get_source_segment(ARMS.read_text(), node) + ns = {} + exec(src, ns) # noqa: S102 - test-only, source is this repo's file + return ns["shim_timeout_s"] + raise AssertionError("shim_timeout_s not found in harness/arms.py") + + +def _arm_openswarm_source() -> str: + tree = ast.parse(ARMS.read_text()) + for node in ast.walk(tree): + if isinstance(node, ast.FunctionDef) and node.name == "arm_openswarm": + return ast.get_source_segment(ARMS.read_text(), node) + raise AssertionError("arm_openswarm not found in harness/arms.py") + + +class TestShimTimeout(unittest.TestCase): + def test_below_openswarm_budget_for_all_budgets(self): + f = _shim_timeout_s() + for budget_min in range(1, 61): + self.assertLess( + f(budget_min), budget_min * 60, + f"SHIM_TIMEOUT must stay below the openswarm budget " + f"({budget_min} min)", + ) + + def test_validation_cell_budget(self): + # The #213 validation cell: planner 286.2s -> budget_min=14 + # (840s budget); the old 870s timeout lost the events file. + f = _shim_timeout_s() + self.assertEqual(f(14), 780) + self.assertLessEqual(f(14), 800) + + def test_small_budget_floor(self): + f = _shim_timeout_s() + self.assertEqual(f(1), 30) + self.assertGreater(f(1), 0) + + def test_call_site_uses_helper(self): + src = _arm_openswarm_source() + self.assertIn("SHIM_TIMEOUT=str(shim_timeout_s(budget_min))", src) + self.assertNotIn("budget_min * 60 + 30", ARMS.read_text()) + + +if __name__ == "__main__": + unittest.main()