Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions bench/swebench/harness/arms.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,12 @@
OPENSWARM_DIR = Path(os.environ.get("OPENSWARM_DIR", "<openswarm>"))
NODE22_BIN = "<node22>/bin"
CAP_S = 20 * 60

# Planner budget in seconds (default 600, override with EVALID_PLANNER_TIMEOUT_S):
# 600s keeps planner + executor inside the CAP_S 1200s cell budget (executor
# budget is derived from the remaining time, see arm_openswarm). Raising it
# trades executor time 1:1, so only raise it deliberately, never to the full cap.
PLANNER_TIMEOUT_S = int(os.environ.get("EVALID_PLANNER_TIMEOUT_S", "600"))
GIT = E.GIT

# Dead-end proxy (port 9 = discard) plus offline installer flags: the agent's shell gets no
Expand Down Expand Up @@ -265,7 +271,7 @@ def arm_openswarm(inst, repo, venv, events, tid):
t0 = time.time()
plan = PLANNER.format(task_file=task_file, verify_file=verify_file, python=f"{venv}/bin/python", repo=repo,
skill=(OPENSWARM_DIR / "SKILL.md").read_text(), task=base_prompt(inst, repo, venv))
# Planner budget 600s (was 300s): the 300s cap was calibrated on DeepSeek,
# Planner budget PLANNER_TIMEOUT_S (default 600s, was 300s): the 300s cap was calibrated on DeepSeek,
# whose planner finished in 87-244s on these 6 instances. With MiniMax
# M3.1-Flash-Preview at high thinking the planner exceeds 300s on all 6
# pilot2 cells — killed mid-work (rc 143) before writing the task file,
Expand All @@ -276,7 +282,8 @@ def arm_openswarm(inst, repo, venv, events, tid):
# reproduction, no filesystem search): the 600s validation cell showed
# the planner spending 13 turns / 23 tools exploring instead of
# converging to the task file, so the budget alone cannot fix it.
calls = [run_pi(tid + "-plan", rundir, plan, events, 600, path_env(venv), label="planner")]
# Override via EVALID_PLANNER_TIMEOUT_S; default stays 600s (M16 slice 2, #211).
calls = [run_pi(tid + "-plan", rundir, plan, events, PLANNER_TIMEOUT_S, path_env(venv), label="planner")]
if task_file.exists():
shutil.copy(task_file, events / "executor-task.md")
if verify_file.exists():
Expand Down
11 changes: 6 additions & 5 deletions bench/swebench/test_planner_prompt.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
by the CI node suite, which shells out here with plain python3.
"""
import ast
import re
import unittest
from pathlib import Path

Expand Down Expand Up @@ -76,11 +75,13 @@ def test_still_forbids_touching_the_repo(self):
self.assertIn("Do not commit", PLANNER)

def test_planner_timeout_stays_600s(self):
text = ARMS.read_text()
self.assertIn('"EVALID_PLANNER_TIMEOUT_S", "600"', text,
"planner timeout default must stay 600s (PR #41); "
"override only via EVALID_PLANNER_TIMEOUT_S")
source = _arm_openswarm_source()
match = re.search(r"run_pi\(\s*tid\s*\+\s*\"-plan\".*?,\s*(\d+)\s*,", source, re.DOTALL)
self.assertIsNotNone(match, "planner run_pi call not found in arm_openswarm")
self.assertEqual(match.group(1), "600",
"planner timeout must stay 600s (PR #41); only the prompt changes")
self.assertRegex(source, r"run_pi\(\s*tid\s*\+\s*\"-plan\".*?,\s*PLANNER_TIMEOUT_S\s*,",
"planner run_pi call must pass PLANNER_TIMEOUT_S")


if __name__ == "__main__":
Expand Down