feat(benchmark): add test-time scaling workflow

Cargo.toml

@@ -10,6 +10,8 @@ members = [
     "crates/protocol",
     "crates/switchyard-server",
     "crates/switchyard-skill-distillation",
+    "crates/switchyard-test-time-scaling",
+    "crates/switchyard-test-time-scaling-runner",
     "crates/switchyard-translation",
 ]
 
@@ -40,6 +42,7 @@ switchyard-libsy = { path = "crates/libsy", version = "0.2.0" }
 switchyard-llm-client = { path = "crates/libsy-llm-client", version = "0.2.0" }
 switchyard-protocol = { path = "crates/protocol", version = "0.2.0" }
 switchyard-server = { path = "crates/switchyard-server", version = "0.2.0" }
+switchyard-test-time-scaling = { path = "crates/switchyard-test-time-scaling", version = "0.2.0" }
 switchyard-translation = { path = "crates/switchyard-translation", version = "0.2.0" }
 thiserror = "2"
 tokio = { version = "1", features = ["full"] }

benchmark/README.md

@@ -129,6 +129,15 @@ uv run --no-sync python benchmark/prepare_harbor_dataset.py \
   --overwrite
 ```
 
+SWE-bench Verified is the benchmark used by the test-time scaling runner:
+
+```bash
+uv run --no-sync python benchmark/prepare_harbor_dataset.py \
+  --source-dataset swebench-verified@1.0 \
+  --output-dir benchmark/datasets/swebench-verified-closed-book \
+  --overwrite
+```
+
 SWE-Bench Pro is supported with the Harbor dataset `cais/swebenchpro`. The generated dataset uses
 the same pinned-agent and closed-book proxy path without opening dataset-specific agent egress.
 
@@ -139,6 +148,35 @@ uv run --no-sync python benchmark/prepare_harbor_dataset.py \
   --overwrite
 ```
 
+## Run Paper Test-Time Scaling
+
+The test-time scaling runner follows the paper's two-iteration schedule. One full task uses 32 agent
+attempts, 32 summaries, and 240 comparison calls. Start with smaller settings before running the
+paper defaults.
+
+The config writer records every choice that the paper does not state. It labels the result as a
+conceptual replication because those missing choices prevent an exact numerical replication.
+
+```bash
+export NVIDIA_API_KEY="..."
+
+uv run --no-sync python benchmark/test_time_scaling_config.py \
+  --dataset benchmark/datasets/swebench-verified-closed-book \
+  --task astropy__astropy-7606 \
+  --model azure/anthropic/claude-sonnet-4-5 \
+  --output benchmark/tb_runs/test-time-scaling-canary
+
+cargo run -p switchyard-test-time-scaling-runner -- \
+  benchmark/tb_runs/test-time-scaling-canary/config/runner.json
+```
+
+The default settings are N=16, K=4, G=2, and V=8. For a small end-to-end check, add
+`--rollouts 2 --refinement-count 1 --votes 1` to the config command.
+
+The runner saves `run.json` before it starts official verification. Selection cannot read verifier
+results because Harbor does not create them during the agent attempts. After selection, the grader
+applies every saved patch to a fresh task container and writes `evaluation.json`.
+
 ## Run Without Switchyard
 
 Omit `--server-config` to fully disable Switchyard. The runner still creates the benchmark

benchmark/agent-versions.env

@@ -4,6 +4,7 @@
 CLAUDE_CODE_VERSION=2.1.211
 CODEX_VERSION=0.144.5
 OPENCODE_VERSION=1.18.3
+MINI_SWE_AGENT_VERSION=2.4.6
 NODE_VERSION=20.11.1
 # Hermes (NousResearch hermes-agent), installed from GitHub at dataset-bake time.
 # Must be a full 40-character commit SHA; anything else is rejected at build time.

benchmark/prepare_harbor_dataset.py

@@ -3,8 +3,6 @@
 
 """Prepare a local closed-book Harbor dataset with prebaked coding agents."""
 
-from __future__ import annotations
-
 import argparse
 import hashlib
 import json
@@ -155,7 +153,11 @@ def _find_exported_dataset_root(download_root: Path, source_dataset: str) -> Pat
             return candidate
 
     dirs_with_tasks = sorted(
-        {task.parent.parent for task in download_root.rglob("task.toml") if task.parent != download_root}
+        {
+            task.parent.parent
+            for task in download_root.rglob("task.toml")
+            if task.parent != download_root
+        }
     )
     if len(dirs_with_tasks) == 1:
         return dirs_with_tasks[0]
@@ -164,7 +166,9 @@ def _find_exported_dataset_root(download_root: Path, source_dataset: str) -> Pat
     raise FileNotFoundError(f"could not find exported tasks under {download_root}")
 
 
-def _run_download(source_dataset: str, download_root: Path, harbor_command: str, overwrite: bool) -> Path:
+def _run_download(
+    source_dataset: str, download_root: Path, harbor_command: str, overwrite: bool
+) -> Path:
     download_root.mkdir(parents=True, exist_ok=True)
     command = [
         *shlex.split(harbor_command),
@@ -199,6 +203,7 @@ def _install_layer(pins: dict[str, str]) -> str:
     node_version = pins["NODE_VERSION"]
     claude_version = pins["CLAUDE_CODE_VERSION"]
     codex_version = pins["CODEX_VERSION"]
+    mini_swe_agent_version = pins["MINI_SWE_AGENT_VERSION"]
     opencode_version = pins["OPENCODE_VERSION"]
     # Hermes (NousResearch hermes-agent) is a per-user uv app installed from
     # GitHub, not an npm package. Baking it here (build-time, with host network)
@@ -228,7 +233,7 @@ def _install_layer(pins: dict[str, str]) -> str:
     return f"""
 
 # Switchyard benchmark prebaked coding agents.
-ENV SWITCHYARD_PREBAKED_AGENT_VERSIONS="claude-code={claude_version},codex={codex_version},opencode={opencode_version},node={node_version},hermes={hermes_version}"
+ENV SWITCHYARD_PREBAKED_AGENT_VERSIONS="claude-code={claude_version},codex={codex_version},mini-swe-agent={mini_swe_agent_version},opencode={opencode_version},node={node_version},hermes={hermes_version}"
 RUN set -eux; \\
     if command -v apt-get >/dev/null 2>&1; then \\
         apt-get update; \\
@@ -282,6 +287,12 @@ def _install_layer(pins: dict[str, str]) -> str:
     curl -fsSL https://raw.githubusercontent.com/NousResearch/hermes-agent/{hermes_version}/scripts/install.sh \\
         | bash -s -- --skip-setup --commit {hermes_version} --force-commit; \\
     hermes version
+RUN set -eux; \\
+    export HOME=/root; \\
+    export PATH="/root/.local/bin:$PATH"; \\
+    curl -LsSf https://astral.sh/uv/0.7.13/install.sh | sh; \\
+    uv tool install mini-swe-agent=={mini_swe_agent_version}; \\
+    uv tool list | grep -F "mini-swe-agent v{mini_swe_agent_version}"
 """
 
 
@@ -329,9 +340,7 @@ def _rewrite_task_image(task_dir: Path, pins: dict[str, str]) -> dict[str, Any]:
     entrypoint_layer = _entrypoint_layer()
 
     if docker_image:
-        dockerfile.write_text(
-            f"FROM {docker_image}\nUSER root\n{layer.lstrip()}{entrypoint_layer}"
-        )
+        dockerfile.write_text(f"FROM {docker_image}\nUSER root\n{layer.lstrip()}{entrypoint_layer}")
         task_toml.write_text(
             _remove_toml_key_from_table(task_toml.read_text(), "environment", "docker_image")
         )
@@ -389,7 +398,9 @@ def _append_proxy_allowlist(proxy_assets: Path, hosts: tuple[str, ...]) -> None:
     allowlist_path = proxy_assets / "allowlist-base.txt"
     base = allowlist_path.read_text().rstrip()
     additions = "\n".join(dict.fromkeys(hosts))
-    allowlist_path.write_text(f"{base}\n\n# Dataset-required package and data sources.\n{additions}\n")
+    allowlist_path.write_text(
+        f"{base}\n\n# Dataset-required package and data sources.\n{additions}\n"
+    )
 
 
 def _merge_compose(task_dir: Path, proxy_allowlist_hosts: tuple[str, ...]) -> dict[str, Any]:
@@ -449,7 +460,11 @@ def _merge_compose(task_dir: Path, proxy_allowlist_hosts: tuple[str, ...]) -> di
         "proxy": {"condition": "service_healthy"},
     }
     main["volumes"] = [
-        *([str(item) for item in main.get("volumes", [])] if isinstance(main.get("volumes"), list) else []),
+        *(
+            [str(item) for item in main.get("volumes", [])]
+            if isinstance(main.get("volumes"), list)
+            else []
+        ),
         "proxy-ca-public:/etc/proxy-ca:ro",
     ]
 
@@ -506,7 +521,6 @@ def _merge_compose(task_dir: Path, proxy_allowlist_hosts: tuple[str, ...]) -> di
 
 
 def prepare_dataset(
-    *,
     source_dataset: str,
     source_dir: Path | None,
     output_dir: Path,
@@ -518,6 +532,7 @@ def prepare_dataset(
         "CLAUDE_CODE_VERSION",
         "CODEX_VERSION",
         "HERMES_VERSION",
+        "MINI_SWE_AGENT_VERSION",
         "NODE_VERSION",
         "OPENCODE_VERSION",
     }
@@ -582,7 +597,9 @@ def _cli_main(argv: list[str] | None = None) -> int:
     parser.add_argument("--source-dataset", default=DEFAULT_SOURCE_DATASET)
     parser.add_argument("--source-dir", type=Path, default=None)
     parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT_DIR)
-    parser.add_argument("--harbor-command", default=os.environ.get("HARBOR_COMMAND", "uv run --no-sync harbor"))
+    parser.add_argument(
+        "--harbor-command", default=os.environ.get("HARBOR_COMMAND", "uv run --no-sync harbor")
+    )
     parser.add_argument("--overwrite", action="store_true")
     ns = parser.parse_args(argv)
 

benchmark/run-baseline.sh

@@ -24,6 +24,7 @@ fi
 CLAUDE_CODE_VERSION="${CLAUDE_CODE_VERSION:-2.1.211}"
 CODEX_VERSION="${CODEX_VERSION:-0.144.5}"
 OPENCODE_VERSION="${OPENCODE_VERSION:-1.18.3}"
+MINI_SWE_AGENT_VERSION="${MINI_SWE_AGENT_VERSION:-2.4.6}"
 NODE_VERSION="${NODE_VERSION:-20.11.1}"
 
 DEFAULT_HARBOR_MODEL="openai/gpt-5.2"
@@ -38,6 +39,7 @@ SERVER_CONFIG=""
 MODEL=""
 ROUTE_MODEL=""
 AGENT="terminus-2"
+AGENT_IMPORT_PATH=""
 HARBOR_MODEL="${DEFAULT_HARBOR_MODEL}"
 HARBOR_PATH=""
 HARBOR_BIN="${HARBOR_BIN:-}"
@@ -101,6 +103,7 @@ Main options:
                                agents, and MODEL for claude-code/codex.
   --route-model MODEL          Deprecated alias for --model in Switchyard mode.
   --agent NAME                 Harbor agent (default: terminus-2)
+  --agent-import-path PATH     Custom Harbor agent in module.path:ClassName form.
   --harbor-model MODEL         Explicit Harbor model label override.
   --upstream-base-url URL      Direct-upstream OpenAI-compatible base URL
                                (default: https://openrouter.ai/api/v1).
@@ -305,6 +308,7 @@ while [[ $# -gt 0 ]]; do
         --model) MODEL="$2"; shift 2 ;;
         --route-model) ROUTE_MODEL="$2"; shift 2 ;;
         --agent) AGENT="$2"; shift 2 ;;
+        --agent-import-path) AGENT_IMPORT_PATH="$2"; shift 2 ;;
         --harbor-model) HARBOR_MODEL="$2"; HARBOR_MODEL_SET=1; shift 2 ;;
         --upstream-base-url) UPSTREAM_BASE_URL="$2"; shift 2 ;;
         --upstream-api-key-env) UPSTREAM_API_KEY_ENV="$2"; shift 2 ;;
@@ -538,15 +542,23 @@ fi
 WRAPPER="${RUN_DIR}/run-background.sh"
 HARBOR_JOB_DIR="${RUN_DIR}/jobs/${JOB_NAME}"
 
-HARBOR_CMD=("${HARBOR_CMD_PREFIX[@]}" run
-    --agent "${AGENT}"
-    --model "${HARBOR_MODEL}"
+HARBOR_CMD=("${HARBOR_CMD_PREFIX[@]}" run)
+if [[ -n "${AGENT_IMPORT_PATH}" ]]; then
+    HARBOR_CMD+=(--agent-import-path "${AGENT_IMPORT_PATH}")
+else
+    HARBOR_CMD+=(--agent "${AGENT}")
+fi
+HARBOR_CMD+=(--model "${HARBOR_MODEL}"
     --jobs-dir "${RUN_DIR}/jobs"
     --job-name "${JOB_NAME}"
     -n "${N_CONCURRENT}"
     --max-retries "${MAX_RETRIES}"
     --agent-timeout-multiplier "${AGENT_TIMEOUT_MULTIPLIER}")
 
+if [[ "${AGENT}" == "mini-swe-agent" ]]; then
+    HARBOR_CMD+=(--ak "version=${MINI_SWE_AGENT_VERSION}")
+fi
+
 HARBOR_CMD+=(--path "${HARBOR_PATH}")
 
 if [[ -n "${CODEX_MODEL_CATALOG_HOST}" ]]; then
@@ -710,7 +722,7 @@ print_resolved() {
     shell_join "${HARBOR_PYTHON_CMD[@]}"
     echo
     echo "  harbor_patch:  verified"
-    echo "  agent/model:   ${AGENT} / ${HARBOR_MODEL}"
+    echo "  agent/model:   ${AGENT_IMPORT_PATH:-${AGENT}} / ${HARBOR_MODEL}"
     echo "  reasoning:     ${REASONING_EFFORT:-unset}"
     if [[ -n "${CODEX_MODEL_CATALOG_HOST}" ]]; then
         echo "  codex_catalog: ${CODEX_MODEL_CATALOG_HOST}"
@@ -747,7 +759,8 @@ fi
 
 AGENT_VERSIONS_JSON="$(json_object_from_pairs \
     claude_code "${CLAUDE_CODE_VERSION}" codex "${CODEX_VERSION}" \
-    opencode "${OPENCODE_VERSION}" node "${NODE_VERSION}")"
+    opencode "${OPENCODE_VERSION}" mini_swe_agent "${MINI_SWE_AGENT_VERSION}" \
+    node "${NODE_VERSION}")"
 HARBOR_EXTRA_JSON="$(json_array)"
 if [[ "${#HARBOR_EXTRA[@]}" -gt 0 ]]; then
     HARBOR_EXTRA_JSON="$(json_array "${HARBOR_EXTRA[@]}")"
@@ -855,11 +868,31 @@ cleanup() {
         docker rm -f "\${SWITCHYARD_DOCKER_CONTAINER}" >/dev/null 2>&1 || true
     fi
     if [[ "\${DOCKER_NETWORK_CREATED}" == "1" ]]; then
-        docker network rm "\${SWITCHYARD_DOCKER_NETWORK}" >/dev/null 2>&1 || true
+        docker_with_timeout network rm "\${SWITCHYARD_DOCKER_NETWORK}" >/dev/null 2>&1 || true
     fi
 }
 trap cleanup EXIT
 
+docker_with_timeout() {
+    if command -v timeout >/dev/null 2>&1; then
+        timeout 30 docker "\$@"
+    else
+        docker "\$@"
+    fi
+}
+
+ensure_docker_network() {
+    if docker_with_timeout network inspect "\${SWITCHYARD_DOCKER_NETWORK}" >/dev/null 2>&1; then
+        return
+    fi
+    if ! docker_with_timeout network create "\${SWITCHYARD_DOCKER_NETWORK}" >/dev/null; then
+        echo "ERROR: Docker could not inspect or create network \${SWITCHYARD_DOCKER_NETWORK}." >&2
+        echo "       Check the Docker network API before retrying this benchmark." >&2
+        exit 127
+    fi
+    DOCKER_NETWORK_CREATED=1
+}
+
 export PYTHONHASHSEED=0
 export LC_ALL=C.UTF-8
 
@@ -872,10 +905,7 @@ if [[ "\${SERVER_ENABLED}" == "1" ]]; then
     if [[ "\${SWITCHYARD_DOCKER_BUILD}" != "0" ]]; then
         docker build -f "\${SWITCHYARD_DOCKERFILE}" -t "\${SWITCHYARD_DOCKER_IMAGE}" .
     fi
-    if ! docker network inspect "\${SWITCHYARD_DOCKER_NETWORK}" >/dev/null 2>&1; then
-        docker network create "\${SWITCHYARD_DOCKER_NETWORK}"
-        DOCKER_NETWORK_CREATED=1
-    fi
+    ensure_docker_network
     docker rm -f "\${SWITCHYARD_DOCKER_CONTAINER}" >/dev/null 2>&1 || true
     DOCKER_RUN_ARGS=(
         -d --rm
@@ -920,10 +950,7 @@ else
         echo "Switchyard disabled; using direct upstream"
         echo "  network: \${SWITCHYARD_DOCKER_NETWORK}"
         echo "  upstream: \${HARBOR_BASE_URL}"
-        if ! docker network inspect "\${SWITCHYARD_DOCKER_NETWORK}" >/dev/null 2>&1; then
-            docker network create "\${SWITCHYARD_DOCKER_NETWORK}"
-            DOCKER_NETWORK_CREATED=1
-        fi
+        ensure_docker_network
     } > "\${SERVER_LOG}" 2>&1
     echo "\${SWITCHYARD_DOCKER_NETWORK}" > "\${RUN_DIR}/docker_network"
 fi
@@ -979,6 +1006,7 @@ else
     export OPENAI_API_KEY="\${!UPSTREAM_API_KEY_ENV}"
 fi
 export OPENAI_BASE_URL="\${HARBOR_BASE_URL}"
+export OPENAI_API_BASE="\${HARBOR_BASE_URL}"
 if [[ "\${BOOK_MODE}" == "closed" ]]; then
     export CLOSED_BOOK_MODE=1
 else