From 91bd64b071154ef0ae14b187209e670c4aaaa474 Mon Sep 17 00:00:00 2001
From: Gen TANG <gen.tang86@gmail.com>
Date: Mon, 3 Aug 2026 16:19:35 +0800
Subject: [PATCH 1/3] simplify the procedure of yada eval

---
 docs/cli-reference.md                 | 167 +++++++++++++++++---------
 src/yada/evals/benchmarks/swebench.py | 121 +++++++++++--------
 src/yada/evals/cli.py                 |  96 ++++-----------
 tests/evals/test_eval_cli.py          |  38 ++++++
 tests/evals/test_local_benchmark.py   |   8 +-
 tests/evals/test_swebench.py          |  76 +++++++++---
 6 files changed, 307 insertions(+), 199 deletions(-)
 create mode 100644 tests/evals/test_eval_cli.py

diff --git a/docs/cli-reference.md b/docs/cli-reference.md
index cd6f6c1..1612d46 100644
--- a/docs/cli-reference.md
+++ b/docs/cli-reference.md
@@ -53,48 +53,68 @@ keyboard interruption. Argument errors also use the standard argparse status `2`
 ## `yada eval`
 
 `yada eval` runs an agent behind a benchmark-neutral adapter and persists a
-schema-versioned result even for ordinary adapter failures.
+schema-versioned result even for ordinary adapter failures. Unlike a direct
+`yada` run, an evaluation always has a task adapter and a grader.
 
-### Checked-in development case
+### Choose a run mode
+
+```mermaid
+flowchart TD
+    start{"What do you want to run?"}
+    start -->|"Edit one repository"| direct["yada TASK --workspace REPO"]
+    start -->|"Run a reproducible evaluation"| eval["yada eval"]
+
+    eval --> source{"Which task source?"}
+    source -->|"Checked-in case directory"| case["--case PATH"]
+    source -->|"Official SWE-bench Verified"| swe["--swebench INSTANCE_ID"]
+
+    case --> local_adapter["Local adapter\ncase.json + declared grader"]
+    local_adapter --> local_result["Development/private result\nnot an official SWE-bench score"]
+
+    swe --> official["Generate patch, then run\nofficial SWE-bench Docker Harness"]
+    official --> official_result["Official-compatible SWE-bench result"]
+```
+
+`yada eval` deliberately exposes only two task selectors:
+
+| Invocation | Meaning | Grader |
+| --- | --- | --- |
+| `--case PATH` | Run a portable local case. A directory resolves to `PATH/case.json`; a manifest file may also be passed directly. | The command declared by the case manifest |
+| `--swebench INSTANCE_ID` | Load one SWE-bench Verified task, generate a prediction, and delegate grading to the official Docker Harness. | Official SWE-bench Docker Harness |
+
+There is no `--local`, `--benchmark`, or separate `--manifest` selector. Pass a
+custom manifest directly to `--case`. A case is called local because its
+manifest controls preparation and grading; it may still fetch a Git repository
+and call the DeepSeek API. Each invocation runs exactly one task.
+
+### Checked-in development case (`--case`)
+
+Use a checked-in case for fast development and regression testing:
 
 ```bash
 uv run yada eval \
   --case benchmarks/swebench_verified/pytest-10051 \
   --agent yada \
-  --yes
+  --yes \
+  --trace-level debug
 ```
 
 The first run fetches the pinned pytest commit and builds the case's locked
 Python environment. Later runs reuse caches but create a fresh agent workspace.
-This local grader is a development feedback loop, not an official SWE-bench
-Docker score.
-
-### Common options
-
-| Option | Meaning | Default |
-| --- | --- | --- |
-| `--benchmark local\|swebench` | Benchmark adapter. | Inferred from `--case`, otherwise required |
-| `--case PATH` | Portable local case directory or `case.json`. | — |
-| `--instance ID` | Benchmark instance ID. | Manifest ID for local cases |
-| `--agent yada\|command` | Agent adapter. | `yada` |
-| `--output PATH` | Result JSON path. | `eval-results/<task>__<time>.json` |
-| `--artifact-dir PATH` | Workspace, logs, patch, trace, and grader artifacts. | Sibling `<result>.artifacts` |
-| `--run-id ID` | Stable correlation ID. | Generated |
-| `--max-steps N` | Model-turn budget. | `30` |
-| `--wall-time SECONDS` | Comparable wall-time budget. | `1800` |
-| `--max-output-tokens N` | Per-completion token limit. | `16384` |
+The case's grader is a development feedback loop, not the official SWE-bench
+Docker grader, even though the task originated in SWE-bench Verified.
 
-The native agent also accepts the model, thinking, timeout, command-policy, and
-trace-level options documented for `yada`. A deployment-level supervisor should
-enforce a hard wall-time limit for an in-process native agent.
+For a private or machine-local task, pass its manifest through the same entry
+point:
 
-Evaluation exits with `0` for `resolved`, `1` for `unresolved`, and `2` for
-errors or non-verdict outcomes such as skipped grading.
-
-### Local manifest
+```bash
+uv run yada eval \
+  --case /path/to/task.local.json \
+  --agent yada \
+  --yes
+```
 
-Use `--benchmark local --manifest FILE` for a private or machine-local task.
-A minimal version 1 manifest is:
+A minimal version 1 case manifest is:
 
 ```json
 {
@@ -151,14 +171,54 @@ Portable Git checkouts are cached under `--cache-dir` (default
 `.yada/cache/evals`) but never used directly as the mutable agent workspace.
 `install_workspace` accepts `editable` and `legacy-editable`.
 
-### External command agent
+### Official SWE-bench evaluation (`--swebench`)
+
+The official SWE-bench Harness is the single source for both public instance
+metadata and Docker grading. `--swebench` does not read the checked-in local
+case's `instance.json`, so there are no local and online copies to keep in sync.
+Gold patches, test patches, and hidden test IDs do not cross the public task
+boundary. Yada fetches the exact base commit, prepares a clean candidate
+workspace, and generates the patch that the Harness grades.
+
+Install and start Docker. The official `swebench` package is published on PyPI,
+so no SWE-bench Git clone or separately managed virtual environment is needed.
+Use uv's cached dependency overlay to run the pinned Harness version:
+
+```bash
+uv run --with 'swebench==4.1.0' yada eval \
+  --swebench pytest-dev__pytest-10051 \
+  --agent yada \
+  --yes \
+  --trace-level debug
+```
+
+The first run downloads the Harness and its dependencies into uv's cache;
+subsequent runs reuse them. This keeps Yada's core dependency-free and leaves
+the project environment unchanged. If `swebench==4.1.0` is already installed in
+Yada's active environment, the shorter `uv run yada eval ...` form also works.
+
+The adapter intentionally fixes the current public evaluation policy instead of
+exposing Harness internals as Yada CLI flags:
+
+| Setting | Built-in policy |
+| --- | --- |
+| Dataset | `princeton-nlp/SWE-bench_Verified` |
+| Split | `test` |
+| Grading | Official Docker Harness |
+| Image cache | Keep environment images |
+| Grading timeout | 1800 seconds |
+| Docker namespace | `swebench`; automatically disabled on Apple Silicon so images build locally |
+
+Apple Silicon support in SWE-bench remains experimental. Building images locally
+takes substantially more time and disk space than running a checked-in case.
+
+### External command agent (`--agent command`)
 
 The command adapter runs a non-interactive argv without a shell:
 
 ```bash
 uv run yada eval \
-  --benchmark local \
-  --manifest /path/to/task.local.json \
+  --case /path/to/task.local.json \
   --agent command \
   --agent-name another-agent \
   --agent-command \
@@ -170,34 +230,27 @@ The template supports `{task}`, `{task_file}`, `{workspace}`, `{output_patch}`,
 and `{run_dir}`. If the command does not write `{output_patch}`, Yada collects
 the complete Git diff, including untracked files.
 
-### SWE-bench
+### Common evaluation options
 
-The SWE-bench adapter loads public instance metadata from `--instance-file`, or
-from Hugging Face when the optional `datasets` package is installed. It discards
-gold patches, test patches, and hidden test IDs at the public task boundary.
-
-Install the official SWE-bench package and Docker separately, then run:
-
-```bash
-uv run yada eval \
-  --benchmark swebench \
-  --instance pytest-dev__pytest-10051 \
-  --instance-file benchmarks/swebench_verified/pytest-10051/instance.json \
-  --workspace /path/to/clean/pytest-base-repo \
-  --agent yada \
-  --yes \
-  --swebench-python /path/to/swebench-env/bin/python \
-  --output results/yada-pytest-10051.json
-```
+| Option | Meaning | Default |
+| --- | --- | --- |
+| `--case PATH` | Portable local case directory or `case.json`. | Mutually exclusive with `--swebench` |
+| `--swebench ID` | One official SWE-bench Verified instance. | Mutually exclusive with `--case` |
+| `--agent yada\|command` | Agent adapter. | `yada` |
+| `--output PATH` | Result JSON path. | `eval-results/<task>__<time>.json` |
+| `--artifact-dir PATH` | Workspace, logs, patch, trace, and grader artifacts. | Sibling `<result>.artifacts` |
+| `--run-id ID` | Stable correlation ID. | Generated |
+| `--max-steps N` | Model-turn budget. | `30` |
+| `--wall-time SECONDS` | Comparable wall-time budget. | `1800` |
+| `--max-output-tokens N` | Per-completion token limit. | `16384` |
 
-`--workspace` is optional; without it, Yada fetches the exact base commit. It
-prepares the candidate workspace but does not replace official Docker grading.
-Use `--grade-mode none` to test preparation and prediction generation without
-Docker; the result is `skipped`, never `resolved`.
+The native agent also accepts the model, thinking, timeout, command-policy, and
+trace-level options documented for `yada`. A deployment-level supervisor should
+enforce a hard wall-time limit for an in-process native agent. Use the same task,
+base commit, model budget, network policy, and grader when comparing agents.
 
-Other SWE-bench options are `--dataset-name`, `--split`, `--cache-level`,
-`--clean`, `--namespace`, and `--grade-timeout`. Use the same task, base commit,
-model budget, network policy, and official grader when comparing agents.
+Evaluation exits with `0` for `resolved`, `1` for `unresolved`, and `2` for
+errors or non-verdict outcomes such as skipped grading.
 
 ## `yada-trace`
 
diff --git a/src/yada/evals/benchmarks/swebench.py b/src/yada/evals/benchmarks/swebench.py
index 6a89b45..ec10763 100644
--- a/src/yada/evals/benchmarks/swebench.py
+++ b/src/yada/evals/benchmarks/swebench.py
@@ -12,6 +12,34 @@
 
 from yada.evals.base import AgentRunResult, EvalTask, GradeResult, PreparedTask
 
+_PUBLIC_INSTANCE_FIELDS = (
+    "instance_id",
+    "repo",
+    "base_commit",
+    "problem_statement",
+    "version",
+    "difficulty",
+    "dataset_revision",
+    "created_at",
+    "environment_setup_commit",
+)
+_SWEBENCH_PACKAGE = "swebench==4.1.0"
+_INSTANCE_PREFIX = "__YADA_SWEBENCH_INSTANCE__="
+_INSTANCE_LOADER = f"""
+import json
+import sys
+from swebench.harness.utils import load_swebench_dataset
+
+dataset_name, split, instance_id = sys.argv[1:4]
+rows = load_swebench_dataset(dataset_name, split, [instance_id])
+row = next((item for item in rows if item.get("instance_id") == instance_id), None)
+if row is None:
+    raise SystemExit(f"instance {{instance_id!r}} not found")
+public_fields = {_PUBLIC_INSTANCE_FIELDS!r}
+public_row = {{key: row.get(key) for key in public_fields}}
+print({_INSTANCE_PREFIX!r} + json.dumps(public_row, ensure_ascii=False, default=str))
+"""
+
 
 class SWEbenchBenchmark:
     """Prepare a base repository and delegate grading to SWE-bench Harness."""
@@ -23,7 +51,6 @@ def __init__(
         *,
         dataset_name: str = "princeton-nlp/SWE-bench_Verified",
         split: str = "test",
-        instance_file: Path | None = None,
         source_workspace: Path | None = None,
         harness_python: str = sys.executable,
         grade_mode: str = "docker",
@@ -38,9 +65,6 @@ def __init__(
             raise ValueError("invalid SWE-bench cache level")
         self.dataset_name = dataset_name
         self.split = split
-        self.instance_file = (
-            instance_file.expanduser().resolve() if instance_file else None
-        )
         self.source_workspace = (
             source_workspace.expanduser().resolve() if source_workspace else None
         )
@@ -52,14 +76,11 @@ def __init__(
         self.grade_timeout_seconds = grade_timeout_seconds
 
     def load_task(self, instance_id: str) -> EvalTask:
-        row = (
-            _load_instance_file(self.instance_file, instance_id)
-            if self.instance_file
-            else _load_huggingface_instance(
-                self.dataset_name,
-                self.split,
-                instance_id,
-            )
+        row = _load_harness_instance(
+            self.harness_python,
+            self.dataset_name,
+            self.split,
+            instance_id,
         )
         required = ("instance_id", "repo", "base_commit", "problem_statement")
         missing = [key for key in required if not row.get(key)]
@@ -67,7 +88,8 @@ def load_task(self, instance_id: str) -> EvalTask:
             raise ValueError(f"SWE-bench instance is missing fields: {missing}")
         if row["instance_id"] != instance_id:
             raise ValueError(
-                f"instance file contains {row['instance_id']!r}, not {instance_id!r}"
+                f"SWE-bench Harness returned {row['instance_id']!r}, "
+                f"not {instance_id!r}"
             )
 
         # Gold patches, hidden test patches, and test IDs deliberately do not cross
@@ -244,48 +266,53 @@ def grade(
         return GradeResult("error", None, duration_ms, details=details)
 
 
-def _load_instance_file(path: Path, instance_id: str) -> dict[str, Any]:
-    try:
-        text = path.read_text(encoding="utf-8")
-    except OSError as exc:
-        raise ValueError(f"cannot read SWE-bench instance file: {exc}") from exc
-    try:
-        data = json.loads(text)
-        if isinstance(data, dict):
-            rows: list[Any] = data.get("rows", [data])
-        elif isinstance(data, list):
-            rows = data
-        else:
-            rows = []
-    except json.JSONDecodeError:
-        rows = [json.loads(line) for line in text.splitlines() if line.strip()]
-
-    for item in rows:
-        row = item.get("row", item) if isinstance(item, dict) else {}
-        if row.get("instance_id") == instance_id:
-            return dict(row)
-    raise ValueError(f"instance {instance_id!r} not found in {path}")
-
-
-def _load_huggingface_instance(
+def _load_harness_instance(
+    harness_python: str,
     dataset_name: str,
     split: str,
     instance_id: str,
 ) -> dict[str, Any]:
     try:
-        from datasets import load_dataset
-    except ImportError as exc:
+        process = subprocess.run(
+            [harness_python, "-c", _INSTANCE_LOADER, dataset_name, split, instance_id],
+            text=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            timeout=300,
+            check=False,
+        )
+    except FileNotFoundError as exc:
         raise RuntimeError(
-            "loading SWE-bench from Hugging Face requires the optional "
-            "'datasets' package; install it or pass --instance-file"
+            f"SWE-bench Harness Python does not exist: {harness_python}. "
+            "Set SWEBENCH_PYTHON to its Python executable."
         ) from exc
-    dataset = load_dataset(dataset_name, split=split)
-    for row in dataset:
-        if row.get("instance_id") == instance_id:
-            return dict(row)
-    raise ValueError(
-        f"instance {instance_id!r} not found in {dataset_name} split {split}"
+    except subprocess.TimeoutExpired as exc:
+        raise RuntimeError("timed out while loading the SWE-bench instance") from exc
+    if process.returncode:
+        detail = process.stderr.strip() or process.stdout.strip()
+        raise RuntimeError(
+            "cannot load the SWE-bench instance with the official Harness. "
+            f"Run Yada with `uv run --with '{_SWEBENCH_PACKAGE}' yada eval "
+            "--swebench INSTANCE_ID ...`. "
+            f"{detail}"
+        )
+    record = next(
+        (
+            line.removeprefix(_INSTANCE_PREFIX)
+            for line in process.stdout.splitlines()
+            if line.startswith(_INSTANCE_PREFIX)
+        ),
+        None,
     )
+    if record is None:
+        raise RuntimeError("official SWE-bench Harness returned no instance metadata")
+    try:
+        row = json.loads(record)
+    except json.JSONDecodeError as exc:
+        raise RuntimeError("official SWE-bench Harness returned invalid metadata") from exc
+    if not isinstance(row, dict):
+        raise RuntimeError("official SWE-bench Harness returned invalid metadata")
+    return {key: row.get(key) for key in _PUBLIC_INSTANCE_FIELDS}
 
 
 def _require_head(workspace: Path, expected: str) -> None:
diff --git a/src/yada/evals/cli.py b/src/yada/evals/cli.py
index d8a2fab..b15b55a 100644
--- a/src/yada/evals/cli.py
+++ b/src/yada/evals/cli.py
@@ -4,6 +4,7 @@
 
 import argparse
 import os
+import platform
 import shlex
 import sys
 from pathlib import Path
@@ -22,16 +23,17 @@ def build_parser() -> argparse.ArgumentParser:
         prog="yada eval",
         description="Run one coding agent against one reproducible benchmark task.",
     )
-    parser.add_argument("--benchmark", choices=["local", "swebench"])
-    parser.add_argument(
+    task = parser.add_mutually_exclusive_group(required=True)
+    task.add_argument(
         "--case",
         type=Path,
-        help=(
-            "Portable local benchmark case directory or case.json. Implies "
-            "--benchmark local."
-        ),
+        help="Portable local case directory or case.json.",
+    )
+    task.add_argument(
+        "--swebench",
+        metavar="INSTANCE_ID",
+        help="Official SWE-bench Verified instance to run and grade in Docker.",
     )
-    parser.add_argument("--instance", help="Benchmark instance ID.")
     parser.add_argument("--agent", choices=["yada", "command"], default="yada")
     parser.add_argument(
         "--agent-command",
@@ -89,41 +91,6 @@ def build_parser() -> argparse.ArgumentParser:
         default="summary",
     )
 
-    local = parser.add_argument_group("local benchmark")
-    local.add_argument("--manifest", type=Path)
-    local.add_argument(
-        "--cache-dir",
-        type=Path,
-        default=Path(".yada/cache/evals"),
-        help="Git checkout cache used by portable local cases.",
-    )
-
-    swebench = parser.add_argument_group("SWE-bench")
-    swebench.add_argument(
-        "--dataset-name",
-        default="princeton-nlp/SWE-bench_Verified",
-    )
-    swebench.add_argument("--split", default="test")
-    swebench.add_argument("--instance-file", type=Path)
-    swebench.add_argument(
-        "--workspace",
-        type=Path,
-        help="Optional clean source repo to clone instead of fetching GitHub.",
-    )
-    swebench.add_argument("--swebench-python", default=sys.executable)
-    swebench.add_argument("--grade-mode", choices=["docker", "none"], default="docker")
-    swebench.add_argument(
-        "--cache-level",
-        choices=["none", "base", "env", "instance"],
-        default="env",
-    )
-    swebench.add_argument("--clean", action="store_true")
-    swebench.add_argument(
-        "--namespace",
-        default="swebench",
-        help="Docker image namespace; use 'none' to build locally.",
-    )
-    swebench.add_argument("--grade-timeout", type=int, default=1_800)
     return parser
 
 
@@ -133,46 +100,21 @@ def run_cli(argv: list[str] | None = None) -> int:
     parser = build_parser()
     args = parser.parse_args(argv)
     if args.case is not None:
-        if args.benchmark not in {None, "local"}:
-            parser.error("--case can only be used with --benchmark local")
-        if args.manifest is not None:
-            parser.error("--case and --manifest are mutually exclusive")
         case_path = args.case.expanduser().resolve()
         manifest_path = case_path / "case.json" if case_path.is_dir() else case_path
         if not manifest_path.is_file():
             parser.error(f"benchmark case does not exist: {manifest_path}")
-        benchmark = LocalBenchmark(
-            manifest_path,
-            cache_root=args.cache_dir,
-        )
-        instance_id = args.instance or _manifest_instance_id(benchmark)
+        benchmark = LocalBenchmark(manifest_path)
+        instance_id = _manifest_instance_id(benchmark)
         task_name = instance_id or case_path.stem
-    elif args.benchmark == "local":
-        if args.manifest is None:
-            parser.error("--manifest is required for --benchmark local")
-        benchmark = LocalBenchmark(args.manifest, cache_root=args.cache_dir)
-        instance_id = args.instance or _manifest_instance_id(benchmark)
-        task_name = instance_id or args.manifest.stem
-    elif args.benchmark == "swebench":
-        if not args.instance:
-            parser.error("--instance is required for --benchmark swebench")
-        namespace = None if args.namespace.lower() == "none" else args.namespace
+    else:
         benchmark = SWEbenchBenchmark(
-            dataset_name=args.dataset_name,
-            split=args.split,
-            instance_file=args.instance_file,
-            source_workspace=args.workspace,
-            harness_python=args.swebench_python,
-            grade_mode=args.grade_mode,
-            cache_level=args.cache_level,
-            clean=args.clean,
-            namespace=namespace,
-            grade_timeout_seconds=args.grade_timeout,
+            harness_python=os.environ.get("SWEBENCH_PYTHON", sys.executable),
+            namespace=_default_swebench_namespace(),
         )
-        instance_id = args.instance
+        instance_id = args.swebench
+        assert instance_id is not None
         task_name = instance_id
-    else:
-        parser.error("provide --case or --benchmark")
 
     output_path = (
         args.output.expanduser().resolve()
@@ -254,3 +196,9 @@ def _default_output_path(task_name: str) -> Path:
 def _manifest_instance_id(benchmark: LocalBenchmark) -> str:
     value = benchmark.manifest.get("instance_id")
     return value if isinstance(value, str) else ""
+
+
+def _default_swebench_namespace() -> str | None:
+    if sys.platform == "darwin" and platform.machine().lower() in {"arm64", "aarch64"}:
+        return None
+    return "swebench"
diff --git a/tests/evals/test_eval_cli.py b/tests/evals/test_eval_cli.py
new file mode 100644
index 0000000..15e288c
--- /dev/null
+++ b/tests/evals/test_eval_cli.py
@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+import pytest
+
+from yada.evals.cli import _default_swebench_namespace, build_parser
+
+
+def test_eval_cli_has_two_task_selectors() -> None:
+    parser = build_parser()
+
+    case = parser.parse_args(["--case", "case-dir"])
+    swebench = parser.parse_args(["--swebench", "owner__repo-1"])
+
+    assert str(case.case) == "case-dir"
+    assert case.swebench is None
+    assert swebench.case is None
+    assert swebench.swebench == "owner__repo-1"
+
+
+def test_eval_cli_rejects_multiple_task_selectors() -> None:
+    with pytest.raises(SystemExit):
+        build_parser().parse_args(
+            ["--case", "case-dir", "--swebench", "owner__repo-1"]
+        )
+
+
+def test_swebench_namespace_is_automatic_on_apple_silicon(monkeypatch) -> None:
+    monkeypatch.setattr("yada.evals.cli.sys.platform", "darwin")
+    monkeypatch.setattr("yada.evals.cli.platform.machine", lambda: "arm64")
+
+    assert _default_swebench_namespace() is None
+
+
+def test_swebench_namespace_uses_prebuilt_images_elsewhere(monkeypatch) -> None:
+    monkeypatch.setattr("yada.evals.cli.sys.platform", "linux")
+    monkeypatch.setattr("yada.evals.cli.platform.machine", lambda: "x86_64")
+
+    assert _default_swebench_namespace() == "swebench"
diff --git a/tests/evals/test_local_benchmark.py b/tests/evals/test_local_benchmark.py
index 4b32188..ce29141 100644
--- a/tests/evals/test_local_benchmark.py
+++ b/tests/evals/test_local_benchmark.py
@@ -190,14 +190,18 @@ def test_local_grader_keeps_virtualenv_python_symlink(tmp_path: Path) -> None:
     assert resolved == str(python_link)
 
 
-def test_checked_in_case_shares_prompt_with_swebench_adapter() -> None:
+def test_checked_in_case_shares_prompt_with_swebench_adapter(monkeypatch) -> None:
     case_dir = (
         Path(__file__).resolve().parents[2]
         / "benchmarks/swebench_verified/pytest-10051"
     )
     local_task = LocalBenchmark(case_dir / "case.json").load_task("")
+    instance = json.loads((case_dir / "instance.json").read_text(encoding="utf-8"))
+    monkeypatch.setattr(
+        "yada.evals.benchmarks.swebench._load_harness_instance",
+        lambda *_: instance,
+    )
     swebench_task = SWEbenchBenchmark(
-        instance_file=case_dir / "instance.json",
         grade_mode="none",
     ).load_task("pytest-dev__pytest-10051")
 
diff --git a/tests/evals/test_swebench.py b/tests/evals/test_swebench.py
index 43ce32a..489011f 100644
--- a/tests/evals/test_swebench.py
+++ b/tests/evals/test_swebench.py
@@ -6,6 +6,7 @@
 
 from yada.evals import AgentRunResult
 from yada.evals.benchmarks import SWEbenchBenchmark
+from yada.evals.benchmarks.swebench import _INSTANCE_PREFIX, _load_harness_instance
 
 
 def _source_repo(tmp_path: Path) -> tuple[Path, str]:
@@ -29,31 +30,65 @@ def _source_repo(tmp_path: Path) -> tuple[Path, str]:
     return source, head
 
 
-def _instance_file(tmp_path: Path, head: str) -> Path:
-    path = tmp_path / "instance.json"
-    path.write_text(
-        json.dumps(
-            {
-                "instance_id": "owner__repo-1",
-                "repo": "owner/repo",
-                "base_commit": head,
-                "problem_statement": "Fix VALUE.",
-                "patch": "SECRET GOLD PATCH",
-                "test_patch": "SECRET TEST PATCH",
-                "FAIL_TO_PASS": "hidden test",
-            }
-        ),
-        encoding="utf-8",
+def _instance_row(head: str) -> dict[str, str]:
+    return {
+        "instance_id": "owner__repo-1",
+        "repo": "owner/repo",
+        "base_commit": head,
+        "problem_statement": "Fix VALUE.",
+    }
+
+
+def test_swebench_metadata_comes_from_harness_and_stays_public(monkeypatch) -> None:
+    def fake_run(argv, **kwargs):
+        assert argv[0] == "python-for-swebench"
+        assert argv[-3:] == [
+            "princeton-nlp/SWE-bench_Verified",
+            "test",
+            "owner__repo-1",
+        ]
+        payload = {
+            "instance_id": "owner__repo-1",
+            "repo": "owner/repo",
+            "base_commit": "abc123",
+            "problem_statement": "Fix VALUE.",
+            "patch": "SECRET GOLD PATCH",
+        }
+        return subprocess.CompletedProcess(
+            argv,
+            0,
+            _INSTANCE_PREFIX + json.dumps(payload) + "\n",
+            "",
+        )
+
+    monkeypatch.setattr(
+        "yada.evals.benchmarks.swebench.subprocess.run",
+        fake_run,
+    )
+
+    row = _load_harness_instance(
+        "python-for-swebench",
+        "princeton-nlp/SWE-bench_Verified",
+        "test",
+        "owner__repo-1",
     )
-    return path
 
+    assert row["problem_statement"] == "Fix VALUE."
+    assert "patch" not in row
 
-def test_swebench_task_hides_gold_data_and_writes_prediction(tmp_path: Path) -> None:
+
+def test_swebench_loads_task_with_harness_and_writes_prediction(
+    tmp_path: Path, monkeypatch
+) -> None:
     source, head = _source_repo(tmp_path)
+    monkeypatch.setattr(
+        "yada.evals.benchmarks.swebench._load_harness_instance",
+        lambda *_: _instance_row(head),
+    )
     benchmark = SWEbenchBenchmark(
-        instance_file=_instance_file(tmp_path, head),
         source_workspace=source,
         grade_mode="none",
+        harness_python="python-for-swebench",
     )
 
     task = benchmark.load_task("owner__repo-1")
@@ -81,8 +116,11 @@ def test_swebench_task_hides_gold_data_and_writes_prediction(tmp_path: Path) ->
 
 def test_swebench_parses_official_run_report(tmp_path: Path, monkeypatch) -> None:
     source, head = _source_repo(tmp_path)
+    monkeypatch.setattr(
+        "yada.evals.benchmarks.swebench._load_harness_instance",
+        lambda *_: _instance_row(head),
+    )
     benchmark = SWEbenchBenchmark(
-        instance_file=_instance_file(tmp_path, head),
         source_workspace=source,
         grade_mode="docker",
         harness_python="python-for-swebench",

From 4d5a6e10eb760dd6a9758cb69aca5134eccd24ba Mon Sep 17 00:00:00 2001
From: Gen TANG <gen.tang86@gmail.com>
Date: Mon, 3 Aug 2026 23:25:39 +0800
Subject: [PATCH 2/3] upgrade eval procedure

---
 .gitignore                            |   1 +
 README-cn.md                          |   7 +
 README.md                             |   7 +
 benchmarks/README.md                  |   5 +
 docs/cli-reference.md                 |  78 ++++-
 docs/configuration.md                 |  83 ++++-
 docs/dev/architecture.md              |  49 ++-
 docs/evaluation.md                    | 423 ++++++++++++++++++++++++
 src/yada/environments/__init__.py     |  15 +-
 src/yada/environments/commands.py     | 254 +++++++++++++++
 src/yada/evals/agents/yada.py         |  76 ++++-
 src/yada/evals/benchmarks/swebench.py | 441 +++++++++++++++++++++++---
 src/yada/evals/cli.py                 |   5 +-
 src/yada/tools/base.py                |   2 +
 src/yada/tools/command.py             |  52 +--
 src/yada/tools/runner.py              |  14 +-
 tests/environments/test_commands.py   |  73 +++++
 tests/evals/test_eval_cli.py          |   4 +-
 tests/evals/test_local_benchmark.py   |   4 +
 tests/evals/test_swebench.py          | 219 ++++++++++++-
 20 files changed, 1684 insertions(+), 128 deletions(-)
 create mode 100644 docs/evaluation.md
 create mode 100644 src/yada/environments/commands.py
 create mode 100644 tests/environments/test_commands.py

diff --git a/.gitignore b/.gitignore
index dbde422..c2d3459 100644
--- a/.gitignore
+++ b/.gitignore
@@ -9,3 +9,4 @@ dist/
 *.egg-info/
 .yada/
 eval-results/
+logs/
diff --git a/README-cn.md b/README-cn.md
index 72f2c70..fc45408 100644
--- a/README-cn.md
+++ b/README-cn.md
@@ -20,6 +20,11 @@ Yada 目前处于 Alpha 阶段。仓库已经测试本地 Agent 闭环，但尚
 - [uv](https://docs.astral.sh/uv/)（推荐）
 - DeepSeek API Key
 
+直接运行 `yada` 和执行 `yada eval --case` 不需要 Docker；官方
+`yada eval --swebench` 评测要求安装并启动仍受维护的 Docker Desktop 或
+Docker Engine。安装、验证、版本策略与资源要求见
+[Docker requirements](docs/configuration.md#docker-requirements)。
+
 ## 快速开始
 
 ```bash
@@ -53,6 +58,8 @@ Trace 保存在目标仓库的 `.yada/runs/` 目录下。
 - [配置](docs/configuration.md)：其他安装方式、API Key、模型参数、命令策略和
   Trace Level。
 - [CLI 参考](docs/cli-reference.md)：`yada`、`yada eval` 和 `yada-trace`。
+- [评测生命周期](docs/evaluation.md)：`--case` 与 `--swebench` 会加载、修改、
+  缓存、评分和写入哪些内容。
 - [贡献指南](CONTRIBUTING.md)：开发环境、验证命令和基于 Rebase 的 PR 流程。
 - [架构](docs/dev/architecture.md)：Agent 循环、工具、Patch 事务、评测边界与
   安全不变量。
diff --git a/README.md b/README.md
index 8f5802e..a51e15f 100644
--- a/README.md
+++ b/README.md
@@ -20,6 +20,11 @@ project does not claim a comparative benchmark result yet.
 - [uv](https://docs.astral.sh/uv/) (recommended)
 - A DeepSeek API key
 
+Docker is optional for direct runs and `yada eval --case`. Official
+`yada eval --swebench` runs require a running, maintained Docker Desktop or
+Docker Engine release; see [Docker requirements](docs/configuration.md#docker-requirements)
+for installation, verification, version policy, and resource guidance.
+
 ## Quick start
 
 ```bash
@@ -54,6 +59,8 @@ complete OS sandbox; use a disposable VM or container for unfamiliar projects.
 - [Configuration](docs/configuration.md): installation alternatives, API key,
   model settings, command policy, and trace levels.
 - [CLI reference](docs/cli-reference.md): `yada`, `yada eval`, and `yada-trace`.
+- [Evaluation lifecycle](docs/evaluation.md): what `--case` and `--swebench`
+  load, mutate, cache, grade, and write.
 - [Contributing](CONTRIBUTING.md): development setup, validation, and the rebase
   pull-request workflow.
 - [Architecture](docs/dev/architecture.md): agent loop, tools, patch transaction,
diff --git a/benchmarks/README.md b/benchmarks/README.md
index abd9648..1bd8bfc 100644
--- a/benchmarks/README.md
+++ b/benchmarks/README.md
@@ -23,3 +23,8 @@ Cases under `swebench_verified/` use the canonical public problem statement and
 base commit. Their local grader is intended for fast development feedback. A
 published SWE-bench score must still come from the official Docker Harness.
 
+This local path is intentional rather than a duplicate of `--swebench`: it
+keeps prompt and tool regression tests fast, inspectable, and usable without
+Docker. Use `--swebench INSTANCE_ID` when environment parity and an
+official-compatible verdict matter; Yada then requires Docker before model
+inference and separates the Agent command container from the final grader.
diff --git a/docs/cli-reference.md b/docs/cli-reference.md
index 1612d46..2e32401 100644
--- a/docs/cli-reference.md
+++ b/docs/cli-reference.md
@@ -55,6 +55,8 @@ keyboard interruption. Argument errors also use the standard argparse status `2`
 `yada eval` runs an agent behind a benchmark-neutral adapter and persists a
 schema-versioned result even for ordinary adapter failures. Unlike a direct
 `yada` run, an evaluation always has a task adapter and a grader.
+See [Evaluation lifecycle](evaluation.md) for the complete load, workspace,
+agent, grading, cache, and artifact sequence.
 
 ### Choose a run mode
 
@@ -68,25 +70,40 @@ flowchart TD
     source -->|"Checked-in case directory"| case["--case PATH"]
     source -->|"Official SWE-bench Verified"| swe["--swebench INSTANCE_ID"]
 
-    case --> local_adapter["Local adapter\ncase.json + declared grader"]
+    case --> local_adapter["Local adapter\ncase.json + host/uv environment"]
     local_adapter --> local_result["Development/private result\nnot an official SWE-bench score"]
 
-    swe --> official["Generate patch, then run\nofficial SWE-bench Docker Harness"]
+    swe --> agent_container["Public instance image\nAgent commands in Docker"]
+    agent_container --> official["Separate official Harness\ngrading container"]
     official --> official_result["Official-compatible SWE-bench result"]
 ```
 
 `yada eval` deliberately exposes only two task selectors:
 
-| Invocation | Meaning | Grader |
-| --- | --- | --- |
-| `--case PATH` | Run a portable local case. A directory resolves to `PATH/case.json`; a manifest file may also be passed directly. | The command declared by the case manifest |
-| `--swebench INSTANCE_ID` | Load one SWE-bench Verified task, generate a prediction, and delegate grading to the official Docker Harness. | Official SWE-bench Docker Harness |
+| Invocation | Meaning | Native Yada command environment | Grader |
+| --- | --- | --- | --- |
+| `--case PATH` | Run a portable local case. A directory resolves to `PATH/case.json`; a manifest file may also be passed directly. | Host/uv environment declared by the case; Docker is not required by Yada | Command declared by the case manifest |
+| `--swebench INSTANCE_ID` | Load one SWE-bench Verified task and generate an official-compatible prediction. | Public SWE-bench instance image in Docker | Separate official SWE-bench Harness container |
 
 There is no `--local`, `--benchmark`, or separate `--manifest` selector. Pass a
 custom manifest directly to `--case`. A case is called local because its
 manifest controls preparation and grading; it may still fetch a Git repository
 and call the DeepSeek API. Each invocation runs exactly one task.
 
+#### Why `--case` is not redundant
+
+`--case` is the small, no-Docker evaluation path for prompt/tool regression,
+private repositories, and custom test subsets. It is fast, inspectable, and can
+run from fully local inputs. Its verdict belongs to the manifest and must not be
+published as an official SWE-bench score.
+
+`--swebench` is the comparability path. It fixes the dataset and Harness policy,
+requires a running Docker daemon before the first model request, runs native
+Yada commands in the public instance image, and grades the patch in an
+independent container that receives the evaluation tests. Collapsing these
+selectors would either force Docker onto ordinary Yada development or make
+local and official-looking results dangerously easy to confuse.
+
 ### Checked-in development case (`--case`)
 
 Use a checked-in case for fast development and regression testing:
@@ -173,16 +190,23 @@ Portable Git checkouts are cached under `--cache-dir` (default
 
 ### Official SWE-bench evaluation (`--swebench`)
 
-The official SWE-bench Harness is the single source for both public instance
-metadata and Docker grading. `--swebench` does not read the checked-in local
-case's `instance.json`, so there are no local and online copies to keep in sync.
-Gold patches, test patches, and hidden test IDs do not cross the public task
-boundary. Yada fetches the exact base commit, prepares a clean candidate
-workspace, and generates the patch that the Harness grades.
+The official SWE-bench Harness is the single source for public instance
+metadata, the public instance image, and Docker grading. `--swebench` does not
+read the checked-in local case's `instance.json`, so there are no local and
+online copies to keep in sync. Gold patches, test patches, and hidden test IDs
+do not cross the public task boundary.
+
+Install and start a maintained Docker Desktop or Docker Engine release, then
+verify both `docker --version` and `docker info`. SWE-bench 4.1.0 does not state
+an exact minimum Docker version, so Yada checks working client/daemon behavior
+instead of enforcing an invented version number. Legacy Docker Toolbox and
+obsolete standalone clients are unsupported. See
+[Docker requirements](configuration.md#docker-requirements) for installation,
+platform checks, and resource guidance.
 
-Install and start Docker. The official `swebench` package is published on PyPI,
-so no SWE-bench Git clone or separately managed virtual environment is needed.
-Use uv's cached dependency overlay to run the pinned Harness version:
+The official `swebench` package is published on PyPI, so no SWE-bench Git clone
+or separately managed virtual environment is needed. Use uv's cached dependency
+overlay to run the pinned Harness version:
 
 ```bash
 uv run --with 'swebench==4.1.0' yada eval \
@@ -196,6 +220,24 @@ The first run downloads the Harness and its dependencies into uv's cache;
 subsequent runs reuse them. This keeps Yada's core dependency-free and leaves
 the project environment unchanged. If `swebench==4.1.0` is already installed in
 Yada's active environment, the shorter `uv run yada eval ...` form also works.
+Yada checks the Docker CLI and daemon before dataset loading or model inference.
+
+The Harness first pulls or builds the public instance image. Yada exports its
+prepared `/testbed` as the mutable artifact workspace. For native
+`--agent yada`, file tools edit that workspace on the host while `run_command`
+executes against it in an ephemeral container made from the same image. Final
+grading uses another Harness-owned container; only that container receives the
+evaluation patch and script.
+
+During image preparation, Harness stdout and stderr are streamed to the
+terminal and flushed live to `swebench-agent-image.stdout.log` and
+`swebench-agent-image.stderr.log` in the artifact directory. A heartbeat is
+printed after each 30-second interval without output. This happens before the
+Agent phase, so `yada-trace.jsonl` is not the place to diagnose an image pull or
+build. Official grading is streamed the same way to `swebench.stdout.log` and
+`swebench.stderr.log`. See
+[Evaluation lifecycle](evaluation.md#official-swe-bench---swebench-instance_id)
+for the exact cache, workspace, container, and grading sequence.
 
 The adapter intentionally fixes the current public evaluation policy instead of
 exposing Harness internals as Yada CLI flags:
@@ -205,6 +247,7 @@ exposing Harness internals as Yada CLI flags:
 | Dataset | `princeton-nlp/SWE-bench_Verified` |
 | Split | `test` |
 | Grading | Official Docker Harness |
+| Native Agent commands | Public instance image in an ephemeral container |
 | Image cache | Keep environment images |
 | Grading timeout | 1800 seconds |
 | Docker namespace | `swebench`; automatically disabled on Apple Silicon so images build locally |
@@ -228,7 +271,10 @@ uv run yada eval \
 
 The template supports `{task}`, `{task_file}`, `{workspace}`, `{output_patch}`,
 and `{run_dir}`. If the command does not write `{output_patch}`, Yada collects
-the complete Git diff, including untracked files.
+the complete Git diff, including untracked files. For `--swebench`, this
+external command is host-managed and is responsible for entering its own
+container; Yada's automatic Agent command container applies only to the native
+`--agent yada` adapter.
 
 ### Common evaluation options
 
diff --git a/docs/configuration.md b/docs/configuration.md
index 7b7186c..ad7855b 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -127,7 +127,88 @@ WORKSPACE/.yada/runs/<task>__<system-local-time>.jsonl
 Use `--trace PATH` to choose an exact location. Default names use local time at
 minute precision and append `(1)`, `(2)`, and so on when a name already exists.
 
-## Docker
+## Docker requirements
+
+Docker is optional for direct `yada` runs and `yada eval --case`. It is required
+only for official `yada eval --swebench` evaluation, where Yada uses Docker for
+both the public Agent command environment and the independent Harness grader.
+
+### Supported installation and version policy
+
+Use a currently maintained release of one of:
+
+- [Docker Desktop](https://docs.docker.com/desktop/) on macOS or Windows; or
+- [Docker Engine](https://docs.docker.com/engine/install/) on Linux.
+
+Legacy Docker Toolbox installations and obsolete standalone clients are not
+supported. SWE-bench 4.1.0 does not publish an exact minimum Docker
+Desktop/Engine version, so Yada does not claim or enforce an arbitrary semantic
+version floor. Compatibility is functional: the `docker` CLI must execute
+natively on the host, connect to a running daemon without `sudo`, pull or build
+Linux images, bind-mount a workspace, and run containers for the required
+platform. Upgrade to the latest release supported by the host OS when any of
+these checks fail.
+
+Yada pins the Python Harness separately at `swebench==4.1.0` through uv's
+temporary dependency overlay. Installing Docker does not install the Harness,
+and installing the Harness does not install or start Docker.
+
+### Install and verify
+
+On macOS, install the correct Docker Desktop build for the machine's CPU from
+the [official Mac installation guide](https://docs.docker.com/desktop/setup/install/mac-install/).
+With Homebrew, the equivalent installation is:
+
+```bash
+brew install --cask docker
+open -a Docker
+```
+
+Wait until Docker Desktop reports that its engine is running. On Linux, follow
+Docker's distribution-specific Engine installation and
+[post-installation guide](https://docs.docker.com/engine/install/linux-postinstall/);
+the current user must be able to run Docker without `sudo`.
+
+Verify the complete client/daemon path before starting an evaluation:
+
+```bash
+command -v docker
+file "$(command -v docker)"
+docker --version
+docker info --format '{{.ServerVersion}}'
+docker run --rm hello-world
+```
+
+On an Apple Silicon Mac, also verify the Linux `amd64` path used by many
+SWE-bench images:
+
+```bash
+docker run --rm --platform linux/amd64 hello-world
+```
+
+`docker --version` must exit normally. A crash, architecture error, or legacy
+client is an installation problem even if a file named `docker` exists on
+`PATH`. `docker info` must print a server version; a client-only installation is
+not sufficient.
+
+### SWE-bench resources and architecture
+
+The upstream SWE-bench guidance recommends an `x86_64` machine with 8 CPU
+cores, 16 GB RAM, and roughly 120 GB of free Docker storage. These are capacity
+recommendations rather than Yada parser requirements, but image preparation can
+fail or exhaust disk below them. Configure Docker Desktop's virtual machine
+resources accordingly.
+
+Apple Silicon and other `arm64` hosts are experimental in SWE-bench. Yada
+automatically disables the prebuilt `swebench` image namespace on Apple Silicon
+and builds images locally, which is substantially slower and still requires
+Docker to support the requested Linux platform.
+
+See the [SWE-bench Docker setup guide](https://www.swebench.com/SWE-bench/guides/docker_setup/)
+and [Evaluation lifecycle](evaluation.md#official-swe-bench---swebench-instance_id)
+for the resource and container boundaries.
+
+### Run Yada itself in Docker
 
 The included image isolates the mounted filesystem but does not block container
 network access:
diff --git a/docs/dev/architecture.md b/docs/dev/architecture.md
index abad9d9..f656d6c 100644
--- a/docs/dev/architecture.md
+++ b/docs/dev/architecture.md
@@ -34,6 +34,7 @@ run/cli.py
     │   │   └── tools/runner.py
     │   │       ├── environments/workspace.py
     │   │       ├── environments/approval.py
+    │   │       ├── environments/commands.py
     │   │       └── tools/{search,read,replace,patch,command,finish}.py
     │   ├── models/base.py ← models/deepseek.py
     │   └── traces/jsonl.py ← traces/report.py
@@ -46,7 +47,7 @@ run/cli.py
 - `run`: parses public configuration and assembles a direct run.
 - `agents`: coordinates model turns, planning policy, and tool execution.
 - `models`: defines the completion boundary and DeepSeek transport.
-- `environments`: owns workspace containment and command approval.
+- `environments`: owns workspace containment, command approval, and execution backends.
 - `tools`: implements one handler per model-facing tool.
 - `traces`: writes append-only events and builds source-located reports.
 - `evals`: composes benchmark preparation/grading with interchangeable agents.
@@ -137,8 +138,13 @@ no file changes until the generated multi-file patch passes validation.
 
 `run_command` accepts argv, not a shell string. It checks the executable
 allowlist, disables shell `-c`, restricts Git to read-only subcommands, applies
-the configured approval policy, sanitizes secret-looking environment variables,
-and separately records stdout, stderr, exit code, timeout, and duration.
+the configured approval policy, and then delegates execution to a small backend
+interface. Direct runs and local cases use the host backend. Official
+SWE-bench runs use a persistent ephemeral container made from the task's public
+instance image, with the mutable workspace mounted at `/testbed`. Neither
+backend adds a Python runtime dependency to Yada's core. Both sanitize
+secret-looking environment variables and return stdout, stderr, exit code,
+timeout, and duration through the same tool result.
 
 The model labels a command as `inspect`, `test`, or `build`. Only a successful
 `test` or `build` verifies the current workspace revision. `finish` rejects the
@@ -191,11 +197,29 @@ external agents. Local and SWE-bench benchmark adapters own provenance, hidden
 grading inputs, and the final verdict. Gold patches and hidden tests never cross
 the public task boundary.
 
+The official SWE-bench adapter deliberately uses two Docker lifetimes:
+
+```text
+public instance image
+        ├── export /testbed ── host artifact workspace ── file tools
+        └── ephemeral Agent container ─────────────────── run_command
+
+collected model patch
+        └── independent Harness container + evaluation patch ── verdict
+```
+
+The Agent container and grader container are never the same container. The
+first contains only public repository setup; the second is created after the
+agent stops and is the only environment that receives evaluation tests. A
+Docker preflight happens during official task loading, before model inference.
+Local cases do not use this backend unless their own manifest commands do so.
+
 Fair comparisons use the same instance, base commit, public prompt, model
 endpoint, token/cost and wall-time budgets, network policy, and official grader.
 Resolve rate is the primary outcome; tokens, steps, duration, and cost are
 diagnostics. See [CLI reference: `yada eval`](../cli-reference.md#yada-eval) for
-manifest and command examples.
+command examples and [Evaluation lifecycle](../evaluation.md) for the exact
+load, workspace, grading, cache, and artifact sequence.
 
 ## Core invariants
 
@@ -208,13 +232,18 @@ manifest and command examples.
 7. `finish` requires verification of the latest revision.
 8. Trace events are append-only and self-correlating.
 9. Benchmark grading happens outside the agent's tool boundary.
+10. Hidden grading inputs never enter the official Agent command container.
 
 ## Security boundary
 
 Yada's path checks, argv policy, environment sanitization, and approval gate
-reduce accidental damage; they do not isolate arbitrary repository code. Test
-runners, interpreters, build systems, dependencies, and scripts can access the
-host permissions available to the process.
-
-Production evaluation should put each candidate workspace in a disposable
-container or VM and run hidden grading outside the agent environment.
+reduce accidental damage. Direct runs and local cases still do not isolate
+arbitrary repository code: test runners, interpreters, build systems,
+dependencies, and scripts can access the host permissions available to the
+process.
+
+Native official SWE-bench commands run in a disposable container and hidden
+grading runs in a second one. This reduces host-environment drift and prevents
+the agent from observing grading inputs, but a Docker daemon and bind-mounted
+workspace remain privileged infrastructure rather than a complete security
+boundary.
diff --git a/docs/evaluation.md b/docs/evaluation.md
new file mode 100644
index 0000000..f452c96
--- /dev/null
+++ b/docs/evaluation.md
@@ -0,0 +1,423 @@
+# Evaluation lifecycle
+
+This document explains what Yada does after either public evaluation command:
+
+```bash
+uv run yada eval --case PATH [OPTIONS]
+uv run --with 'swebench==4.1.0' yada eval --swebench INSTANCE_ID [OPTIONS]
+```
+
+For option syntax, see the [CLI reference](cli-reference.md). For the internal
+adapter contracts, see the [architecture](dev/architecture.md). This document
+focuses on observable execution: inputs, network access, workspaces, grading,
+caches, and output files.
+
+## Common pipeline
+
+Both selectors run one task through the same benchmark-neutral pipeline:
+
+```mermaid
+sequenceDiagram
+    participant CLI as yada eval
+    participant Runner as EvalRunner
+    participant Benchmark as Benchmark adapter
+    participant Agent as Agent adapter
+    participant Grader as Benchmark grader
+    participant Disk as Result and artifacts
+
+    CLI->>Runner: selector, agent, budgets, output paths
+    Runner->>Benchmark: load_task(instance)
+    Benchmark-->>Runner: public issue and safe metadata
+    Runner->>Benchmark: prepare(task, artifact_dir)
+    Benchmark-->>Runner: isolated candidate workspace
+    Runner->>Agent: run(task, workspace, budgets)
+    Agent-->>Runner: Git patch, status, usage, trace
+    Runner->>Grader: grade(candidate patch)
+    Grader-->>Runner: resolved, unresolved, skipped, or error
+    Runner->>Disk: atomically write result JSON
+```
+
+Before this pipeline starts, the CLI:
+
+1. requires exactly one of `--case` or `--swebench`;
+2. resolves the result and artifact paths;
+3. creates a run ID containing UTC time and a random suffix;
+4. configures the native Yada or external command agent; and
+5. requires `DEEPSEEK_API_KEY` when the native Yada agent is selected.
+
+The default result name uses system-local time at minute precision. A collision
+adds `(1)`, `(2)`, and so on. The result and artifact directory always receive
+the same suffix.
+
+Once `EvalRunner` starts, it creates the artifact directory and attempts every
+phase in order: load, prepare, agent, then grade. Ordinary adapter exceptions are
+recorded in the result JSON instead of losing the whole run. Argument errors and
+a missing API key happen before the runner starts and therefore do not create a
+result.
+
+The grader owns the final verdict. The agent's `finish` call and summary do not
+make an evaluation `resolved`.
+
+## Why Yada keeps both selectors
+
+`--case` and `--swebench` are not two spellings for the same operation. They
+preserve two different contracts:
+
+| | `--case PATH` | `--swebench INSTANCE_ID` |
+| --- | --- | --- |
+| Primary use | Fast development, regression cases, private tasks, and custom benchmarks | Comparable SWE-bench Verified runs |
+| Task and grader authority | The checked-in or local `case.json` | The pinned official Harness and dataset |
+| Native Yada commands | Host/uv environment declared by the case | Public SWE-bench instance image in Docker |
+| Final grading | Manifest command; may be partial or project-specific | A separate official Harness container |
+| Docker required by Yada | No | Yes, checked before any model request |
+| Result meaning | Local development result | Official-compatible SWE-bench verdict |
+
+Removing `--case` would make every quick tool or prompt regression depend on
+Docker, a network dataset, and the full official grader. Treating a local case
+as an official run would be worse: its dependencies or test selection can
+differ and produce a false pass. Keeping the boundary explicit lets Yada remain
+small and usable without Docker while reserving comparable claims for
+`--swebench`.
+
+Without Docker, use direct `yada ...` for normal repository work or
+`yada eval --case PATH` for a reproducible local evaluation. A case manifest
+may choose to invoke Docker itself, but Docker is not a Yada requirement for
+this path.
+
+## Local case: `--case PATH`
+
+Example:
+
+```bash
+uv run yada eval \
+  --case benchmarks/swebench_verified/pytest-10051 \
+  --yes \
+  --trace-level debug
+```
+
+`PATH` may be a case directory or a manifest file. A directory resolves to
+`PATH/case.json`.
+
+### 1. Load the case recipe
+
+The local adapter reads `case.json`. The manifest must provide exactly one
+public task source:
+
+- `problem_statement`: inline issue text;
+- `task_file`: a UTF-8 file relative to the manifest; or
+- `instance_file`: a local public-instance record.
+
+When an `instance_file` is used, only public metadata such as repository, base
+commit, version, and dataset provenance enters the task. Gold patches and test
+patches are not included in the task passed to the agent.
+
+The checked-in `pytest-10051` case uses its bundled `instance.json`. That file is
+part of the local development recipe; it is not used by official `--swebench`
+runs.
+
+### 2. Prepare the source and environment
+
+The manifest's `workspace` is either a local path or a pinned Git recipe.
+
+For a Git recipe, Yada stores a reusable source checkout under:
+
+```text
+.yada/cache/evals/<cache_key>/repo
+```
+
+On the first run it initializes that cache, fetches only the declared base
+commit, and checks it out detached. On later runs it verifies the remote URL,
+exact `HEAD`, and clean worktree before reuse. The cache is a source template;
+the agent does not edit it directly.
+
+If the case declares a uv environment, Yada runs `uv sync --locked` in the
+case's environment project. This can create or update a persistent `.venv`
+inside the case directory. For the checked-in pytest case, that is why
+`benchmarks/swebench_verified/pytest-10051/.venv` exists after the first run.
+The sync command is invoked on every evaluation so the lock remains the source
+of truth, while uv reuses the existing environment and package cache when they
+already match. For `pytest-10051`, the manifest requests Python 3.9.10 and then
+installs the cached source into that environment with the declared legacy
+editable setup. The persistent `.venv` belongs to the case recipe, not to the
+per-run candidate workspace; seeing it change is therefore expected.
+
+With the default `workspace_mode=copy`, Yada then copies the clean source into a
+new candidate workspace inside this run's artifact directory. Git metadata is
+preserved, while `.yada`, `__pycache__`, and `.pytest_cache` are excluded. The
+manifest can explicitly request `in_place`, but reproducible checked-in cases
+should normally use `copy`.
+
+For `pytest-10051`, the effective layout is:
+
+```text
+case.json and grader.py
+        │
+        ├── persistent source cache at the exact pytest base commit
+        ├── persistent locked Python 3.9 environment in the case directory
+        └── fresh eval-results/<run>.artifacts/workspace for the agent
+```
+
+### 3. Run the agent
+
+The native Yada adapter receives only the public problem statement and candidate
+workspace. It supplies the case environment through `PATH`, `VIRTUAL_ENV`, and
+configured `PYTHONPATH` entries.
+
+This path does not attach Yada's Docker command backend. `run_command` executes
+on the host in the case's prepared uv environment, with the candidate workspace
+as its repository root. `--yes` skips interactive approval for those
+model-requested commands; it does not skip environment preparation or grading.
+
+During the run:
+
+- all model-facing file operations stay inside the candidate workspace;
+- edits are SHA-bound and applied through checked patch transactions;
+- `run_command` uses the configured approval policy;
+- `finish` requires a successful test or build after the latest edit; and
+- events are appended to `yada-trace.jsonl`.
+
+After the agent stops, Yada collects all tracked and untracked Git changes into
+one patch relative to `HEAD`.
+
+### 4. Run the case grader
+
+Yada writes the collected patch to `agent.patch`, substitutes manifest
+placeholders such as `{workspace}`, `{patch}`, `{run_dir}`, and `{run_id}`, and
+runs the declared grader from the case directory.
+
+For the checked-in pytest case, the grader:
+
+1. verifies that the candidate remains at the expected base commit;
+2. copies the candidate to a temporary grading workspace;
+3. applies the case's test patch only inside that grading copy; and
+4. runs one FAIL_TO_PASS plus fifteen PASS_TO_PASS tests.
+
+The candidate workspace given to the agent never contains the test patch.
+
+Local grader exit codes map to verdicts as follows:
+
+| Grader exit | Evaluation status |
+| --- | --- |
+| `0` | `resolved` |
+| `1` | `unresolved` |
+| anything else or timeout | `error` |
+| no grader in manifest | `skipped` |
+
+This verdict is a local development result, not an official SWE-bench score.
+
+## Official SWE-bench: `--swebench INSTANCE_ID`
+
+Example:
+
+```bash
+uv run --with 'swebench==4.1.0' yada eval \
+  --swebench pytest-dev__pytest-10051 \
+  --yes \
+  --trace-level debug
+```
+
+Docker must already be running. Yada checks both the Docker CLI and daemon
+before loading the dataset or making a model request, so a missing Docker setup
+does not spend model tokens. The `--with` option asks uv to make the pinned
+official Harness available to this process. It does not add `swebench` or the
+Docker Python client to Yada's project dependencies and does not require a
+SWE-bench Git clone. The first run downloads the package into uv's cache; later
+runs reuse it.
+
+`uv --with` dependency resolution happens before the `yada` process starts.
+Consequently, an `Installed N packages` message can appear before Yada checks
+Docker, and uv may spend time populating its cache even when the later Docker
+preflight fails. That work is package setup, not a model request, and still uses
+no model tokens.
+
+Use a maintained Docker Desktop or Docker Engine release. SWE-bench 4.1.0 does
+not publish an exact minimum Docker version, so Yada requires functional client,
+daemon, image, platform, mount, and container support rather than checking a
+version string. Legacy Docker Toolbox and obsolete standalone clients are not
+supported. Follow [Docker requirements](configuration.md#docker-requirements)
+to install and validate Docker before running this command.
+
+### 1. Check Docker and load one official public task
+
+The adapter invokes the official Harness environment to read
+`pytest-dev__pytest-10051` from:
+
+```text
+dataset: princeton-nlp/SWE-bench_Verified
+split:   test
+```
+
+The Harness may download or reuse the Hugging Face dataset cache. Before the
+record crosses into Yada, it is reduced to a fixed public-field allowlist:
+
+- instance ID;
+- repository and base commit;
+- public problem statement;
+- version, difficulty, dates, and dataset provenance when available.
+
+Gold patches, test patches, FAIL_TO_PASS IDs, and PASS_TO_PASS IDs are not
+returned to the agent process.
+
+### 2. Prepare the public instance image and candidate workspace
+
+Through the pinned Harness environment, Yada derives the instance's official
+public image name. It pulls that image when available or asks the Harness to
+build it locally. This instance image contains the repository and its public
+setup: installed dependencies, generated source files, and the configured
+`testbed` environment. It does not contain the evaluation test patch or the
+Harness evaluation script.
+
+Image preparation is streamed rather than buffered until completion. Before it
+starts, Yada prints the two artifact log paths. Harness stdout and stderr then
+appear in the terminal with `[swebench image stdout]` and
+`[swebench image stderr]` prefixes while the same raw streams are flushed to:
+
+```text
+swebench-agent-image.stdout.log
+swebench-agent-image.stderr.log
+```
+
+If the Harness or Docker produces no line for 30 seconds, Yada prints a
+`still running` heartbeat with elapsed time. The log files are created at the
+start and retain partial output after a preparation failure or timeout. This
+stage is before the Agent phase, so `yada-trace.jsonl` may not exist yet and no
+model tokens have been used. On Apple Silicon the no-namespace policy commonly
+means a local image build rather than a registry pull; package installation
+inside that build can be the longest quiet-looking stage.
+
+Yada exports the image's `/testbed` to `<artifact_dir>/workspace`, preserving
+`.git`, and verifies that the dataset base commit is an ancestor of its `HEAD`.
+`HEAD` can be a Harness setup commit after the base commit; requiring equality
+would discard legitimate generated setup files. The mutable exported workspace
+is the source of the eventual model patch. It does not contain the future human
+solution. Yada prints messages before this export and after the workspace is
+ready, so a large `docker cp` is distinguishable from image preparation.
+
+### 3. Run Yada and collect the prediction
+
+For the native Yada agent, file tools read and edit the exported host workspace.
+Each `run_command` starts or reuses an ephemeral Agent container from the same
+public instance image, bind-mounts that workspace at `/testbed`, activates the
+image's `testbed` environment, and runs there. The container is removed when
+the agent stops. This prevents host Python state from creating a false pass
+while keeping file application simple and inspectable.
+
+The Agent container has no test patch or evaluation script. It is distinct from
+the grading container created later. An external `--agent command` process is
+still launched by its own host command; that adapter is responsible for its own
+execution isolation.
+
+When the agent stops, Yada collects the complete Git patch and writes
+`predictions.jsonl` with exactly:
+
+- `instance_id`;
+- `model_name_or_path`; and
+- `model_patch`.
+
+### 4. Grade in a separate official Harness container
+
+Yada invokes `swebench.harness.run_evaluation` with the built-in policy:
+
+| Harness setting | Value |
+| --- | --- |
+| Dataset and split | `princeton-nlp/SWE-bench_Verified`, `test` |
+| Instances | the single requested ID |
+| Workers | `1` |
+| Test timeout | `1800` seconds |
+| Image cache level | `env` |
+| Clean existing cache | `false` |
+| Image namespace | `swebench`, or none on Apple Silicon |
+
+The Harness starts a new container independently of the Agent container,
+applies only the collected model patch, injects its evaluation test patch and
+script, and produces the official run report. Hidden grading inputs therefore
+never enter the Agent container or model context. On Apple Silicon, Yada
+automatically disables the prebuilt-image namespace so the Harness builds
+compatible images locally. That path is experimental and can consume
+substantial time and disk space.
+
+Official grader stdout and stderr are also streamed to the terminal and flushed
+live to `swebench.stdout.log` and `swebench.stderr.log`. The same 30-second
+no-output heartbeat applies, so a long test does not look like a hung Yada
+process and partial Harness diagnostics survive a timeout.
+
+Yada maps membership in the official report's `resolved_ids` or
+`unresolved_ids` to its final verdict. A missing report, incomplete instance, or
+Harness failure becomes `error` rather than being guessed as a model failure.
+
+Across one official run there are three distinct container lifetimes:
+
+| Container | Purpose | Lifetime |
+| --- | --- | --- |
+| Export container | A stopped container used only to copy `/testbed` into the artifact workspace | Created and removed during preparation |
+| Agent container | Executes native Yada `run_command` calls against the bind-mounted mutable workspace | Created lazily, reused during the Agent phase, then removed |
+| Grader container | Receives the model patch plus official evaluation inputs and produces the verdict | Owned and removed according to the Harness policy |
+
+Docker or the Harness may also create temporary builder containers while making
+an uncached image. Those are image-build implementation details, not an
+additional place where the agent edits code.
+
+## Files written by a run
+
+With default paths, one evaluation produces a result and a sibling artifact
+directory:
+
+```text
+eval-results/
+├── <task>__<local-time>.json
+└── <task>__<local-time>.artifacts/
+    ├── workspace/
+    └── yada-trace.jsonl
+```
+
+Additional files depend on the selector:
+
+| File | `--case` | `--swebench` |
+| --- | --- | --- |
+| `workspace/` | Fresh copy unless manifest requests `in_place` | `/testbed` exported from the public instance image |
+| `yada-trace.jsonl` | Native Yada execution trace | Created when the Agent phase begins; it does not cover earlier image preparation |
+| `agent.patch` | Patch supplied to the local grader, when configured | Patch is stored inside `predictions.jsonl` |
+| `grader.stdout.log`, `grader.stderr.log` | Local grader output | — |
+| `predictions.jsonl` | — | Official Harness input |
+| `swebench-agent-image.stdout.log`, `swebench-agent-image.stderr.log` | — | Created immediately and updated live during public image preparation; partial logs survive failure |
+| `swebench.stdout.log`, `swebench.stderr.log` | — | Created and updated live during official grading; partial logs survive timeout |
+| `<model>.<run_id>.json` | — | Official Harness summary report |
+| Harness `logs/` and image build data | — | Created by the official Harness |
+
+The top-level result JSON records:
+
+- run, benchmark, instance, and agent identity;
+- start time and duration;
+- a hash and length of the public problem statement;
+- safe task provenance;
+- agent status, patch, steps, tokens, summary, and trace path;
+- grader verdict and diagnostics; and
+- any ordinary pipeline error.
+
+It is written atomically through a temporary file. CLI exit codes are `0` for
+`resolved`, `1` for `unresolved`, and `2` for errors, skipped grading, or other
+non-verdict outcomes.
+
+## Network and persistent state
+
+The two selectors have different side effects:
+
+| Resource | `--case` | `--swebench` |
+| --- | --- | --- |
+| DeepSeek API | Native Yada agent | Native Yada agent |
+| Git network | First pinned-source fetch, then case cache | Only when the Harness must build an uncached image |
+| Python packages | First case `uv sync`, then case `.venv` | First `uv --with`; image builds may also install task dependencies |
+| Dataset network | None when the case is fully local | Hugging Face dataset, then its cache |
+| Docker | Only if the case grader declares it | Required before the model runs; separate Agent and grader containers |
+| Persistent large data | Source cache and case `.venv`, when declared | uv/Hugging Face caches and Docker images |
+| Mutable candidate | Per-run artifact workspace | Per-run artifact workspace |
+
+`--yes` changes the native agent's repository-command approval policy. It does
+not disable task preparation, dependency resolution, Git fetching, or grading;
+those are benchmark-adapter operations outside the model-facing tool loop.
+
+Repository tests and benchmark graders execute code. Direct runs and `--case`
+commands use the host unless their case provides isolation. Native
+`--swebench` repository commands run in Docker, but Docker daemon access and
+host-mounted files remain security-sensitive. Review debug traces before
+publishing them.
diff --git a/src/yada/environments/__init__.py b/src/yada/environments/__init__.py
index 9726bf7..ae32378 100644
--- a/src/yada/environments/__init__.py
+++ b/src/yada/environments/__init__.py
@@ -1,4 +1,17 @@
 from yada.environments.approval import CommandApprover
+from yada.environments.commands import (
+    CommandExecutor,
+    CommandResult,
+    DockerCommandExecutor,
+    LocalCommandExecutor,
+)
 from yada.environments.workspace import Workspace
 
-__all__ = ["CommandApprover", "Workspace"]
+__all__ = [
+    "CommandApprover",
+    "CommandExecutor",
+    "CommandResult",
+    "DockerCommandExecutor",
+    "LocalCommandExecutor",
+    "Workspace",
+]
diff --git a/src/yada/environments/commands.py b/src/yada/environments/commands.py
new file mode 100644
index 0000000..0af6e4f
--- /dev/null
+++ b/src/yada/environments/commands.py
@@ -0,0 +1,254 @@
+"""Pluggable command execution without adding runtime dependencies."""
+
+from __future__ import annotations
+
+import os
+import shlex
+import subprocess
+import uuid
+from collections.abc import Mapping
+from dataclasses import dataclass
+from pathlib import Path, PurePosixPath
+from typing import Protocol
+
+from yada.exceptions import ToolError
+from yada.utils.text import timeout_text
+
+
+@dataclass(frozen=True)
+class CommandResult:
+    """Raw result returned by a local or container command backend."""
+
+    exit_code: int
+    stdout: str
+    stderr: str
+    timed_out: bool = False
+
+
+class CommandExecutor(Protocol):
+    """Execute validated argv inside one prepared workspace environment."""
+
+    name: str
+
+    def run(
+        self,
+        *,
+        argv: list[str],
+        workspace: Path,
+        cwd: Path,
+        environment: Mapping[str, str],
+        timeout_seconds: int,
+    ) -> CommandResult:
+        """Execute one command and return its captured result."""
+
+        ...
+
+    def close(self) -> None:
+        """Release resources owned by this executor."""
+
+        ...
+
+
+class LocalCommandExecutor:
+    """Run commands directly on the host using a sanitized environment."""
+
+    name = "local"
+
+    def run(
+        self,
+        *,
+        argv: list[str],
+        workspace: Path,
+        cwd: Path,
+        environment: Mapping[str, str],
+        timeout_seconds: int,
+    ) -> CommandResult:
+        env = _sanitized_values(os.environ)
+        env.update(_sanitized_values(environment))
+        env["YADA_WORKSPACE"] = str(workspace)
+        try:
+            result = subprocess.run(
+                argv,
+                cwd=cwd,
+                env=env,
+                text=True,
+                capture_output=True,
+                timeout=timeout_seconds,
+                check=False,
+            )
+            return CommandResult(result.returncode, result.stdout, result.stderr)
+        except subprocess.TimeoutExpired as exc:
+            return CommandResult(
+                124,
+                timeout_text(exc.stdout),
+                timeout_text(exc.stderr) + f"\nTimed out after {timeout_seconds}s",
+                timed_out=True,
+            )
+
+    def close(self) -> None:
+        """The host executor owns no persistent resources."""
+
+
+class DockerCommandExecutor:
+    """Run commands in a persistent container with the workspace bind-mounted."""
+
+    name = "docker"
+
+    def __init__(
+        self,
+        image: str,
+        *,
+        container_workspace: str = "/testbed",
+        platform: str | None = None,
+        docker_executable: str = "docker",
+    ) -> None:
+        if not image.strip():
+            raise ValueError("Docker command image must not be empty")
+        if not container_workspace.startswith("/"):
+            raise ValueError("Docker workspace must be an absolute POSIX path")
+        if platform is not None and not platform.strip():
+            raise ValueError("Docker platform must not be empty")
+        self.image = image
+        self.container_workspace = container_workspace.rstrip("/") or "/"
+        self.platform = platform
+        self.docker_executable = docker_executable
+        self.container_name = f"yada-agent-{uuid.uuid4().hex[:12]}"
+        self._started_workspace: Path | None = None
+
+    def run(
+        self,
+        *,
+        argv: list[str],
+        workspace: Path,
+        cwd: Path,
+        environment: Mapping[str, str],
+        timeout_seconds: int,
+    ) -> CommandResult:
+        root = workspace.resolve()
+        self._ensure_started(root)
+        relative = cwd.resolve().relative_to(root)
+        container_cwd = PurePosixPath(self.container_workspace, relative.as_posix())
+        command = [
+            self.docker_executable,
+            "exec",
+            "--workdir",
+            str(container_cwd),
+            "--env",
+            f"YADA_WORKSPACE={self.container_workspace}",
+        ]
+        for key, value in sorted(_sanitized_values(environment).items()):
+            command.extend(["--env", f"{key}={value}"])
+        script = (
+            "if [ -f /opt/miniconda3/etc/profile.d/conda.sh ]; then "
+            ". /opt/miniconda3/etc/profile.d/conda.sh && "
+            "conda activate testbed >/dev/null 2>&1 || true; fi; "
+            f"exec {shlex.join(argv)}"
+        )
+        command.extend(
+            [
+                self.container_name,
+                "timeout",
+                "--signal=TERM",
+                str(timeout_seconds),
+                "/bin/bash",
+                "-c",
+                script,
+            ]
+        )
+        try:
+            result = subprocess.run(
+                command,
+                text=True,
+                capture_output=True,
+                timeout=timeout_seconds + 10,
+                check=False,
+            )
+        except FileNotFoundError as exc:
+            raise ToolError(
+                "Docker command execution requires the `docker` CLI"
+            ) from exc
+        except subprocess.TimeoutExpired as exc:
+            return CommandResult(
+                124,
+                timeout_text(exc.stdout),
+                timeout_text(exc.stderr) + f"\nTimed out after {timeout_seconds}s",
+                timed_out=True,
+            )
+        return CommandResult(
+            result.returncode,
+            result.stdout,
+            result.stderr,
+            timed_out=result.returncode == 124,
+        )
+
+    def close(self) -> None:
+        """Force-remove the ephemeral agent container, if it was started."""
+
+        if self._started_workspace is None:
+            return
+        try:
+            subprocess.run(
+                [self.docker_executable, "rm", "--force", self.container_name],
+                text=True,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                timeout=30,
+                check=False,
+            )
+        except (OSError, subprocess.TimeoutExpired):
+            pass
+        self._started_workspace = None
+
+    def _ensure_started(self, workspace: Path) -> None:
+        if self._started_workspace is not None:
+            if self._started_workspace != workspace:
+                raise ToolError("Docker executor cannot switch workspaces")
+            return
+        mount = f"type=bind,source={workspace},target={self.container_workspace}"
+        command = [self.docker_executable, "run"]
+        if self.platform is not None:
+            command.extend(["--platform", self.platform])
+        command.extend(
+            [
+                "--detach",
+                "--rm",
+                "--name",
+                self.container_name,
+                "--mount",
+                mount,
+                "--workdir",
+                self.container_workspace,
+                self.image,
+                "tail",
+                "-f",
+                "/dev/null",
+            ]
+        )
+        try:
+            result = subprocess.run(
+                command,
+                text=True,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                timeout=120,
+                check=False,
+            )
+        except FileNotFoundError as exc:
+            raise ToolError(
+                "Docker command execution requires the `docker` CLI"
+            ) from exc
+        except subprocess.TimeoutExpired as exc:
+            raise ToolError("timed out while starting the agent container") from exc
+        if result.returncode:
+            detail = result.stderr.strip() or result.stdout.strip()
+            raise ToolError(f"cannot start the agent container: {detail}")
+        self._started_workspace = workspace
+
+
+def _sanitized_values(values: Mapping[str, str]) -> dict[str, str]:
+    secret_markers = ("API_KEY", "TOKEN", "SECRET", "PASSWORD", "CREDENTIAL")
+    return {
+        str(key): str(value)
+        for key, value in values.items()
+        if not any(marker in str(key).upper() for marker in secret_markers)
+    }
diff --git a/src/yada/evals/agents/yada.py b/src/yada/evals/agents/yada.py
index c4857a4..3bf3f1f 100644
--- a/src/yada/evals/agents/yada.py
+++ b/src/yada/evals/agents/yada.py
@@ -7,7 +7,7 @@
 from typing import Callable
 
 from yada.agents import Agent
-from yada.environments import CommandApprover
+from yada.environments import CommandApprover, CommandExecutor, DockerCommandExecutor
 from yada.evals.base import AgentRunResult, PreparedTask, RunBudget
 from yada.evals.patches import collect_git_patch
 from yada.models import CompletionClient, DeepSeekClient
@@ -74,7 +74,9 @@ def run(
             approver=CommandApprover(self.command_policy),
             command_timeout_seconds=self.command_timeout_seconds,
             command_environment=_task_environment(prepared),
+            command_executor=_task_command_executor(prepared),
         )
+        command_provenance = _command_provenance(prepared, tools)
         agent = Agent(
             client=client,
             tools=tools,
@@ -84,23 +86,36 @@ def run(
             ),
             max_steps=budget.max_steps,
             emit=self.emit,
-            trace_metadata={"case_id": prepared.task.instance_id},
+            trace_metadata={
+                "case_id": prepared.task.instance_id,
+                **command_provenance,
+            },
         )
 
         started = time.monotonic()
         try:
-            native_result = agent.run(prepared.task.problem_statement)
-            status = "completed" if native_result.finished else "unfinished"
-            summary = native_result.summary
-            usage = native_result.usage
-            steps = native_result.steps
-            details = {"finished": native_result.finished}
-        except Exception as exc:
-            status = "error"
-            summary = f"{type(exc).__name__}: {exc}"
-            usage = {}
-            steps = None
-            details = {"error_type": type(exc).__name__, "error": str(exc)}
+            try:
+                native_result = agent.run(prepared.task.problem_statement)
+                status = "completed" if native_result.finished else "unfinished"
+                summary = native_result.summary
+                usage = native_result.usage
+                steps = native_result.steps
+                details = {
+                    "finished": native_result.finished,
+                    **command_provenance,
+                }
+            except Exception as exc:
+                status = "error"
+                summary = f"{type(exc).__name__}: {exc}"
+                usage = {}
+                steps = None
+                details = {
+                    "error_type": type(exc).__name__,
+                    "error": str(exc),
+                    **command_provenance,
+                }
+        finally:
+            tools.close()
 
         return AgentRunResult(
             agent=self.name,
@@ -125,3 +140,36 @@ def _task_environment(prepared: PreparedTask) -> dict[str, str]:
         for key, item in value.items()
         if isinstance(key, str) and isinstance(item, str)
     }
+
+
+def _task_command_executor(prepared: PreparedTask) -> CommandExecutor | None:
+    value = prepared.metadata.get("command_backend")
+    if value is None:
+        return None
+    if not isinstance(value, dict) or value.get("type") != "docker":
+        raise ValueError("prepared command_backend must have type='docker'")
+    image = value.get("image")
+    workdir = value.get("workdir", "/testbed")
+    platform = value.get("platform")
+    if not isinstance(image, str) or not image.strip():
+        raise ValueError("Docker command backend requires an image")
+    if not isinstance(workdir, str) or not workdir.startswith("/"):
+        raise ValueError("Docker command backend requires an absolute workdir")
+    if platform is not None and not isinstance(platform, str):
+        raise ValueError("Docker command backend platform must be a string")
+    return DockerCommandExecutor(
+        image,
+        container_workspace=workdir,
+        platform=platform,
+    )
+
+
+def _command_provenance(
+    prepared: PreparedTask,
+    tools: ToolRunner,
+) -> dict[str, str]:
+    provenance = {"command_environment": tools.context.command_executor.name}
+    backend = prepared.metadata.get("command_backend")
+    if isinstance(backend, dict) and isinstance(backend.get("image"), str):
+        provenance["command_image"] = backend["image"]
+    return provenance
diff --git a/src/yada/evals/benchmarks/swebench.py b/src/yada/evals/benchmarks/swebench.py
index ec10763..f2f1289 100644
--- a/src/yada/evals/benchmarks/swebench.py
+++ b/src/yada/evals/benchmarks/swebench.py
@@ -3,12 +3,15 @@
 from __future__ import annotations
 
 import json
+import queue
 import re
+import shutil
 import subprocess
 import sys
+import threading
 import time
 from pathlib import Path
-from typing import Any
+from typing import Any, Callable, TextIO
 
 from yada.evals.base import AgentRunResult, EvalTask, GradeResult, PreparedTask
 
@@ -25,6 +28,7 @@
 )
 _SWEBENCH_PACKAGE = "swebench==4.1.0"
 _INSTANCE_PREFIX = "__YADA_SWEBENCH_INSTANCE__="
+_AGENT_IMAGE_PREFIX = "__YADA_SWEBENCH_AGENT_IMAGE__="
 _INSTANCE_LOADER = f"""
 import json
 import sys
@@ -39,6 +43,47 @@
 public_row = {{key: row.get(key) for key in public_fields}}
 print({_INSTANCE_PREFIX!r} + json.dumps(public_row, ensure_ascii=False, default=str))
 """
+_AGENT_IMAGE_PREPARER = f"""
+import json
+import sys
+import docker
+from swebench.harness.docker_build import build_instance_images
+from swebench.harness.test_spec.test_spec import make_test_spec
+from swebench.harness.utils import load_swebench_dataset
+
+dataset_name, split, instance_id, namespace_value = sys.argv[1:5]
+namespace = None if namespace_value == "none" else namespace_value
+rows = load_swebench_dataset(dataset_name, split, [instance_id])
+row = next((item for item in rows if item.get("instance_id") == instance_id), None)
+if row is None:
+    raise SystemExit(f"instance {{instance_id!r}} not found")
+
+client = docker.from_env()
+client.ping()
+spec = make_test_spec(row, namespace=namespace)
+try:
+    client.images.get(spec.instance_image_key)
+except docker.errors.ImageNotFound:
+    if spec.is_remote_image:
+        client.images.pull(spec.instance_image_key)
+    else:
+        build_instance_images(
+            client=client,
+            dataset=[row],
+            force_rebuild=False,
+            max_workers=1,
+            namespace=None,
+            tag="latest",
+            env_image_tag="latest",
+        )
+client.images.get(spec.instance_image_key)
+payload = {{
+    "image": spec.instance_image_key,
+    "platform": spec.platform,
+    "workdir": "/testbed",
+}}
+print({_AGENT_IMAGE_PREFIX!r} + json.dumps(payload, ensure_ascii=False))
+"""
 
 
 class SWEbenchBenchmark:
@@ -58,6 +103,7 @@ def __init__(
         clean: bool = False,
         namespace: str | None = "swebench",
         grade_timeout_seconds: int = 1_800,
+        emit: Callable[[str], None] = print,
     ) -> None:
         if grade_mode not in {"docker", "none"}:
             raise ValueError("grade_mode must be 'docker' or 'none'")
@@ -74,8 +120,13 @@ def __init__(
         self.clean = clean
         self.namespace = namespace
         self.grade_timeout_seconds = grade_timeout_seconds
+        self.emit = emit
 
     def load_task(self, instance_id: str) -> EvalTask:
+        if self.source_workspace is None:
+            self.emit("Checking the Docker CLI and daemon for official SWE-bench.")
+            _require_docker()
+            self.emit(f"Loading public SWE-bench task {instance_id}.")
         row = _load_harness_instance(
             self.harness_python,
             self.dataset_name,
@@ -119,6 +170,7 @@ def prepare(self, task: EvalTask, run_dir: Path) -> PreparedTask:
         repo = str(task.metadata["repo"])
         base_commit = str(task.metadata["base_commit"])
 
+        command_backend: dict[str, str] | None = None
         if self.source_workspace is not None:
             _require_head(self.source_workspace, base_commit)
             result = _run(
@@ -136,25 +188,51 @@ def prepare(self, task: EvalTask, run_dir: Path) -> PreparedTask:
                 raise RuntimeError(
                     result.stderr.strip() or "failed to clone source workspace"
                 )
+            _require_head(workspace, base_commit)
         else:
-            workspace.mkdir(parents=True)
-            commands = [
-                ["git", "init", "--quiet"],
-                ["git", "remote", "add", "origin", f"https://github.com/{repo}.git"],
-                ["git", "fetch", "--quiet", "--depth", "1", "origin", base_commit],
-                ["git", "checkout", "--quiet", "--detach", "FETCH_HEAD"],
-            ]
-            for argv in commands:
-                result = _run(argv, workspace, timeout=600)
-                if result.returncode:
-                    raise RuntimeError(
-                        result.stderr.strip() or f"failed command: {argv}"
-                    )
-        _require_head(workspace, base_commit)
+            stdout_path = run_dir / "swebench-agent-image.stdout.log"
+            stderr_path = run_dir / "swebench-agent-image.stderr.log"
+            self.emit(
+                "Preparing the public SWE-bench instance image; "
+                "the first run may take several minutes."
+            )
+            self.emit(
+                "Image preparation logs: "
+                f"{stdout_path} (stdout), {stderr_path} (stderr)"
+            )
+            image, _, _ = _prepare_agent_image(
+                self.harness_python,
+                self.dataset_name,
+                self.split,
+                task.instance_id,
+                self.namespace,
+                stdout_path=stdout_path,
+                stderr_path=stderr_path,
+                emit=self.emit,
+            )
+            self.emit(
+                f"Exporting {image['workdir']} from the prepared image "
+                "into the artifact workspace."
+            )
+            _export_image_workspace(image, workspace)
+            _require_base_ancestor(workspace, base_commit)
+            self.emit(f"Prepared agent workspace: {workspace}")
+            command_backend = {
+                "type": "docker",
+                "image": image["image"],
+                "platform": image["platform"],
+                "workdir": image["workdir"],
+            }
+        metadata: dict[str, Any] = {
+            "repo": repo,
+            "base_commit": base_commit,
+        }
+        if command_backend is not None:
+            metadata["command_backend"] = command_backend
         return PreparedTask(
             task,
             workspace,
-            {"repo": repo, "base_commit": base_commit},
+            metadata,
         )
 
     def grade(
@@ -211,30 +289,30 @@ def grade(
             "--namespace",
             self.namespace if self.namespace is not None else "none",
         ]
+        stdout_path = run_dir / "swebench.stdout.log"
+        stderr_path = run_dir / "swebench.stderr.log"
+        self.emit(
+            "Starting the official SWE-bench grader; this may take several minutes."
+        )
+        self.emit(
+            f"Official grading logs: {stdout_path} (stdout), {stderr_path} (stderr)"
+        )
         started = time.monotonic()
         timed_out = False
         try:
-            process = subprocess.run(
+            process = _run_streaming(
                 argv,
                 cwd=run_dir,
-                text=True,
-                stdout=subprocess.PIPE,
-                stderr=subprocess.PIPE,
                 timeout=self.grade_timeout_seconds + 600,
-                check=False,
+                stdout_path=stdout_path,
+                stderr_path=stderr_path,
+                emit=self.emit,
+                label="swebench grading",
             )
             return_code = process.returncode
-            stdout = process.stdout
-            stderr = process.stderr
-        except subprocess.TimeoutExpired as exc:
+        except subprocess.TimeoutExpired:
             timed_out = True
             return_code = 124
-            stdout = _timeout_text(exc.stdout)
-            stderr = _timeout_text(exc.stderr)
-        stdout_path = run_dir / "swebench.stdout.log"
-        stderr_path = run_dir / "swebench.stderr.log"
-        stdout_path.write_text(stdout, encoding="utf-8")
-        stderr_path.write_text(stderr, encoding="utf-8")
         report_path = run_dir / f"{model_name.replace('/', '__')}.{run_id}.json"
         duration_ms = round((time.monotonic() - started) * 1000)
 
@@ -309,12 +387,305 @@ def _load_harness_instance(
     try:
         row = json.loads(record)
     except json.JSONDecodeError as exc:
-        raise RuntimeError("official SWE-bench Harness returned invalid metadata") from exc
+        raise RuntimeError(
+            "official SWE-bench Harness returned invalid metadata"
+        ) from exc
     if not isinstance(row, dict):
         raise RuntimeError("official SWE-bench Harness returned invalid metadata")
     return {key: row.get(key) for key in _PUBLIC_INSTANCE_FIELDS}
 
 
+def _require_docker(docker_executable: str = "docker") -> None:
+    if shutil.which(docker_executable) is None:
+        raise RuntimeError(
+            "official SWE-bench evaluation requires the `docker` CLI. "
+            "Use `yada eval --case PATH` for a local no-Docker evaluation."
+        )
+    try:
+        process = subprocess.run(
+            [docker_executable, "info", "--format", "{{.ServerVersion}}"],
+            text=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            timeout=30,
+            check=False,
+        )
+    except (FileNotFoundError, subprocess.TimeoutExpired) as exc:
+        raise RuntimeError(
+            "official SWE-bench evaluation requires a running Docker daemon. "
+            "Use `yada eval --case PATH` for a local no-Docker evaluation."
+        ) from exc
+    if process.returncode:
+        detail = process.stderr.strip() or process.stdout.strip()
+        raise RuntimeError(
+            "official SWE-bench evaluation requires a running Docker daemon. "
+            "Use `yada eval --case PATH` for a local no-Docker evaluation. "
+            f"{detail}"
+        )
+
+
+def _prepare_agent_image(
+    harness_python: str,
+    dataset_name: str,
+    split: str,
+    instance_id: str,
+    namespace: str | None,
+    *,
+    stdout_path: Path | None = None,
+    stderr_path: Path | None = None,
+    emit: Callable[[str], None] | None = None,
+) -> tuple[dict[str, str], str, str]:
+    argv = [
+        harness_python,
+        "-u",
+        "-c",
+        _AGENT_IMAGE_PREPARER,
+        dataset_name,
+        split,
+        instance_id,
+        namespace if namespace is not None else "none",
+    ]
+    try:
+        process = _run_streaming(
+            argv,
+            timeout=3_600,
+            stdout_path=stdout_path,
+            stderr_path=stderr_path,
+            emit=emit,
+        )
+    except FileNotFoundError as exc:
+        raise RuntimeError(
+            f"SWE-bench Harness Python does not exist: {harness_python}"
+        ) from exc
+    except subprocess.TimeoutExpired as exc:
+        raise RuntimeError(
+            "timed out while preparing the SWE-bench agent image"
+            + _log_hint(stdout_path, stderr_path)
+        ) from exc
+    if process.returncode:
+        detail = _tail(process.stderr.strip() or process.stdout.strip())
+        raise RuntimeError(
+            f"cannot prepare the SWE-bench agent image: {detail}"
+            + _log_hint(stdout_path, stderr_path)
+        )
+    record = next(
+        (
+            line.removeprefix(_AGENT_IMAGE_PREFIX)
+            for line in process.stdout.splitlines()
+            if line.startswith(_AGENT_IMAGE_PREFIX)
+        ),
+        None,
+    )
+    if record is None:
+        raise RuntimeError("SWE-bench Harness returned no agent image metadata")
+    try:
+        image = json.loads(record)
+    except json.JSONDecodeError as exc:
+        raise RuntimeError("SWE-bench Harness returned invalid image metadata") from exc
+    if not isinstance(image, dict) or not all(
+        isinstance(image.get(key), str) and image[key]
+        for key in ("image", "platform", "workdir")
+    ):
+        raise RuntimeError("SWE-bench Harness returned invalid image metadata")
+    return image, process.stdout, process.stderr
+
+
+def _run_streaming(
+    argv: list[str],
+    *,
+    timeout: int,
+    cwd: Path | None = None,
+    stdout_path: Path | None = None,
+    stderr_path: Path | None = None,
+    emit: Callable[[str], None] | None = None,
+    heartbeat_seconds: float = 30,
+    label: str = "swebench image",
+) -> subprocess.CompletedProcess[str]:
+    """Run a process while teeing both streams to durable logs and progress."""
+
+    handles: dict[str, TextIO] = {}
+    paths = {"stdout": stdout_path, "stderr": stderr_path}
+    for name, path in paths.items():
+        if path is None:
+            continue
+        path.parent.mkdir(parents=True, exist_ok=True)
+        handles[name] = path.open("w", encoding="utf-8")
+
+    process: subprocess.Popen[str] | None = None
+    output = {"stdout": [], "stderr": []}
+    events: queue.Queue[tuple[str, str | None]] = queue.Queue()
+    threads: list[threading.Thread] = []
+    started = time.monotonic()
+    timed_out = False
+
+    def read_stream(name: str, stream: TextIO) -> None:
+        try:
+            for line in iter(stream.readline, ""):
+                events.put((name, line))
+        finally:
+            stream.close()
+            events.put((name, None))
+
+    def write_progress(message: str) -> None:
+        handle = handles.get("stdout")
+        if handle is not None:
+            handle.write(message + "\n")
+            handle.flush()
+        if emit is not None:
+            emit(message)
+
+    try:
+        process = subprocess.Popen(
+            argv,
+            cwd=cwd,
+            text=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            bufsize=1,
+        )
+        assert process.stdout is not None
+        assert process.stderr is not None
+        write_progress(f"[{label}] subprocess started")
+        for name, stream in (
+            ("stdout", process.stdout),
+            ("stderr", process.stderr),
+        ):
+            thread = threading.Thread(
+                target=read_stream,
+                args=(name, stream),
+                daemon=True,
+            )
+            thread.start()
+            threads.append(thread)
+
+        finished_streams: set[str] = set()
+        deadline = started + timeout
+        next_heartbeat = started + heartbeat_seconds
+        while len(finished_streams) < 2:
+            now = time.monotonic()
+            if process.poll() is None and now >= deadline:
+                process.kill()
+                timed_out = True
+            try:
+                name, line = events.get(timeout=0.5)
+            except queue.Empty:
+                line = None
+                name = ""
+            if name and line is None:
+                finished_streams.add(name)
+            elif name:
+                output[name].append(line)
+                handle = handles.get(name)
+                if handle is not None:
+                    handle.write(line)
+                    handle.flush()
+                if emit is not None:
+                    emit(f"[{label} {name}] {line.rstrip()}")
+                next_heartbeat = time.monotonic() + heartbeat_seconds
+
+            now = time.monotonic()
+            if emit is not None and process.poll() is None and now >= next_heartbeat:
+                elapsed = round(now - started)
+                write_progress(
+                    f"[{label}] still running "
+                    f"({elapsed}s elapsed; live logs are being updated)"
+                )
+                next_heartbeat = now + heartbeat_seconds
+
+        return_code = process.wait()
+        stdout = "".join(output["stdout"])
+        stderr = "".join(output["stderr"])
+        if timed_out:
+            write_progress(f"[{label}] timed out after {timeout}s")
+            raise subprocess.TimeoutExpired(
+                argv,
+                timeout,
+                output=stdout,
+                stderr=stderr,
+            )
+        write_progress(f"[{label}] subprocess exited with status {return_code}")
+        return subprocess.CompletedProcess(argv, return_code, stdout, stderr)
+    finally:
+        if process is not None and process.poll() is None:
+            process.kill()
+            process.wait()
+        for thread in threads:
+            thread.join(timeout=1)
+        for handle in handles.values():
+            handle.close()
+
+
+def _tail(value: str, limit: int = 4_000) -> str:
+    return value if len(value) <= limit else "..." + value[-limit:]
+
+
+def _log_hint(stdout_path: Path | None, stderr_path: Path | None) -> str:
+    paths = [str(path) for path in (stdout_path, stderr_path) if path is not None]
+    return f"; see {' and '.join(paths)}" if paths else ""
+
+
+def _export_image_workspace(
+    image: dict[str, str],
+    workspace: Path,
+    docker_executable: str = "docker",
+) -> None:
+    workspace.mkdir(parents=True)
+    created = _run(
+        [
+            docker_executable,
+            "create",
+            "--platform",
+            image["platform"],
+            image["image"],
+            "tail",
+            "-f",
+            "/dev/null",
+        ],
+        workspace.parent,
+    )
+    if created.returncode:
+        raise RuntimeError(
+            created.stderr.strip() or "cannot create the SWE-bench agent container"
+        )
+    container_id = created.stdout.strip()
+    if not container_id:
+        raise RuntimeError("Docker returned no SWE-bench agent container ID")
+    try:
+        copied = _run(
+            [
+                docker_executable,
+                "cp",
+                f"{container_id}:{image['workdir']}/.",
+                str(workspace),
+            ],
+            workspace.parent,
+            timeout=600,
+        )
+        if copied.returncode:
+            raise RuntimeError(
+                copied.stderr.strip() or "cannot export the SWE-bench workspace"
+            )
+    finally:
+        _run(
+            [docker_executable, "rm", "--force", container_id],
+            workspace.parent,
+        )
+    if not (workspace / ".git").exists():
+        raise RuntimeError("SWE-bench agent image contains no /testbed Git workspace")
+    _run(["git", "config", "core.fileMode", "false"], workspace)
+
+
+def _require_base_ancestor(workspace: Path, base_commit: str) -> None:
+    exists = _run(["git", "cat-file", "-e", f"{base_commit}^{{commit}}"], workspace)
+    ancestor = _run(
+        ["git", "merge-base", "--is-ancestor", base_commit, "HEAD"], workspace
+    )
+    if exists.returncode or ancestor.returncode:
+        raise ValueError(
+            f"prepared SWE-bench image is not based on expected commit {base_commit}"
+        )
+
+
 def _require_head(workspace: Path, expected: str) -> None:
     result = _run(["git", "rev-parse", "HEAD"], workspace)
     actual = result.stdout.strip()
@@ -342,11 +713,3 @@ def _run(
 def _model_name(agent_run: AgentRunResult) -> str:
     raw = f"{agent_run.agent}/{agent_run.model}"
     return re.sub(r"[^A-Za-z0-9._/-]+", "-", raw).strip("-") or "unknown"
-
-
-def _timeout_text(value: str | bytes | None) -> str:
-    if value is None:
-        return ""
-    if isinstance(value, bytes):
-        return value.decode("utf-8", errors="replace")
-    return value
diff --git a/src/yada/evals/cli.py b/src/yada/evals/cli.py
index b15b55a..5f835aa 100644
--- a/src/yada/evals/cli.py
+++ b/src/yada/evals/cli.py
@@ -32,7 +32,10 @@ def build_parser() -> argparse.ArgumentParser:
     task.add_argument(
         "--swebench",
         metavar="INSTANCE_ID",
-        help="Official SWE-bench Verified instance to run and grade in Docker.",
+        help=(
+            "Official SWE-bench Verified instance to run and grade; "
+            "requires a running Docker daemon."
+        ),
     )
     parser.add_argument("--agent", choices=["yada", "command"], default="yada")
     parser.add_argument(
diff --git a/src/yada/tools/base.py b/src/yada/tools/base.py
index 26f56fe..4fda248 100644
--- a/src/yada/tools/base.py
+++ b/src/yada/tools/base.py
@@ -7,6 +7,7 @@
 from typing import Any
 
 from yada.environments.approval import CommandApprover
+from yada.environments.commands import CommandExecutor, LocalCommandExecutor
 from yada.environments.workspace import Workspace
 from yada.exceptions import ToolError
 
@@ -31,6 +32,7 @@ class ToolContext:
     command_timeout_seconds: int = 120
     max_output_chars: int = 12_000
     command_environment: dict[str, str] = field(default_factory=dict)
+    command_executor: CommandExecutor = field(default_factory=LocalCommandExecutor)
     state: ToolState = field(default_factory=ToolState)
 
 
diff --git a/src/yada/tools/command.py b/src/yada/tools/command.py
index cdce0dd..3579ee7 100644
--- a/src/yada/tools/command.py
+++ b/src/yada/tools/command.py
@@ -2,16 +2,13 @@
 
 from __future__ import annotations
 
-import os
-import subprocess
 import time
-from collections.abc import Mapping
 from pathlib import Path
 from typing import Any
 
 from yada.exceptions import ToolError
 from yada.tools.base import ToolContext
-from yada.utils.text import timeout_text, truncate_text
+from yada.utils.text import truncate_text
 
 ALLOWED_EXECUTABLES = {
     "bash",
@@ -99,29 +96,18 @@ def run_command(
     effective_timeout = timeout_seconds or context.command_timeout_seconds
     if not isinstance(effective_timeout, int) or not 1 <= effective_timeout <= 1800:
         raise ToolError("timeout_seconds must be between 1 and 1800")
-    env = _sanitized_environment()
-    env.update(_sanitized_values(context.command_environment))
-    env["YADA_WORKSPACE"] = str(context.workspace.root)
     started = time.monotonic()
-    try:
-        result = subprocess.run(
-            argv,
-            cwd=command_cwd,
-            env=env,
-            text=True,
-            capture_output=True,
-            timeout=effective_timeout,
-            check=False,
-        )
-        timed_out = False
-        exit_code = result.returncode
-        stdout = result.stdout
-        stderr = result.stderr
-    except subprocess.TimeoutExpired as exc:
-        timed_out = True
-        exit_code = 124
-        stdout = timeout_text(exc.stdout)
-        stderr = timeout_text(exc.stderr) + f"\nTimed out after {effective_timeout}s"
+    result = context.command_executor.run(
+        argv=argv,
+        workspace=context.workspace.root,
+        cwd=command_cwd,
+        environment=context.command_environment,
+        timeout_seconds=effective_timeout,
+    )
+    timed_out = result.timed_out
+    exit_code = result.exit_code
+    stdout = result.stdout
+    stderr = result.stderr
     duration_ms = round((time.monotonic() - started) * 1000)
     stdout_text, stdout_truncated = truncate_text(stdout, context.max_output_chars)
     stderr_text, stderr_truncated = truncate_text(stderr, context.max_output_chars)
@@ -153,17 +139,5 @@ def run_command(
             if context.state.verified_revision >= 0
             else None
         ),
-    }
-
-
-def _sanitized_environment() -> dict[str, str]:
-    return _sanitized_values(os.environ)
-
-
-def _sanitized_values(values: Mapping[str, str]) -> dict[str, str]:
-    secret_markers = ("API_KEY", "TOKEN", "SECRET", "PASSWORD", "CREDENTIAL")
-    return {
-        key: value
-        for key, value in values.items()
-        if not any(marker in key.upper() for marker in secret_markers)
+        "environment": context.command_executor.name,
     }
diff --git a/src/yada/tools/runner.py b/src/yada/tools/runner.py
index d864828..2233182 100644
--- a/src/yada/tools/runner.py
+++ b/src/yada/tools/runner.py
@@ -5,7 +5,12 @@
 from pathlib import Path
 from typing import Any, Callable
 
-from yada.environments import CommandApprover, Workspace
+from yada.environments import (
+    CommandApprover,
+    CommandExecutor,
+    LocalCommandExecutor,
+    Workspace,
+)
 from yada.exceptions import ToolError
 from yada.tools.base import ToolContext, ToolExecution
 from yada.tools.command import run_command
@@ -30,6 +35,7 @@ def __init__(
         command_timeout_seconds: int = 120,
         max_output_chars: int = 12_000,
         command_environment: dict[str, str] | None = None,
+        command_executor: CommandExecutor | None = None,
         approver: CommandApprover | None = None,
     ) -> None:
         self.context = ToolContext(
@@ -38,6 +44,7 @@ def __init__(
             command_timeout_seconds=command_timeout_seconds,
             max_output_chars=max_output_chars,
             command_environment=dict(command_environment or {}),
+            command_executor=command_executor or LocalCommandExecutor(),
         )
         self._handlers: dict[str, ToolHandler] = {
             "search_code": search_code,
@@ -93,3 +100,8 @@ def final_state(self) -> dict[str, Any]:
         """Collect the bounded Git status and diff used in the final result."""
 
         return final_state(self.context)
+
+    def close(self) -> None:
+        """Release resources owned by the configured command backend."""
+
+        self.context.command_executor.close()
diff --git a/tests/environments/test_commands.py b/tests/environments/test_commands.py
new file mode 100644
index 0000000..0eb4ef2
--- /dev/null
+++ b/tests/environments/test_commands.py
@@ -0,0 +1,73 @@
+from __future__ import annotations
+
+import subprocess
+from pathlib import Path
+
+from yada.environments import CommandApprover, DockerCommandExecutor
+from yada.tools import ToolRunner
+
+
+def test_docker_executor_mounts_workspace_and_routes_command(
+    tmp_path: Path, monkeypatch
+) -> None:
+    calls: list[list[str]] = []
+
+    def fake_run(argv, **kwargs):
+        calls.append(list(argv))
+        if argv[1] == "run":
+            return subprocess.CompletedProcess(argv, 0, "container-id\n", "")
+        if argv[1] == "exec":
+            return subprocess.CompletedProcess(argv, 0, "inside\n", "")
+        if argv[1] == "rm":
+            return subprocess.CompletedProcess(argv, 0, "", "")
+        raise AssertionError(argv)
+
+    monkeypatch.setattr("yada.environments.commands.subprocess.run", fake_run)
+    executor = DockerCommandExecutor(
+        "swebench/agent-image:latest",
+        platform="linux/x86_64",
+    )
+    runner = ToolRunner(
+        tmp_path,
+        approver=CommandApprover("allow"),
+        command_executor=executor,
+    )
+
+    result = runner.execute(
+        "run_command",
+        {
+            "argv": ["python", "-c", "print('ok')"],
+            "purpose": "test",
+        },
+    )
+    runner.close()
+
+    assert result.data["ok"] is True
+    assert result.data["stdout"] == "inside\n"
+    assert result.data["environment"] == "docker"
+    assert result.data["verified_revision"] == 0
+    start = calls[0]
+    assert start[:4] == ["docker", "run", "--platform", "linux/x86_64"]
+    assert start[4:6] == ["--detach", "--rm"]
+    assert any(
+        item == f"type=bind,source={tmp_path.resolve()},target=/testbed"
+        for item in start
+    )
+    execute = calls[1]
+    assert execute[:2] == ["docker", "exec"]
+    assert "conda activate testbed" in execute[-1]
+    assert "exec python -c 'print('" in execute[-1]
+    assert calls[-1][1:3] == ["rm", "--force"]
+
+
+def test_local_executor_remains_the_default(tmp_path: Path) -> None:
+    runner = ToolRunner(tmp_path, approver=CommandApprover("allow"))
+
+    result = runner.execute(
+        "run_command",
+        {"argv": ["python", "-c", "print('local')"], "purpose": "inspect"},
+    )
+
+    assert result.data["ok"] is True
+    assert result.data["stdout"].strip() == "local"
+    assert result.data["environment"] == "local"
diff --git a/tests/evals/test_eval_cli.py b/tests/evals/test_eval_cli.py
index 15e288c..d052814 100644
--- a/tests/evals/test_eval_cli.py
+++ b/tests/evals/test_eval_cli.py
@@ -19,9 +19,7 @@ def test_eval_cli_has_two_task_selectors() -> None:
 
 def test_eval_cli_rejects_multiple_task_selectors() -> None:
     with pytest.raises(SystemExit):
-        build_parser().parse_args(
-            ["--case", "case-dir", "--swebench", "owner__repo-1"]
-        )
+        build_parser().parse_args(["--case", "case-dir", "--swebench", "owner__repo-1"])
 
 
 def test_swebench_namespace_is_automatic_on_apple_silicon(monkeypatch) -> None:
diff --git a/tests/evals/test_local_benchmark.py b/tests/evals/test_local_benchmark.py
index ce29141..4ece339 100644
--- a/tests/evals/test_local_benchmark.py
+++ b/tests/evals/test_local_benchmark.py
@@ -201,6 +201,10 @@ def test_checked_in_case_shares_prompt_with_swebench_adapter(monkeypatch) -> Non
         "yada.evals.benchmarks.swebench._load_harness_instance",
         lambda *_: instance,
     )
+    monkeypatch.setattr(
+        "yada.evals.benchmarks.swebench._require_docker",
+        lambda: None,
+    )
     swebench_task = SWEbenchBenchmark(
         grade_mode="none",
     ).load_task("pytest-dev__pytest-10051")
diff --git a/tests/evals/test_swebench.py b/tests/evals/test_swebench.py
index 489011f..02b332f 100644
--- a/tests/evals/test_swebench.py
+++ b/tests/evals/test_swebench.py
@@ -1,12 +1,23 @@
 from __future__ import annotations
 
 import json
+import shutil
 import subprocess
+import sys
 from pathlib import Path
 
-from yada.evals import AgentRunResult
+import pytest
+
+from yada.evals import AgentRunResult, EvalTask
 from yada.evals.benchmarks import SWEbenchBenchmark
-from yada.evals.benchmarks.swebench import _INSTANCE_PREFIX, _load_harness_instance
+from yada.evals.benchmarks.swebench import (
+    _AGENT_IMAGE_PREFIX,
+    _INSTANCE_PREFIX,
+    _load_harness_instance,
+    _prepare_agent_image,
+    _require_docker,
+    _run_streaming,
+)
 
 
 def _source_repo(tmp_path: Path) -> tuple[Path, str]:
@@ -120,10 +131,12 @@ def test_swebench_parses_official_run_report(tmp_path: Path, monkeypatch) -> Non
         "yada.evals.benchmarks.swebench._load_harness_instance",
         lambda *_: _instance_row(head),
     )
+    messages: list[str] = []
     benchmark = SWEbenchBenchmark(
         source_workspace=source,
         grade_mode="docker",
         harness_python="python-for-swebench",
+        emit=messages.append,
     )
     artifacts = tmp_path / "artifacts"
     task = benchmark.load_task("owner__repo-1")
@@ -149,10 +162,13 @@ def fake_run(argv, **kwargs):
             ),
             encoding="utf-8",
         )
+        kwargs["stdout_path"].write_text("ok\n", encoding="utf-8")
+        kwargs["stderr_path"].write_text("", encoding="utf-8")
+        assert kwargs["label"] == "swebench grading"
         return subprocess.CompletedProcess(argv, 0, "ok", "")
 
     monkeypatch.setattr(
-        "yada.evals.benchmarks.swebench.subprocess.run",
+        "yada.evals.benchmarks.swebench._run_streaming",
         fake_run,
     )
 
@@ -161,3 +177,200 @@ def fake_run(argv, **kwargs):
     assert grade.status == "resolved"
     assert grade.resolved is True
     assert grade.details["official_report"]["schema_version"] == 2
+    assert (artifacts / "swebench.stdout.log").read_text() == "ok\n"
+    assert any("Official grading logs" in message for message in messages)
+
+
+def test_swebench_requires_docker_before_official_work(monkeypatch) -> None:
+    monkeypatch.setattr("yada.evals.benchmarks.swebench.shutil.which", lambda _: None)
+
+    with pytest.raises(RuntimeError, match="--case PATH"):
+        _require_docker()
+
+
+def test_official_load_checks_docker_before_loading_dataset(monkeypatch) -> None:
+    calls: list[str] = []
+
+    def missing_docker():
+        calls.append("docker")
+        raise RuntimeError("Docker unavailable")
+
+    def load_dataset(*_):
+        calls.append("dataset")
+        raise AssertionError("dataset must not load without Docker")
+
+    monkeypatch.setattr(
+        "yada.evals.benchmarks.swebench._require_docker", missing_docker
+    )
+    monkeypatch.setattr(
+        "yada.evals.benchmarks.swebench._load_harness_instance", load_dataset
+    )
+    benchmark = SWEbenchBenchmark(harness_python="python-for-swebench")
+
+    with pytest.raises(RuntimeError, match="Docker unavailable"):
+        benchmark.load_task("owner__repo-1")
+
+    assert calls == ["docker"]
+
+
+def test_swebench_agent_image_metadata_comes_from_harness(monkeypatch) -> None:
+    payload = {
+        "image": "swebench/sweb.eval.x86_64.owner_repo-1:latest",
+        "platform": "linux/x86_64",
+        "workdir": "/testbed",
+    }
+
+    def fake_run(argv, **kwargs):
+        assert argv[0] == "python-for-swebench"
+        assert argv[1] == "-u"
+        assert argv[-4:] == [
+            "princeton-nlp/SWE-bench_Verified",
+            "test",
+            "owner__repo-1",
+            "swebench",
+        ]
+        return subprocess.CompletedProcess(
+            argv,
+            0,
+            "build output\n" + _AGENT_IMAGE_PREFIX + json.dumps(payload) + "\n",
+            "",
+        )
+
+    monkeypatch.setattr("yada.evals.benchmarks.swebench._run_streaming", fake_run)
+
+    image, stdout, stderr = _prepare_agent_image(
+        "python-for-swebench",
+        "princeton-nlp/SWE-bench_Verified",
+        "test",
+        "owner__repo-1",
+        "swebench",
+    )
+
+    assert image == payload
+    assert "build output" in stdout
+    assert stderr == ""
+
+
+def test_streaming_process_tees_live_output_and_heartbeat(tmp_path: Path) -> None:
+    stdout_path = tmp_path / "image.stdout.log"
+    stderr_path = tmp_path / "image.stderr.log"
+    messages: list[str] = []
+
+    process = _run_streaming(
+        [
+            sys.executable,
+            "-u",
+            "-c",
+            (
+                "import sys, time; "
+                "print('building base image'); "
+                "print('download warning', file=sys.stderr); "
+                "time.sleep(0.1); "
+                "print('building instance image')"
+            ),
+        ],
+        timeout=5,
+        stdout_path=stdout_path,
+        stderr_path=stderr_path,
+        emit=messages.append,
+        heartbeat_seconds=0.02,
+    )
+
+    assert process.returncode == 0
+    assert process.stdout == "building base image\nbuilding instance image\n"
+    assert process.stderr == "download warning\n"
+    assert "building base image" in stdout_path.read_text()
+    assert "subprocess exited with status 0" in stdout_path.read_text()
+    assert stderr_path.read_text() == "download warning\n"
+    assert any("still running" in message for message in messages)
+    assert any("building instance image" in message for message in messages)
+
+
+def test_agent_image_failure_keeps_logs_and_reports_their_paths(
+    tmp_path: Path, monkeypatch
+) -> None:
+    stdout_path = tmp_path / "image.stdout.log"
+    stderr_path = tmp_path / "image.stderr.log"
+
+    def fake_run(argv, **kwargs):
+        kwargs["stdout_path"].write_text("partial build output\n", encoding="utf-8")
+        kwargs["stderr_path"].write_text("build failed\n", encoding="utf-8")
+        return subprocess.CompletedProcess(
+            argv, 1, "partial build output\n", "build failed\n"
+        )
+
+    monkeypatch.setattr("yada.evals.benchmarks.swebench._run_streaming", fake_run)
+
+    with pytest.raises(RuntimeError) as error:
+        _prepare_agent_image(
+            "python-for-swebench",
+            "princeton-nlp/SWE-bench_Verified",
+            "test",
+            "owner__repo-1",
+            "swebench",
+            stdout_path=stdout_path,
+            stderr_path=stderr_path,
+        )
+
+    assert "build failed" in str(error.value)
+    assert str(stdout_path) in str(error.value)
+    assert str(stderr_path) in str(error.value)
+    assert stdout_path.read_text() == "partial build output\n"
+    assert stderr_path.read_text() == "build failed\n"
+
+
+def test_swebench_prepares_container_workspace_for_agent(
+    tmp_path: Path, monkeypatch
+) -> None:
+    source, head = _source_repo(tmp_path)
+    image = {
+        "image": "swebench/sweb.eval.x86_64.owner_repo-1:latest",
+        "platform": "linux/x86_64",
+        "workdir": "/testbed",
+    }
+
+    def fake_prepare(*_, stdout_path, stderr_path, **__):
+        stdout_path.write_text("prepared", encoding="utf-8")
+        stderr_path.write_text("", encoding="utf-8")
+        return image, "prepared", ""
+
+    monkeypatch.setattr(
+        "yada.evals.benchmarks.swebench._prepare_agent_image", fake_prepare
+    )
+
+    def fake_export(_image, workspace):
+        shutil.copytree(source, workspace)
+        subprocess.run(
+            ["git", "commit", "--allow-empty", "-qm", "SWE-bench setup"],
+            cwd=workspace,
+            check=True,
+        )
+
+    monkeypatch.setattr(
+        "yada.evals.benchmarks.swebench._export_image_workspace", fake_export
+    )
+    messages: list[str] = []
+    benchmark = SWEbenchBenchmark(
+        harness_python="python-for-swebench",
+        emit=messages.append,
+    )
+    task = EvalTask(
+        "owner__repo-1",
+        "Fix VALUE.",
+        {"repo": "owner/repo", "base_commit": head},
+    )
+
+    prepared = benchmark.prepare(task, tmp_path / "artifacts")
+
+    assert prepared.metadata["command_backend"] == {
+        "type": "docker",
+        "image": image["image"],
+        "platform": image["platform"],
+        "workdir": "/testbed",
+    }
+    assert (prepared.workspace / "module.py").read_text() == "VALUE = 1\n"
+    assert (tmp_path / "artifacts/swebench-agent-image.stdout.log").read_text() == (
+        "prepared"
+    )
+    assert any("first run may take several minutes" in message for message in messages)
+    assert any("Image preparation logs" in message for message in messages)

From 195d387e402d0a28a919c4f56f75913226d3a033 Mon Sep 17 00:00:00 2001
From: Gen TANG <gen.tang86@gmail.com>
Date: Mon, 3 Aug 2026 23:35:41 +0800
Subject: [PATCH 3/3] upgrade test

---
 tests/evals/test_swebench.py | 22 ++++++++++++++++------
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/tests/evals/test_swebench.py b/tests/evals/test_swebench.py
index 02b332f..34780a2 100644
--- a/tests/evals/test_swebench.py
+++ b/tests/evals/test_swebench.py
@@ -254,25 +254,35 @@ def fake_run(argv, **kwargs):
 def test_streaming_process_tees_live_output_and_heartbeat(tmp_path: Path) -> None:
     stdout_path = tmp_path / "image.stdout.log"
     stderr_path = tmp_path / "image.stderr.log"
+    heartbeat_seen = tmp_path / "heartbeat-seen"
     messages: list[str] = []
 
+    def emit(message: str) -> None:
+        messages.append(message)
+        if "still running" in message:
+            heartbeat_seen.write_text("seen", encoding="utf-8")
+
     process = _run_streaming(
         [
             sys.executable,
             "-u",
             "-c",
             (
-                "import sys, time; "
-                "print('building base image'); "
-                "print('download warning', file=sys.stderr); "
-                "time.sleep(0.1); "
-                "print('building instance image')"
+                "import pathlib, sys, time\n"
+                "heartbeat = pathlib.Path(sys.argv[1])\n"
+                "print('building base image')\n"
+                "print('download warning', file=sys.stderr)\n"
+                "deadline = time.monotonic() + 2\n"
+                "while not heartbeat.exists() and time.monotonic() < deadline:\n"
+                "    time.sleep(0.01)\n"
+                "print('building instance image')\n"
             ),
+            str(heartbeat_seen),
         ],
         timeout=5,
         stdout_path=stdout_path,
         stderr_path=stderr_path,
-        emit=messages.append,
+        emit=emit,
         heartbeat_seconds=0.02,
     )