diff --git a/docker/Dockerfile b/docker/Dockerfile index 89717613..a06cb747 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -17,14 +17,47 @@ ENV DEBIAN_FRONTEND=noninteractive \ PIP_NO_CACHE_DIR=1 # System deps: git for repo-source templates; curl/ca-certs for HTTPS; -# build-essential because some Python deps (pylint plugins) compile. +# build-essential because some Python deps (pylint plugins) compile; util-linux +# provides setpriv for the agent UID/GID boundary. RUN apt-get update && apt-get install -y --no-install-recommends \ ca-certificates \ curl \ git \ build-essential \ + util-linux \ && rm -rf /var/lib/apt/lists/* +# The harness remains root. Only the evaluated agent and the protected mock +# daemon are dropped to these identities. The shared group grants access to the +# RPC socket only; neither user receives group access to the other's files. +ARG AGENT_UID=2000 +ARG AGENT_GID=2000 +ARG MOCKD_UID=2100 +ARG MOCKD_GID=2100 +ARG MOCK_RPC_GID=2200 +RUN groupadd --gid ${MOCK_RPC_GID} uip-rpc \ + && groupadd --gid ${AGENT_GID} agent \ + && useradd --uid ${AGENT_UID} --gid ${AGENT_GID} --groups uip-rpc \ + --create-home --home-dir /home/agent --shell /usr/sbin/nologin agent \ + && groupadd --gid ${MOCKD_GID} mockd \ + && useradd --uid ${MOCKD_UID} --gid ${MOCKD_GID} --groups uip-rpc \ + --create-home --home-dir /home/mockd --shell /usr/sbin/nologin mockd \ + && install -d -o agent -g agent -m 0700 /work/agent /home/agent \ + && install -d -o root -g root -m 0755 /opt/coder-eval/agent-skills \ + && install -d -o root -g root -m 0700 \ + /opt/coder-eval/grader \ + /opt/coder-eval/grader/input \ + /opt/coder-eval/grader/output \ + /opt/coder-eval/grader/task_dir \ + /opt/coder-eval/grader/plugins \ + /opt/coder-eval/grader/references \ + /opt/coder-eval/grader/templates \ + && install -d -o mockd -g mockd -m 0500 \ + /opt/coder-eval/mock \ + /opt/coder-eval/mock/server \ + /opt/coder-eval/mock/fixtures \ + && install -d -o mockd -g uip-rpc -m 0750 /run/coder-eval + # Node LTS + the Claude Code CLI, pinned. The agent binary is a dominant # non-model driver of eval results, so it travels with the coder_eval release # tag and is bumped deliberately -- mirrors the codex CLI pin @@ -34,8 +67,24 @@ ARG CLAUDE_CODE_VERSION=2.1.177 RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash - \ && apt-get install -y --no-install-recommends nodejs \ && rm -rf /var/lib/apt/lists/* \ + && npm config set prefix /usr/local \ && npm install -g @anthropic-ai/claude-code@${CLAUDE_CODE_VERSION} +# Every built-in agent backend routes its SDK-owned subprocess through the +# same setpriv policy. Claude's SDK accepts only one executable path, so it +# uses the small backend-specific wrapper below; Codex and Antigravity invoke +# the generic launcher directly. +COPY docker/coder_eval_drop_privilege.sh /usr/local/bin/coder_eval_drop_privilege.sh +COPY docker/coder_eval_claude_agent.sh /usr/local/bin/coder_eval_claude_agent.sh +COPY docker/coder_eval_mock_client /usr/local/bin/coder_eval_mock_client +COPY docker/coder_eval_mockd.sh /usr/local/bin/coder_eval_mockd.sh +RUN chmod 0555 \ + /usr/local/bin/coder_eval_drop_privilege.sh \ + /usr/local/bin/coder_eval_claude_agent.sh \ + /usr/local/bin/coder_eval_mock_client \ + /usr/local/bin/coder_eval_mockd.sh \ + && test "$(command -v claude)" = "/usr/local/bin/claude" + # uv: matches host sandbox.py's `uv venv` + `uv pip install` fast path RUN curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR=/usr/local/bin sh @@ -81,6 +130,7 @@ RUN coder-eval _run-task-internal --help > /dev/null # AFTER the (billed) run completes via the version field in task.json. ARG CODER_EVAL_VERSION=unknown LABEL org.coder-eval.version="${CODER_EVAL_VERSION}" +LABEL org.coder-eval.agent-isolation="uid-gid-v1" # Stamp the pinned agent binary too, so the host can assert it via # `docker image inspect` before a (billed) run — same rationale as above. LABEL org.coder-eval.claude-code-version="${CLAUDE_CODE_VERSION}" diff --git a/docker/coder_eval_claude_agent.sh b/docker/coder_eval_claude_agent.sh new file mode 100644 index 00000000..1b6be19d --- /dev/null +++ b/docker/coder_eval_claude_agent.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail + +exec /usr/local/bin/coder_eval_drop_privilege.sh /usr/local/bin/claude "$@" diff --git a/docker/coder_eval_drop_privilege.sh b/docker/coder_eval_drop_privilege.sh new file mode 100644 index 00000000..f09d0358 --- /dev/null +++ b/docker/coder_eval_drop_privilege.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Execute the evaluated agent under its dedicated identity. The harness invokes +# this as root; every descendant inherits the UID/GID, empty capability sets, +# and no-new-privileges bit. +set -euo pipefail + +if [[ $# -eq 0 ]]; then + echo "coder_eval_drop_privilege: missing command" >&2 + exit 64 +fi + +GROUP_ARGS=(--clear-groups) +if [[ "${CODER_EVAL_AGENT_ALLOW_RPC:-}" == "1" ]]; then + GROUP_ARGS=(--groups=uip-rpc) +fi + +exec setpriv \ + --reuid=agent \ + --regid=agent \ + "${GROUP_ARGS[@]}" \ + --inh-caps=-all \ + --ambient-caps=-all \ + --bounding-set=-all \ + --no-new-privs \ + -- "$@" diff --git a/docker/coder_eval_mock_client b/docker/coder_eval_mock_client new file mode 100644 index 00000000..8982d62e --- /dev/null +++ b/docker/coder_eval_mock_client @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail + +exec /usr/local/bin/python -m coder_eval.protected_mock.client "$@" diff --git a/docker/coder_eval_mockd.sh b/docker/coder_eval_mockd.sh new file mode 100644 index 00000000..a9645c5f --- /dev/null +++ b/docker/coder_eval_mockd.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -eq 0 ]]; then + echo "coder_eval_mockd: missing command" >&2 + exit 64 +fi + +exec setpriv \ + --reuid=mockd \ + --regid=mockd \ + --groups=uip-rpc \ + --inh-caps=-all \ + --ambient-caps=-all \ + --bounding-set=-all \ + --no-new-privs \ + -- "$@" diff --git a/docs/DOCKER_ISOLATION.md b/docs/DOCKER_ISOLATION.md index d3076229..0ddb7cfe 100644 --- a/docs/DOCKER_ISOLATION.md +++ b/docs/DOCKER_ISOLATION.md @@ -29,10 +29,20 @@ make docker-image-full Both build `coder-eval-agent:` and tag it `:latest`. -- **`make docker-image`** installs the core package plus **both built-in agents** — claude-code (baked above) and Codex (`--extra codex`, public PyPI). It needs **no credentials** and covers the common case: claude-code or Codex tasks scored with `run_command` / `file_contains` (incl. converted skillsbench tasks). `llm_judge` / `agent_judge` work here too (they route through the run's Anthropic/Bedrock backend). +- **`make docker-image`** installs the core package plus the built-in agents. It needs **no credentials** and carries the `uid-gid-v1` isolation capability used by secure Docker runs. Static file/transcript criteria and `llm_judge` work in protected mode. Privileged dynamic criteria (`run_command`, `uipath_eval`, and `agent_judge`) currently fail closed; see [compatibility limits](#limitations). - **`make docker-image-full`** additionally installs the `uipath` extra. The `uipath` SDK resolves from **public PyPI** (per `uv.lock`), so the build needs **no credentials**. Use this only for tasks that shell out to the in-host `uipath` CLI. (Codex is already in the default image — no extra needed.) -> **Codex sandbox under Docker.** Codex's Landlock-backed `read-only` / `workspace-write` sandboxes can't initialize inside the eval container — their writes/execs fail silently and the agent produces no artifacts (a `score=0` FAILURE with no loud error). The docker runner sets `CODER_EVAL_IN_CONTAINER=1`, and the Codex agent honors it by falling back to `full-access`: the container itself is the trust boundary. Host runs (tempdir) are unaffected — Landlock works there and the marker is unset. So Codex tasks run under `--driver docker` with their natural `acceptEdits` permission mode; no need to set `bypassPermissions` by hand. +> **Codex sandbox under Docker.** Codex's Landlock-backed `read-only` / `workspace-write` sandboxes can't initialize inside the eval container. The runner therefore uses Codex `full-access` inside the agent's own security domain. The boundary is the dedicated Linux agent UID, cleared capabilities, `no_new_privs`, and the protected harness paths—not Landlock and not a root agent process. + +## Agent/grader identity boundary + +`sandbox.docker.agent_isolation` defaults to `true`. The container harness and grader remain root, while every evaluated Claude, Codex, or Antigravity subprocess runs as `agent:agent` (`2000:2000`). The mock fixture service runs separately as `mockd:mockd` (`2100:2100`). A supplemental `uip-rpc` group grants the agent access only to the Unix socket. + +The agent launcher clears inheritable, ambient, and bounding capabilities and sets `no_new_privs`. Generated work is placed in `/work/agent`. Hidden task data, results, raw task/plugin/reference/template sources, and grader inputs live below root-only `/opt/coder-eval/grader`. Raw source bind mounts remain read-only and are never chmod/chowned; only disposable staging copies and the generated workspace are changed. + +Local plugins are projected on the host into manifest-verified bundles containing only supported discovery subtrees. Absolute, escaping, broken, and excluded-target symlinks fail closed. The complete repository is mounted only at a root-inaccessible grader path; the agent sees the sanitized bundle under `/opt/coder-eval/agent-skills`. + +Older/custom images must declare `org.coder-eval.agent-isolation=uid-gid-v1`. A protected run rejects an image without that label before making an LLM call. Images derived with `FROM coder-eval-agent:` inherit it. Runtime-kit injection into an unrelated base does not yet provide the required Linux users and `setpriv` launchers, so it is not compatible with protected mode. ## Running a task in Docker @@ -130,6 +140,14 @@ sandbox: ### Tasks that bring their own base image: the runtime kit (`coder-eval-runtime`) +> **Protected-mode compatibility:** the current runtime kit does not install the +> dedicated identities, `setpriv` launchers, protected directory layout, or the +> `org.coder-eval.agent-isolation=uid-gid-v1` capability label. Because +> `agent_isolation` defaults to `true`, an inject-mode image fails closed at +> preflight. For now, extend `coder-eval-agent:` for protected runs. +> Setting `agent_isolation: false` permits legacy runtime-kit migration but does +> not provide the boundary described on this page. + The `FROM coder-eval-agent` contract above means a task is **rebased** onto the Debian framework image. That breaks tasks whose Dockerfile was written for a different base image (e.g. a Fedora recipe using `dnf`, which doesn't exist on Debian). To keep the task's own base image and build successfully, coder-eval's runtime need to be copied into the task's image. Use `make coder-eval-runtime` first to make the runtime available for copying. @@ -163,7 +181,7 @@ individual targets when you only need one. FROM coder-eval-agent:latest # inherit runtime + entrypoint RUN apt-get update && apt-get install -y --no-install-recommends poppler-utils RUN pip install --no-cache-dir PyMuPDF==1.24.10 -COPY input/ /root/input/ +COPY input/ /work/agent/input/ ``` Behavior: @@ -263,20 +281,19 @@ sandbox: env_passthrough: ["MY_CUSTOM_TOKEN", "ANTHROPIC_API_KEY"] ``` -### `HOME` is forwarded by default +### Agent HOME and Claude state -The default `env_passthrough` includes `HOME` so the in-container `~/.claude` lookup resolves at the same path as on the host (the mount lands at `$HOME/.claude` symmetrically). Practical contract: +In protected mode, the host `HOME` value is not forwarded to the evaluated subprocess. The agent uses `/home/agent`: -- `Path.home()` inside the container returns the host's `HOME` value (e.g. `/Users/you` on macOS). The directory exists in the container because Docker auto-creates it as the mount parent for `~/.claude`. -- `~/.claude` is **not** the host's real dir — the runner makes a throwaway *lean copy* in a tmp dir per task and mounts that copy **read-write** at `$HOME/.claude`. The copy keeps the small set the container needs (auth via `.credentials.json`, `settings.json`, `plugins/`) and **drops heavy or transient per-session state** — `security/` (often hundreds of MB), `projects/`, `cache/`, `file-history/`, `backups/`, `downloads/`, `sessions/`, `telemetry/`, `shell-snapshots/`, `todos/`, `session-env/`, plus the volatile churn dirs the live CLI rewrites. The skip set is a denylist; the authoritative list is `CLAUDE_COPY_IGNORE` in `src/coder_eval/isolation/docker_runner.py` (a test asserts this doc and that constant agree, so the list never silently drifts). The container may write anywhere under `~/.claude`; those writes hit the copy and are discarded when the task ends — the host's real `~/.claude` is never modified. Note the copy includes the OAuth token (`.credentials.json`) and is mounted read-**write**, so the in-container agent can read and tamper with the token *copy* — contained, since the copy is discarded at task end and the host's real dir is untouched. Opt out entirely with `CODER_EVAL_NO_CLAUDE_MOUNT=1`. -- Writes under `$HOME` outside the `~/.claude` mount land in the container's ephemeral rootfs overlay. Don't expect them to persist or to be visible to the host. -- If a tool *detects platform* from `HOME` (e.g. "starts with `/Users/` → macOS"), it will draw the wrong conclusion. Vanishingly rare in practice. +- `~/.claude` is **not** the host's real directory. The runner makes a throwaway *lean copy* and mounts it read-write at `/home/agent/.claude`. The copy keeps the small authentication/settings/plugin set and **drops heavy or transient per-session state** — `security/`, `projects/`, `cache/`, `file-history/`, `backups/`, `downloads/`, `sessions/`, `telemetry/`, `shell-snapshots/`, `todos/`, `session-env/`, plus volatile CLI churn directories. The authoritative skip set is `CLAUDE_COPY_IGNORE` in `docker_runner.py`. Writes affect only the disposable copy. Opt out with `CODER_EVAL_NO_CLAUDE_MOUNT=1`. +- Other writes below `/home/agent` or `/work/agent` are ephemeral until the harness captures the workspace into the protected output mount. +- Harness-only variables such as `SKILLS_REPO_PATH`, `TASK_DIR`, and `CODER_EVAL_*` are removed from agent SDK environments. Required model API credentials remain available. -Remove `HOME` from `env_passthrough` if you don't want this behavior — the container's image-default `HOME=/root` will win, but then the host's OAuth dir is no longer reachable. +When `agent_isolation: false` is explicitly selected for migration, the legacy host-HOME behavior may still apply. That mode is not a security boundary. ## Run directory safety (`--run-dir`) -The host's run dir is bind-mounted **read-write** into the container at the same absolute path (so `task.json` and artifacts land directly on the host filesystem). This makes `--run-dir` load-bearing for isolation: +The host's run dir is bind-mounted read-write at `/opt/coder-eval/grader/output`, below a root-only parent. The agent cannot traverse it; the harness captures `/work/agent` there after the agent lifecycle. This still makes `--run-dir` load-bearing for host safety: - **Do not** point `--run-dir` at a symlink. Docker resolves the source of a bind mount; following a symlink would silently grant the container RW access to a different host location. - **Do not** point `--run-dir` at a sensitive parent (e.g. `$HOME` directly, `/etc`, a repo root). Use a dedicated `runs/` subtree. @@ -286,24 +303,29 @@ The host's run dir is bind-mounted **read-write** into the container at the same | Layer | Location | |---|---| -| Agent process (Claude Code SDK) | inside container | -| Sandbox + per-row criterion checking | inside container | +| Agent process and descendants | container, UID/GID `2000:2000`, `/work/agent` | +| Harness + supported criterion checking | container, root, `/opt/coder-eval/grader` | +| Protected fixture service | container, UID/GID `2100:2100`, exact-command Unix RPC | | **`task.json` serialization** | **container → host bind mount** | | Per-criterion `aggregate()` (P/R/F1, suite thresholds) | host | | Reports, run summary, experiment rollups | host | -`task.json` is the only artifact crossing the boundary. Aggregation reads it via the existing host pipeline unchanged. +`task.json`, logs, and captured workspace artifacts cross through the protected output bind mount. Aggregation reads `task.json` through the existing host pipeline unchanged. ## Limitations -- **Relative template paths**: `template_sources[].path` is resolved to a host absolute path *before* staging, so it won't exist inside the container unless you also forward the parent dir via `sandbox.docker.extra_mounts`. +- **Dynamic privileged graders**: `run_command`, `uipath_eval`, and `agent_judge` are rejected in protected mode until a separate minimal-input grader sandbox exists. This prevents candidate-controlled code from turning a privileged grader into a confused deputy. Migrate to static built-in criteria or explicitly disable isolation only for a trusted transitional run. +- **Custom work directories and extra mounts**: protected mode currently rejects `docker.working_dir` and `docker.extra_mounts` because their agent/private audience is ambiguous. Use the generated `/work/agent` workspace and `template_sources`. +- **Runtime-kit injection**: not yet compatible with protected mode. Extend the current framework image instead. - **No container reuse across tasks**: each task = one fresh container. Adds ~1–3 s startup overhead per task; negligible vs. LLM latency. - **macOS Keychain auth**: not reachable from the container; set `ANTHROPIC_API_KEY` (direct) or Bedrock credentials instead. ## Architecture -The host's `DockerRunner` (`coder_eval/isolation/docker_runner.py`) renders the `docker run` argv, bind-mounts task inputs at `/work/input`, allocates an output dir at `/work/output`, and tails container stdout into `docker.log` in the task's run dir. +The host's `DockerRunner` builds sanitized plugin bundles, rewrites host paths to protected container paths, renders `docker run`, and tails container stdout into `docker.log`. Inputs land at `/opt/coder-eval/grader/input`, output at `/opt/coder-eval/grader/output`, the raw task directory at `/opt/coder-eval/grader/task_dir`, and the agent workspace at `/work/agent`. + +Inside the container, the root entrypoint verifies the protected parent and starts optional `mockd`. The standard Orchestrator prepares the workspace as root, grants only that generated tree to UID 2000, and launches the selected agent through the shared privilege-drop policy. The host reads the final result from the protected output mount and feeds the existing aggregation pipeline. -Inside the container, the entrypoint invokes `coder-eval _run-task-internal` (hidden subcommand), which loads the staged YAML + context, runs the standard in-process Orchestrator (driver auto-coerced back to `tempdir`), and writes `task.json` to the output mount. Host reads it and feeds the existing aggregation pipeline. +Protected runs use Docker's init reaper and default to a 512-process limit when `limits.max_pids` is not specified. An explicit `max_pids` value takes precedence. Before trusted post-run/finalization begins, the harness stops the SDK, repeatedly kills every remaining UID-2000 process, and fails closed if that UID cannot be emptied. A `result_kind` discriminator on `CriterionResult` ensures `ClassificationCriterionResult` subclasses survive the JSON round-trip — without it, host-side aggregation would silently lose `observed_label`/`expected_label`. diff --git a/docs/TASK_DEFINITION_GUIDE.md b/docs/TASK_DEFINITION_GUIDE.md index ef97b474..0da730f7 100644 --- a/docs/TASK_DEFINITION_GUIDE.md +++ b/docs/TASK_DEFINITION_GUIDE.md @@ -541,6 +541,47 @@ Notes: - **Collisions are rejected.** If a `mock_path_dirs` entry already provides an executable of the same name, setup raises rather than letting directory order decide which one runs. - **It stubs a tool; it does not proxy one, and it does not serve per-invocation responses.** Recording a *real* executable on the way through, or returning different output per invocation, stays a hand-written mock under `mock_path_dirs` — both depend on state the harness cannot guarantee (the tool being installed, PATH order, live credentials, a fixture set). +### Protected Fixture-Backed CLIs + +Use `protected_mocks` when `uip` or another mock needs different fixture-backed responses and the fixture itself must not be readable by the evaluated agent. Runtime sealing is unnecessary: the protected fixture is staged directly for `mockd` and never enters the agent workspace. This requires the Docker driver and its default UID/GID isolation: + +```yaml +sandbox: + driver: docker + protected_mocks: + - tool: uip + fixture: ./fixtures/uip-troubleshoot.json + max_requests: 100 + passthrough_argv_prefixes: + - [docsai, ask] +``` + +The agent receives a thin `cli_mocks/uip` wrapper. The fixture is copied into a per-run staging directory, mounted below the private `mockd` filesystem parent, and read only by the `mockd` UID. The client speaks a bounded Unix-socket protocol and has no file-read, path, glob, search, dump, or debug operation. Calls use the existing `cli_mocks/calls.jsonl` schema. + +Fixture files map exact argument lists to responses: + +```json +{ + "version": 1, + "responses": [ + { + "argv": ["rpa", "get-errors", "--output", "json"], + "exit_code": 0, + "stdout": "{\"errors\":[]}\n", + "stderr": "" + } + ], + "default": { + "exit_code": 2, + "stderr": "command not configured for this scenario\n" + } +} +``` + +Matching defaults to exact argv equality. A response may opt into `"match_mode": "normalized"`; this still selects from a finite command map but ignores `--output `, treats `--flag=value` like `--flag value`, and permits token reordering. It never performs subset or substring matching. Duplicate keys, malformed responses, oversized output, request-budget exhaustion, and service startup failures all fail loudly. + +`passthrough_argv_prefixes` is for deliberately public live operations such as `uip docsai ask`. `mockd` invokes the real tool only when argv begins with one of these typed prefixes, caches the response in memory for the run, and never reveals the executable path to the agent. Do not use a broad prefix such as `[or]` or `[auth]`. `protected_mocks` and `record_cli` cannot claim the same tool name. + ## Template Sources Tasks can start with preset files instead of an empty sandbox. Multiple sources are applied sequentially (last wins for conflicts). diff --git a/src/coder_eval/agents/antigravity_agent.py b/src/coder_eval/agents/antigravity_agent.py index 4e1475be..137c6ea9 100644 --- a/src/coder_eval/agents/antigravity_agent.py +++ b/src/coder_eval/agents/antigravity_agent.py @@ -22,6 +22,9 @@ import contextlib import logging import os +import shlex +import shutil +import tempfile import time from collections.abc import AsyncIterator, Callable from contextlib import AsyncExitStack @@ -39,7 +42,10 @@ TurnTimeoutError, truncate_crash_message, ) +from coder_eval.isolation.agent_identity import agent_isolation_enabled from coder_eval.models import ( + AGENT_HOME, + CONTAINER_DROP_SHIM, AgentKind, AntigravityAgentConfig, ApiRoute, @@ -66,7 +72,12 @@ TurnEndStatus, TurnStartEvent, ) -from coder_eval.utils import expand_env_vars +from coder_eval.utils import ( + AGENT_ENV_PASSTHROUGH_VARS, + AGENT_ENV_SCRUB_PREFIXES, + AGENT_ENV_SCRUB_VARS, + expand_env_vars, +) logger = logging.getLogger(__name__) @@ -215,6 +226,7 @@ def __init__( # Absolute dirs to prepend to PATH so sandbox mock CLIs shadow real ones # for the harness's run_command tool — applied at spawn (see start()). self._env_path_prepend: list[str] = [] + self._drop_shim_dir: Path | None = None # _state / _iteration / _iteration_was_incremented / pending_turn lifecycle # bookkeeping lives on the Agent base class (shared defaults + helpers). self._log = PrefixedAdapter(logger, {"prefix": instance_name}) @@ -223,6 +235,22 @@ def _effective_model(self) -> str: """Resolve the model: task ``agent.model`` > ``ANTIGRAVITY_MODEL`` > default.""" return self.config.model or settings.antigravity_model or _DEFAULT_MODEL + def _stage_localharness_drop_shim(self) -> Path: + """Shadow localharness with a wrapper around the shared setpriv policy.""" + + real = shutil.which("localharness") + if real is None: + raise RuntimeError("agent isolation is enabled but localharness is not available on PATH") + shim_dir = Path(tempfile.mkdtemp(prefix="antigravity-drop-")) + wrapper = shim_dir / "localharness" + wrapper.write_text( + f'#!/usr/bin/env bash\nexec {CONTAINER_DROP_SHIM} {shlex.quote(real)} "$@"\n', + encoding="utf-8", + ) + wrapper.chmod(0o555) + self._drop_shim_dir = shim_dir + return shim_dir + def _resolve_skills_paths(self, plugin_tools_dir: str | None) -> list[str]: """Resolve skill search-path roots for the harness's native ``skills_paths``. @@ -316,6 +344,8 @@ async def start( """ self.working_directory = Path(working_directory) self._env_path_prepend = list(env_path_prepend or []) + if agent_isolation_enabled(): + self._env_path_prepend.insert(0, str(self._stage_localharness_drop_shim())) self._state = AgentState.WORKING try: @@ -391,20 +421,34 @@ async def _harness_spawn_guard(self) -> AsyncIterator[None]: PATH window, or its harness would inherit another task's mock dirs. """ async with _harness_spawn_lock(): - if not self._env_path_prepend: - yield - return + scrubbed = { + name: os.environ.pop(name) + for name in list(os.environ) + if name in AGENT_ENV_SCRUB_VARS + or (name.startswith(AGENT_ENV_SCRUB_PREFIXES) and name not in AGENT_ENV_PASSTHROUGH_VARS) + } path_key = next((k for k in os.environ if k.upper() == "PATH"), "PATH") - original = os.environ.get(path_key) - os.environ[path_key] = os.pathsep.join([*self._env_path_prepend, original or ""]) - self._log.debug("PATH prepend for harness spawn: %s", os.pathsep.join(self._env_path_prepend)) + original_path = os.environ.get(path_key) + original_home = os.environ.get("HOME") + if self._env_path_prepend: + os.environ[path_key] = os.pathsep.join([*self._env_path_prepend, original_path or ""]) + self._log.debug("PATH prepend for harness spawn: %s", os.pathsep.join(self._env_path_prepend)) + if agent_isolation_enabled(): + os.environ["HOME"] = AGENT_HOME try: yield finally: - if original is None: - os.environ.pop(path_key, None) - else: - os.environ[path_key] = original + os.environ.update(scrubbed) + if self._env_path_prepend: + if original_path is None: + os.environ.pop(path_key, None) + else: + os.environ[path_key] = original_path + if agent_isolation_enabled(): + if original_home is None: + os.environ.pop("HOME", None) + else: + os.environ["HOME"] = original_home async def communicate( self, @@ -579,6 +623,10 @@ async def _teardown(self) -> None: if stack is not None: with contextlib.suppress(Exception): await stack.aclose() + shim_dir, self._drop_shim_dir = self._drop_shim_dir, None + if shim_dir is not None: + with contextlib.suppress(Exception): + await asyncio.to_thread(shutil.rmtree, shim_dir, ignore_errors=True) class _AntigravityTurnState: diff --git a/src/coder_eval/agents/claude_code_agent.py b/src/coder_eval/agents/claude_code_agent.py index 71cb2267..de678f72 100644 --- a/src/coder_eval/agents/claude_code_agent.py +++ b/src/coder_eval/agents/claude_code_agent.py @@ -37,7 +37,10 @@ format_timeout_reason, ) from coder_eval.formatting import format_messages, format_payload +from coder_eval.isolation.agent_identity import agent_isolation_enabled from coder_eval.models import ( + AGENT_HOME, + CONTAINER_CLAUDE_SHIM, AgentKind, ApiRoute, BedrockRoute, @@ -70,7 +73,7 @@ TurnEndStatus, TurnStartEvent, ) -from coder_eval.utils import dump_dataclass, process_plugins +from coder_eval.utils import dump_dataclass, process_plugins, scrub_agent_env_overrides logger = logging.getLogger(__name__) @@ -765,9 +768,11 @@ def _build_sdk_env( Returns: Tuple of (env_vars_dict, model_override_or_None). """ - base_env: dict[str, str] = {} + base_env: dict[str, str] = scrub_agent_env_overrides() if path := os.environ.get("PATH"): base_env["PATH"] = path + if agent_isolation_enabled(): + base_env["HOME"] = AGENT_HOME if path_prepend: prefix = os.pathsep.join(path_prepend) @@ -1199,6 +1204,10 @@ def _build_claude_query( if isinstance(self.config.claude_settings, dict) else self.config.claude_settings, mcp_servers=self._extra_mcp_servers, + # The SDK accepts a single CLI executable path. The baked wrapper + # invokes the real Claude binary through the same setpriv policy as + # the other backends (UID/GID drop, no capabilities, no_new_privs). + cli_path=CONTAINER_CLAUDE_SHIM if agent_isolation_enabled() else None, **self.config.sdk_options, ) diff --git a/src/coder_eval/agents/codex_agent.py b/src/coder_eval/agents/codex_agent.py index c6e4b2d3..68a6262e 100644 --- a/src/coder_eval/agents/codex_agent.py +++ b/src/coder_eval/agents/codex_agent.py @@ -25,7 +25,10 @@ TurnTimeoutError, truncate_crash_message, ) +from coder_eval.isolation.agent_identity import agent_isolation_enabled, grant_agent_workspace from coder_eval.models import ( + AGENT_HOME, + CONTAINER_DROP_SHIM, AgentKind, ApiRoute, AssistantMessage, @@ -52,7 +55,7 @@ TurnEndStatus, TurnStartEvent, ) -from coder_eval.utils import expand_env_vars +from coder_eval.utils import expand_env_vars, scrub_agent_env_overrides logger = logging.getLogger(__name__) @@ -660,6 +663,7 @@ def __init__( self.working_directory: Path | None = None self._env_path_prepend: list[str] = [] self._login_shell_home: Path | None = None + self._runtime_codex_home: Path | None = None # _state / _iteration / _iteration_was_incremented / pending_turn lifecycle # bookkeeping lives on the Agent base class (shared defaults + helpers). self._log = PrefixedAdapter(logger, {"prefix": instance_name}) @@ -689,6 +693,13 @@ async def start( self.working_directory = Path(working_directory) self._env_path_prepend = list(env_path_prepend or []) self._setup_login_shell_home() + if agent_isolation_enabled(): + if self._login_shell_home is None: + self._login_shell_home = Path(tempfile.mkdtemp(prefix="coder-eval-codex-home-")) + self._runtime_codex_home = Path(AGENT_HOME) / ".codex" + await asyncio.to_thread(self._runtime_codex_home.mkdir, parents=True, exist_ok=True) + grant_agent_workspace(self._login_shell_home) + grant_agent_workspace(self._runtime_codex_home) self._state = AgentState.WORKING try: @@ -696,7 +707,11 @@ async def start( # Build CodexConfig with environment variables for custom API configuration env_override = self._build_codex_env() - config = CodexConfig(env=env_override) if env_override else None + launch_args_override = self._drop_privilege_launch_args() + if env_override is not None or launch_args_override is not None: + config = CodexConfig(env=env_override, launch_args_override=launch_args_override) + else: + config = None # Initialize the Codex client (context manager compatible). Close any # prior client first: start() is driven through execute_with_retry, so @@ -1077,6 +1092,16 @@ def _effective_model(self) -> str | None: """ return self.config.model or settings.codex_model + @staticmethod + def _drop_privilege_launch_args() -> tuple[str, ...] | None: + """Replace the SDK app-server argv with the shared UID-drop launcher.""" + + if not agent_isolation_enabled(): + return None + from codex_cli_bin import bundled_codex_path + + return (CONTAINER_DROP_SHIM, str(bundled_codex_path()), "app-server", "--listen", "stdio://") + def _build_codex_env(self) -> dict[str, str] | None: """Build the environment passed to the Codex app-server. @@ -1091,7 +1116,7 @@ def _build_codex_env(self) -> dict[str, str] | None: (and normalizes the PATH key case-insensitively), so a full PATH value here safely replaces the inherited one. """ - env: dict[str, str] = {} + env: dict[str, str] = scrub_agent_env_overrides() api_key = os.getenv("CODEX_API_KEY") if api_key: env["CODEX_API_KEY"] = api_key @@ -1116,6 +1141,12 @@ def _build_codex_env(self) -> dict[str, str] | None: codex_home = self._codex_home() codex_home.mkdir(parents=True, exist_ok=True) env["CODEX_HOME"] = str(codex_home) + elif agent_isolation_enabled(): + env["HOME"] = AGENT_HOME + env["ZDOTDIR"] = AGENT_HOME + codex_home = self._codex_home() + codex_home.mkdir(parents=True, exist_ok=True) + env["CODEX_HOME"] = str(codex_home) return env if env else None @staticmethod @@ -1157,10 +1188,14 @@ def _setup_login_shell_home(self) -> None: self._cleanup_login_shell_home() if not (self._env_path_prepend and self._login_shell_profiles_supported()): return - original_home = os.environ.get("HOME", "") + # The harness remains root in protected Docker runs. Its HOME and + # ZDOTDIR are private grader state and must never be restored by an + # agent login shell. Use the dedicated agent home as both the runtime + # home and the only profile source in that mode. + original_home = AGENT_HOME if agent_isolation_enabled() else os.environ.get("HOME", "") # Where the user's REAL zsh dotfiles live: their own ZDOTDIR when set, # else their home (zsh's fallback). - original_zdotdir = os.environ.get("ZDOTDIR", "") or original_home + original_zdotdir = AGENT_HOME if agent_isolation_enabled() else os.environ.get("ZDOTDIR", "") or original_home # The profile only ever executes under a POSIX shell, so the PATH # separator is ':' regardless of the host building it. quoted_prepend = shlex.quote(":".join(self._env_path_prepend)) @@ -1766,9 +1801,10 @@ async def _recover_subagent_tool_calls( # Best-effort: a recovery hiccup must never fail the turn. self._log.debug("CodexAgent: sub-agent recovery failed for %s: %s", thread_id, exc) - @staticmethod - def _codex_home() -> Path: + def _codex_home(self) -> Path: """Codex data directory (rollouts live under ``/sessions``).""" + if self._runtime_codex_home is not None: + return self._runtime_codex_home return Path(os.environ.get("CODEX_HOME") or (Path.home() / ".codex")) async def _await_rollout_file(self, home: Path, thread_id: str, *, attempts: int = 20) -> Path | None: diff --git a/src/coder_eval/cli/run_task_internal_command.py b/src/coder_eval/cli/run_task_internal_command.py index 3157e4a7..0ad67327 100644 --- a/src/coder_eval/cli/run_task_internal_command.py +++ b/src/coder_eval/cli/run_task_internal_command.py @@ -1,9 +1,10 @@ """Internal CLI subcommand executed inside the Docker container. Not part of the public CLI surface -- the host's :class:`DockerRunner` -invokes it via ``docker run``. It loads the staged task + context from -``/work/input``, runs one full evaluation cycle in-process (driver=tempdir), -and writes ``task.json`` + ``task.html`` to ``/work/output``. +invokes it via ``docker run``. It loads the staged task + context from the +root-only grader input directory, runs one full evaluation cycle in-process +(driver=tempdir), and writes ``task.json`` + ``task.html`` to the root-only +grader output directory. The container always exits 0 once ``task.json`` is written, even if the task itself failed -- criterion failures are signaled via the final_status @@ -29,6 +30,8 @@ ) from coder_eval.logging_config import setup_logging from coder_eval.models import ( + AGENT_HOME, + CONTAINER_GRADER_DIR, CONTAINER_INPUT_DIR, CONTAINER_OUTPUT_DIR, CONTAINER_TASK_DIR, @@ -82,6 +85,27 @@ def run_task_internal_command( log_level = "DEBUG" if verbose else settings.log_level setup_logging(level=log_level) + from coder_eval.isolation.agent_identity import ( + agent_isolation_enabled, + grant_agent_workspace, + require_isolation_runtime, + ) + + if agent_isolation_enabled(): + require_isolation_runtime() + grader_root = Path(CONTAINER_GRADER_DIR) + grader_stat = grader_root.stat() + if grader_stat.st_uid != 0 or grader_stat.st_mode & 0o077: + raise RuntimeError( + f"protected grader root must be root-owned mode 0700: {grader_root} " + + f"(uid={grader_stat.st_uid}, mode={oct(grader_stat.st_mode & 0o777)})" + ) + claude_state = Path(AGENT_HOME) / ".claude" + if claude_state.exists(): + # This is the disposable host-side copy mounted for the run, never + # the user's real ~/.claude directory. + grant_agent_workspace(claude_state) + # Start the host-heartbeat watchdog: if the host process dies # ungracefully (SIGKILL, Claude-Code Escape, crash) before it can # `docker kill` us, the heartbeat file in output_dir goes stale and @@ -158,6 +182,8 @@ def _watch_host_heartbeat() -> None: # Absent -> None -> standard run_dir/artifacts workspace. workspace_dir_raw = context.get("workspace_dir") workspace_dir = Path(workspace_dir_raw) if workspace_dir_raw else None + protected_mock_config_raw = context.get("protected_mock_config") + protected_mock_config = Path(protected_mock_config_raw) if protected_mock_config_raw else None config_lineage = {k: ConfigLineageEntry.model_validate(v) for k, v in (context.get("config_lineage") or {}).items()} # Prefer the host's raw source_yaml so task.json's audit trail matches # the in-process driver. Fall back to the staged (post-override) YAML @@ -206,5 +232,8 @@ def _watch_host_heartbeat() -> None: orchestrator.stream_callback = StdoutNDJsonCallback() - asyncio.run(orchestrator.run()) + from coder_eval.protected_mock.runtime import running_mock_server + + with running_mock_server(protected_mock_config): + asyncio.run(orchestrator.run()) # Orchestrator.run() writes task.json to run_dir (== output_dir). Done. diff --git a/src/coder_eval/isolation/agent_identity.py b/src/coder_eval/isolation/agent_identity.py new file mode 100644 index 00000000..db0dbb95 --- /dev/null +++ b/src/coder_eval/isolation/agent_identity.py @@ -0,0 +1,113 @@ +"""Linux identity helpers for the in-container evaluated agent.""" + +from __future__ import annotations + +import contextlib +import os +import signal +import sys +import time +from pathlib import Path + +from coder_eval.models import AGENT_GID, AGENT_UID + + +AGENT_ISOLATION_ENV = "CODER_EVAL_AGENT_ISOLATION" +AGENT_KILL_TIMEOUT_SECONDS = 2.0 + + +def agent_isolation_enabled() -> bool: + """Whether the Docker host requested the UID/GID agent boundary.""" + + return os.environ.get(AGENT_ISOLATION_ENV) == "1" + + +def require_isolation_runtime() -> None: + """Fail closed unless Linux root can perform the requested UID drop.""" + + if not agent_isolation_enabled(): + return + if sys.platform != "linux" or not hasattr(os, "geteuid") or os.geteuid() != 0: + raise RuntimeError("agent UID/GID isolation requires a native Linux container running the harness as root") + + +def grant_agent_workspace(path: Path) -> None: + """Give the unprivileged identity ownership of a generated workspace. + + The caller may pass only disposable sandbox content, never a raw host source + checkout. Symlinks are chowned without following their targets. + """ + + if not agent_isolation_enabled(): + return + require_isolation_runtime() + if not path.is_absolute() or not path.exists(): + raise RuntimeError(f"agent workspace must be an existing absolute path: {path}") + + failures: list[str] = [] + chown = getattr(os, "chown", None) + if chown is None: + raise RuntimeError("agent UID/GID isolation requires os.chown") + + def grant(candidate: Path) -> None: + try: + chown(candidate, AGENT_UID, AGENT_GID, follow_symlinks=False) + except OSError as exc: + failures.append(f"{candidate}: {exc}") + + grant(path) + if path.is_dir() and not path.is_symlink(): + for root_name, dirnames, filenames in os.walk(path, followlinks=False): + root = Path(root_name) + for name in (*dirnames, *filenames): + grant(root / name) + + if failures: + detail = "; ".join(failures[:5]) + raise RuntimeError(f"failed to grant generated workspace to agent uid {AGENT_UID}: {detail}") + + +def _agent_pids() -> list[int]: + """Return processes whose real/effective/saved/fs UID includes the agent.""" + + pids: list[int] = [] + for entry in Path("/proc").iterdir(): + if not entry.name.isdigit(): + continue + try: + status_lines = (entry / "status").read_text(encoding="utf-8").splitlines() + uid_line = next(line for line in status_lines if line.startswith("Uid:")) + uids = [int(value) for value in uid_line.split()[1:]] + except (OSError, StopIteration, ValueError): + continue + if AGENT_UID in uids: + pids.append(int(entry.name)) + return pids + + +def _signal_agent_pids(pids: list[int], sig: signal.Signals) -> None: + for pid in pids: + with contextlib.suppress(OSError, ProcessLookupError): + os.kill(pid, sig) + + +def terminate_agent_processes() -> None: + """Terminate and verify removal of every process owned by the agent UID.""" + + if not agent_isolation_enabled(): + return + require_isolation_runtime() + + _signal_agent_pids(_agent_pids(), signal.SIGTERM) + time.sleep(0.1) + + sigkill = getattr(signal, "SIGKILL", signal.SIGTERM) + deadline = time.monotonic() + AGENT_KILL_TIMEOUT_SECONDS + while pids := _agent_pids(): + _signal_agent_pids(pids, sigkill) + if time.monotonic() >= deadline: + residual = _agent_pids() + if residual: + raise RuntimeError(f"agent UID {AGENT_UID} still owns processes after SIGKILL: {residual[:10]}") + return + time.sleep(0.02) diff --git a/src/coder_eval/isolation/docker_runner.py b/src/coder_eval/isolation/docker_runner.py index 5185494d..1ada9814 100644 --- a/src/coder_eval/isolation/docker_runner.py +++ b/src/coder_eval/isolation/docker_runner.py @@ -26,8 +26,13 @@ from coder_eval.logging_config import DEFAULT_LOG_TAIL_MAX_BYTES from coder_eval.models import ( + AGENT_HOME, + CONTAINER_AGENT_SKILLS_DIR, + CONTAINER_AGENT_WORK_DIR, CONTAINER_INPUT_DIR, CONTAINER_OUTPUT_DIR, + CONTAINER_PRIVATE_PLUGIN_DIR, + CONTAINER_TASK_DIR, CONTAINER_WORK_DIR, RESERVED_CONTAINER_DIRS, AgentKind, @@ -37,6 +42,7 @@ PreservationMode, ResourceLimits, ) +from coder_eval.plugin_bundle import stage_bundle from coder_eval.streaming.callbacks import safe_emit from coder_eval.streaming.wire import deserialize_event, has_prefix from coder_eval.utils import get_default_docker_image_tag @@ -64,6 +70,7 @@ # explicitly via `--add-host host.docker.internal:host-gateway`. _DOCKER_HOST_ALIAS = "host.docker.internal" _LOOPBACK_HOSTS = frozenset({"localhost", "127.0.0.1", "::1"}) +DEFAULT_AGENT_ISOLATION_MAX_PIDS = 512 def _rewrite_loopback_for_container(url: str) -> str | None: @@ -256,6 +263,37 @@ def _preflight_image_version(image: str) -> None: ) +def _preflight_agent_isolation_image(image: str) -> None: + """Require an image that contains the declared UID/GID launch boundary.""" + + try: + result = subprocess.run( + [ + "docker", + "image", + "inspect", + "--format", + '{{ index .Config.Labels "org.coder-eval.agent-isolation" }}', + image, + ], + check=True, + capture_output=True, + text=True, + encoding="utf-8", + timeout=10, + ) + except (subprocess.CalledProcessError, subprocess.TimeoutExpired, FileNotFoundError) as exc: + raise DockerRunError( + f"cannot verify UID/GID isolation support for image {image!r}; build or pull the image first" + ) from exc + capability = result.stdout.strip() + if capability != "uid-gid-v1": + raise DockerRunError( + f"image {image!r} does not declare org.coder-eval.agent-isolation=uid-gid-v1; " + + "rebuild it from the latest coder-eval-agent image or disable isolation explicitly" + ) + + _CONTAINER_NAME_INVALID = re.compile(r"[^a-zA-Z0-9_.-]") # A leading Windows drive letter (``C:\foo`` / ``c:/foo``). Used so the colon @@ -482,6 +520,14 @@ def __init__( # _build_argv mounts read-write. None when there is no ~/.claude to # forward or the mount is opted out (CODER_EVAL_NO_CLAUDE_MOUNT). self._claude_mount_src: Path | None = None + # Prepared before argv rendering. Agent-visible plugin mounts contain + # only manifest-verified projections; raw sources are mounted under the + # root-only grader parent at unrelated container paths. + self._agent_plugin_mounts: list[tuple[Path, str]] = [] + self._private_source_mounts: list[tuple[Path, str]] = [] + self._host_to_private_paths: dict[str, str] = {} + self._host_plugin_to_agent_paths: dict[str, str] = {} + self._mock_fixture_mount: Path | None = None # Resolved in run() (needs the built image for "auto"). Concrete WORKDIR the # agent runs at + copies out from; None = standard artifacts workspace. self._workspace_dir: str | None = None @@ -503,6 +549,7 @@ async def run(self) -> EvaluationResult: dispatcher converts that to an ERROR-status EvaluationResult. """ _preflight() + self._validate_agent_isolation_compatibility() # Resolve the run image: build from a Dockerfile if configured (which # overrides `image`), else use the configured image. The build is # side-effecting, so it runs in a worker thread like the other docker @@ -521,6 +568,8 @@ async def run(self) -> EvaluationResult: # a task-supplied Dockerfile won't carry the org.coder-eval.version label. if not self._docker_config.dockerfile_path: await asyncio.to_thread(_preflight_image_version, image) + if self._docker_config.agent_isolation: + await asyncio.to_thread(_preflight_agent_isolation_image, image) await asyncio.to_thread(self.rt.run_dir.mkdir, parents=True, exist_ok=True) # Docker WORKDIR alignment: resolve the concrete workspace path @@ -528,6 +577,20 @@ async def run(self) -> EvaluationResult: # /root). Forwarded to the in-container orchestrator via the staged context # and rendered as `docker run -w`. None keeps the standard artifacts workspace. self._workspace_dir = await asyncio.to_thread(_resolve_workspace_dir, self._docker_config.working_dir, image) + if self._docker_config.agent_isolation: + if self._workspace_dir is not None: + raise DockerRunError( + "docker.agent_isolation does not yet support docker.working_dir; " + + "use the default generated /work/agent workspace or disable isolation explicitly" + ) + if self._docker_config.extra_mounts: + raise DockerRunError( + "docker.agent_isolation rejects extra_mounts because their agent/private audience is ambiguous; " + + "stage the required input through template_sources or disable isolation explicitly" + ) + self._workspace_dir = CONTAINER_AGENT_WORK_DIR + elif self.rt.task.sandbox.protected_mocks: + raise DockerRunError("sandbox.protected_mocks requires docker.agent_isolation: true") # Stage only the inputs (task YAML + context). The *output* dir is # the host's run_dir itself, bind-mounted at the same path inside @@ -612,9 +675,13 @@ async def _stage_inputs(self, input_dir: Path) -> None: # has since mutated rt.task in-memory (e.g. --model, -D run_limits.max_turns), and the # container needs to see those mutations. task_yaml_in = input_dir / "task.yaml" + task_payload = self.rt.task.model_dump(mode="json") + if self._docker_config.agent_isolation: + await asyncio.to_thread(self._prepare_isolated_sources, input_dir.parent) + task_payload = self._rewrite_task_paths(task_payload) def _dump_task_yaml() -> str: - return yaml.safe_dump(self.rt.task.model_dump(mode="json"), sort_keys=False) + return yaml.safe_dump(task_payload, sort_keys=False) task_yaml_text = await asyncio.to_thread(_dump_task_yaml) await asyncio.to_thread(task_yaml_in.write_text, task_yaml_text, encoding="utf-8") @@ -633,10 +700,203 @@ def _dump_task_yaml() -> str: # Docker WORKDIR alignment: concrete path the in-container # orchestrator runs at + captures out (None = standard workspace). "workspace_dir": self._workspace_dir, + "protected_mock_config": ( + "/opt/coder-eval/mock/fixtures/mock-config.json" if self._mock_fixture_mount else None + ), } ) await asyncio.to_thread((input_dir / "context.json").write_text, context_payload, encoding="utf-8") + def _validate_agent_isolation_compatibility(self) -> None: + """Reject task features whose privileged behavior is not isolated yet.""" + + if not self._docker_config.agent_isolation: + return + agent_type = str(self.rt.task.agent.type) if self.rt.task.agent and self.rt.task.agent.type else "" + supported_agents = { + AgentKind.CLAUDE_CODE.value, + AgentKind.CODEX.value, + AgentKind.ANTIGRAVITY.value, + AgentKind.NONE.value, + } + if agent_type not in supported_agents: + raise DockerRunError( + f"docker.agent_isolation has no verified UID-drop launch seam for agent type {agent_type!r}" + ) + + # These criterion implementations can execute another agent or arbitrary + # task-authored commands in the privileged harness. If that execution + # imports candidate-controlled code, it can act as a confused deputy and + # publish hidden grader bytes. A separate minimal-input grader sandbox is + # required before they can run in protected mode. + unsupported_criteria = sorted( + { + criterion.type + for criterion in self.rt.task.success_criteria + if criterion.type in {"agent_judge", "run_command", "uipath_eval"} + } + ) + if unsupported_criteria: + raise DockerRunError( + "docker.agent_isolation rejects privileged dynamic criteria until they have a separate grader " + + f"sandbox: {unsupported_criteria}. Use static/built-in criteria or explicitly disable isolation." + ) + + def _prepare_isolated_sources(self, staging: Path) -> None: + """Prepare public plugin bundles and private raw-source mount mappings. + + This method never changes ownership or permissions on a source checkout. + Raw sources remain read-only and are mounted only below the image's + root-owned ``/opt/coder-eval/grader`` directory. + """ + + self._agent_plugin_mounts = [] + self._private_source_mounts = [] + self._host_to_private_paths = {} + self._host_plugin_to_agent_paths = {} + self._mock_fixture_mount = None + + task_dir = self.rt.task_file.parent.resolve() if self.rt.task_file else None + if task_dir is not None: + self._host_to_private_paths[str(task_dir)] = CONTAINER_TASK_DIR + + if self.rt.task.agent and self.rt.task.agent.system_prompt_file: + raise DockerRunError( + "docker.agent_isolation requires system_prompt_file to be resolved to inline system_prompt " + + "before container staging" + ) + + plugins = (self.rt.task.agent.plugins if self.rt.task.agent else None) or [] + bundle_root = staging / "agent-skills" + for index, plugin in enumerate(plugins): + raw = plugin.get("path") if isinstance(plugin, dict) else None + if not raw: + continue + source = Path(os.path.expandvars(os.path.expanduser(raw))).resolve() + if not source.is_dir(): + raise DockerRunError(f"agent plugin source does not exist or is not a directory: {source}") + + bundle_source = bundle_root / f"plugin-{index}" + manifest = stage_bundle(source, bundle_source) + public_target = f"{CONTAINER_AGENT_SKILLS_DIR}/plugin-{index}" + private_target = f"{CONTAINER_PRIVATE_PLUGIN_DIR}/plugin-{index}" + self._agent_plugin_mounts.append((bundle_source, public_target)) + self._register_private_mount(source, private_target) + self._host_plugin_to_agent_paths[str(source)] = public_target + logger.info( + "Prepared agent-visible plugin bundle %s -> %s (%d files, digest %s)", + source, + public_target, + len(manifest.files), + manifest.digest[:12], + ) + + from coder_eval.models import TemplateDirSource + + template_index = 0 + for source in self.rt.task.sandbox.template_sources or []: + if not isinstance(source, TemplateDirSource): + continue + host_path = Path(source.path).resolve() + if task_dir is not None and (host_path == task_dir or task_dir in host_path.parents): + continue + self._register_private_mount(host_path, f"/opt/coder-eval/grader/templates/source-{template_index}") + template_index += 1 + + reference = self.rt.task.reference + if reference is not None: + if reference.directory: + self._register_external_private_path( + Path(reference.directory), "/opt/coder-eval/grader/references/directory" + ) + if reference.file: + reference_file = Path(reference.file).resolve() + self._register_external_private_path( + reference_file.parent, "/opt/coder-eval/grader/references/file-parent" + ) + + protected_mocks = self.rt.task.sandbox.protected_mocks or [] + if protected_mocks: + mock_root = staging / "protected-mock-fixtures" + mock_root.mkdir(parents=True, exist_ok=True) + tools: list[dict[str, object]] = [] + for index, spec in enumerate(protected_mocks): + source = Path(spec.fixture).resolve() + if not source.is_file(): + raise DockerRunError(f"protected mock fixture does not exist: {source}") + filename = f"fixture-{index}.json" + destination = mock_root / filename + shutil.copy2(source, destination) + destination.chmod(0o444) + container_fixture = f"/opt/coder-eval/mock/fixtures/{filename}" + self._host_to_private_paths[str(source)] = container_fixture + tools.append( + { + "tool": spec.tool, + "fixture": container_fixture, + "max_requests": spec.max_requests, + "passthrough_argv_prefixes": spec.passthrough_argv_prefixes, + } + ) + config_path = mock_root / "mock-config.json" + config_path.write_text(json.dumps({"version": 1, "tools": tools}), encoding="utf-8") + config_path.chmod(0o444) + mock_root.chmod(0o555) + self._mock_fixture_mount = mock_root + + def _register_external_private_path(self, source: Path, target: str) -> None: + source = source.resolve() + task_dir = self.rt.task_file.parent.resolve() if self.rt.task_file else None + if task_dir is not None and (source == task_dir or task_dir in source.parents): + return + self._register_private_mount(source, target) + + def _register_private_mount(self, source: Path, target: str) -> None: + source = source.resolve() + key = str(source) + if key in self._host_to_private_paths: + return + self._host_to_private_paths[key] = target + self._private_source_mounts.append((source, target)) + + def _rewrite_task_paths(self, payload: dict[str, object]) -> dict[str, object]: + """Rewrite host paths to their protected container mount locations.""" + + replacements = sorted(self._host_to_private_paths.items(), key=lambda item: len(item[0]), reverse=True) + + def rewrite(value: object) -> object: + if isinstance(value, str): + for source, target in replacements: + value = value.replace(source, target) + return value + if isinstance(value, list): + return [rewrite(item) for item in value] + if isinstance(value, dict): + return {key: rewrite(item) for key, item in value.items()} + return value + + rewritten = rewrite(payload) + if not isinstance(rewritten, dict): + raise DockerRunError("internal error rewriting staged task paths") + + # The harness and criteria use private paths, but plugin discovery must + # point at the public projections. Override these fields after the broad + # path rewrite so no raw plugin path can survive serialization. + agent = rewritten.get("agent") + if isinstance(agent, dict): + staged_plugins = agent.get("plugins") + original_plugins = (self.rt.task.agent.plugins if self.rt.task.agent else None) or [] + if isinstance(staged_plugins, list): + for original, staged in zip(original_plugins, staged_plugins, strict=False): + raw = original.get("path") if isinstance(original, dict) else None + if not raw or not isinstance(staged, dict): + continue + resolved = str(Path(os.path.expandvars(os.path.expanduser(raw))).resolve()) + public_target = self._host_plugin_to_agent_paths.get(resolved) + if public_target is not None: + staged["path"] = public_target + return rewritten + async def _stream_container_output(self, proc: asyncio.subprocess.Process, log_fh: TextIO) -> int: """Stream the container's stdout, returning its exit code. @@ -1067,7 +1327,7 @@ def _assert_runtime_image(self, image: str, dockerfile: Path) -> None: + "task-specific layers on top. See docs/DOCKER_ISOLATION.md." ) - def _build_argv( + def _build_argv( # noqa: PLR0912, PLR0915 - one ordered rendering pipeline mirrors docker-run argv self, input_dir: Path, output_dir: Path, *, container_name: str, image: str | None = None ) -> list[str]: cfg = self._docker_config @@ -1078,7 +1338,10 @@ def _build_argv( if image is None: image = cfg.image - argv: list[str] = ["docker", "run", "--rm", "--name", container_name] + # tini as PID 1 reaps orphaned grandchildren. The harness also scans + # and kills the dedicated agent UID before finalization; --init keeps a + # double-forked process from surviving only as an unreapable zombie. + argv: list[str] = ["docker", "run", "--rm", "--init", "--name", container_name] # Pin the framework entrypoint at run time rather than trusting whatever # the task image baked into ENTRYPOINT. This makes the orchestrator launch @@ -1099,6 +1362,8 @@ def _build_argv( argv += ["--cpus", str(self._limits.max_cpus)] if self._limits.max_pids is not None: argv += ["--pids-limit", str(self._limits.max_pids)] + elif cfg.agent_isolation: + argv += ["--pids-limit", str(DEFAULT_AGENT_ISOLATION_MAX_PIDS)] # Forward environment variables: explicit allowlist (optionally extended via env_passthrough_extra). # `--env VAR` (name-only) tells docker to copy the value from our current env at @@ -1113,11 +1378,21 @@ def _build_argv( for env_var in merged_allowlist: # LITELLM_BASE_URL / LITELLM_COST_LOG are forwarded below with a value # rewrite (host alias / absolute mount path), not name-only. - if env_var in ("LITELLM_BASE_URL", "LITELLM_COST_LOG"): + if env_var in ("LITELLM_BASE_URL", "LITELLM_COST_LOG", "SKILLS_REPO_PATH"): + continue + if cfg.agent_isolation and env_var == "HOME": continue if env_var in os.environ: argv += ["--env", env_var] + if cfg.agent_isolation and (skills_repo := os.environ.get("SKILLS_REPO_PATH")): + resolved_skills = str(Path(skills_repo).expanduser().resolve()) + private_skills = self._host_to_private_paths.get(resolved_skills) + if private_skills is not None: + argv += ["--env", f"SKILLS_REPO_PATH={private_skills}"] + else: + logger.debug("Not forwarding unstaged SKILLS_REPO_PATH into protected container: %s", resolved_skills) + # LITELLM_BASE_URL points at a proxy on the HOST. A bridge-network container # can't reach the host's loopback, so rewrite localhost/127.0.0.1 to the # docker host alias and publish that alias (`--add-host`) for Linux parity @@ -1151,6 +1426,10 @@ def _build_argv( # sandbox: Codex's Landlock-backed read-only / workspace-write sandboxes # can't initialize inside a container and otherwise fail writes silently. argv += ["--env", "CODER_EVAL_IN_CONTAINER=1"] + if cfg.agent_isolation: + argv += ["--env", "CODER_EVAL_AGENT_ISOLATION=1"] + if self.rt.task.sandbox.protected_mocks: + argv += ["--env", "CODER_EVAL_AGENT_ALLOW_RPC=1"] # Hard-disable telemetry INSIDE the container. The app ships a baked-in # default connection string, so without this the in-container orchestrator @@ -1173,7 +1452,7 @@ def _build_argv( host_task_dir: Path | None = None if self.rt.task_file: host_task_dir = self.rt.task_file.parent.resolve() - argv += ["-v", f"{host_task_dir}:{host_task_dir}:ro"] + argv += ["-v", f"{host_task_dir}:{CONTAINER_TASK_DIR}:ro"] # Forward the host's Claude Code OAuth state so the in-container CLI # inherits the same login as the host. We mount a *throwaway lean copy* # of ~/.claude (made by _prepare_host_mounts) read-WRITE at the host's @@ -1184,7 +1463,15 @@ def _build_argv( # doesn't exist or the mount is opted out (CODER_EVAL_NO_CLAUDE_MOUNT=1). if self._claude_mount_src is not None: host_claude_dir = Path.home() / ".claude" - argv += ["-v", f"{self._claude_mount_src}:{host_claude_dir}"] + claude_target = Path(AGENT_HOME) / ".claude" if cfg.agent_isolation else host_claude_dir + argv += ["-v", f"{self._claude_mount_src}:{claude_target}"] + + for source, target in self._agent_plugin_mounts: + argv += ["-v", f"{source.resolve()}:{target}:ro"] + for source, target in self._private_source_mounts: + argv += ["-v", f"{source.resolve()}:{target}:ro"] + if self._mock_fixture_mount is not None: + argv += ["-v", f"{self._mock_fixture_mount.resolve()}:/opt/coder-eval/mock/fixtures:ro"] # Auto-mount host paths the task references so they resolve inside # the container at the *same* path they have on the host. @@ -1226,22 +1513,24 @@ def _auto_mount(raw_path: str | None, *, dir_only: bool = True) -> None: argv.extend(["-v", f"{target}:{target}:ro"]) plugins = (self.rt.task.agent.plugins if self.rt.task.agent else None) or [] - for plugin in plugins: - _auto_mount(plugin.get("path") if isinstance(plugin, dict) else None) + if not cfg.agent_isolation: + for plugin in plugins: + _auto_mount(plugin.get("path") if isinstance(plugin, dict) else None) from coder_eval.models import TemplateDirSource sandbox_cfg = self.rt.task.sandbox - for source in (sandbox_cfg.template_sources or []) if sandbox_cfg else []: - if isinstance(source, TemplateDirSource): - _auto_mount(source.path) + if not cfg.agent_isolation: + for source in (sandbox_cfg.template_sources or []) if sandbox_cfg else []: + if isinstance(source, TemplateDirSource): + _auto_mount(source.path) # Defensive: system_prompt_file is normally inlined into # system_prompt by load_task / experiment resolution, but a variant # could conceivably inject an absolute path that survives. Cover # that path so the in-container Orchestrator can read it. agent_cfg = self.rt.task.agent - if agent_cfg and agent_cfg.system_prompt_file: + if not cfg.agent_isolation and agent_cfg and agent_cfg.system_prompt_file: _auto_mount(agent_cfg.system_prompt_file, dir_only=False) # reference.file / reference.directory: if a task ships absolute @@ -1249,7 +1538,7 @@ def _auto_mount(raw_path: str | None, *, dir_only: bool = True) -> None: # ``..``), they must be mounted explicitly. Relative paths under # task_dir are already covered by the symmetric task_dir mount. reference = self.rt.task.reference - if reference is not None: + if not cfg.agent_isolation and reference is not None: _auto_mount(reference.file, dir_only=False) _auto_mount(reference.directory) for mount in cfg.extra_mounts: @@ -1261,7 +1550,12 @@ def _auto_mount(raw_path: str | None, *, dir_only: bool = True) -> None: # runs the agent there). NO bind mount targets it -- capture is a copy-out # (see Orchestrator._cleanup), not a mount, so baked inputs/HOME survive. if self._workspace_dir is not None: - _assert_workspace_not_reserved(self._workspace_dir) + # Under isolation the workspace is the framework's OWN agent dir + # (assigned above, not task-authored), and that path is deliberately + # in RESERVED_CONTAINER_DIRS. The assertion guards task/image-supplied + # values, so exempt exactly the isolation-managed constant. + if not (cfg.agent_isolation and self._workspace_dir == CONTAINER_AGENT_WORK_DIR): + _assert_workspace_not_reserved(self._workspace_dir) argv += ["-w", self._workspace_dir] argv += [image] @@ -1271,7 +1565,7 @@ def _auto_mount(raw_path: str | None, *, dir_only: bool = True) -> None: argv += ["-v"] argv += ["--output", str(CONTAINER_OUTPUT_DIR)] if host_task_dir is not None: - argv += ["--task-dir", str(host_task_dir)] + argv += ["--task-dir", str(CONTAINER_TASK_DIR)] return argv diff --git a/src/coder_eval/models/__init__.py b/src/coder_eval/models/__init__.py index 51504e92..2d8335da 100644 --- a/src/coder_eval/models/__init__.py +++ b/src/coder_eval/models/__init__.py @@ -19,10 +19,25 @@ # Container paths (leaf constants; re-exported so consumers obey CE001) from coder_eval.models.container_paths import ( + AGENT_GID, + AGENT_HOME, + AGENT_UID, + AGENT_USERNAME, + CONTAINER_AGENT_SKILLS_DIR, + CONTAINER_AGENT_WORK_DIR, + CONTAINER_CLAUDE_SHIM, + CONTAINER_DROP_SHIM, + CONTAINER_GRADER_DIR, CONTAINER_INPUT_DIR, CONTAINER_OUTPUT_DIR, + CONTAINER_PRIVATE_PLUGIN_DIR, CONTAINER_TASK_DIR, CONTAINER_WORK_DIR, + MOCK_RPC_GID, + MOCK_RPC_GROUP, + MOCKD_GID, + MOCKD_UID, + MOCKD_USERNAME, RESERVED_CONTAINER_DIRS, ) @@ -160,6 +175,7 @@ DockerBuildConfig, DockerDriverConfig, NodeEnvConfig, + ProtectedMockConfig, PythonEnvConfig, RecordedCli, ResourceLimits, @@ -265,14 +281,30 @@ "TemplateSource", # Sandbox "DockerBuildConfig", + "AGENT_GID", + "AGENT_HOME", + "AGENT_UID", + "AGENT_USERNAME", + "CONTAINER_AGENT_SKILLS_DIR", + "CONTAINER_AGENT_WORK_DIR", + "CONTAINER_CLAUDE_SHIM", + "CONTAINER_DROP_SHIM", + "CONTAINER_GRADER_DIR", "CONTAINER_INPUT_DIR", "CONTAINER_OUTPUT_DIR", + "CONTAINER_PRIVATE_PLUGIN_DIR", "CONTAINER_TASK_DIR", "CONTAINER_WORK_DIR", + "MOCK_RPC_GID", + "MOCK_RPC_GROUP", + "MOCKD_GID", + "MOCKD_UID", + "MOCKD_USERNAME", "RESERVED_CONTAINER_DIRS", "DockerDriverConfig", "NodeEnvConfig", "PythonEnvConfig", + "ProtectedMockConfig", "SandboxConfig", "RecordedCli", "RECORD_CLI_DIR", diff --git a/src/coder_eval/models/container_paths.py b/src/coder_eval/models/container_paths.py index 0114fe42..4232751c 100644 --- a/src/coder_eval/models/container_paths.py +++ b/src/coder_eval/models/container_paths.py @@ -14,13 +14,48 @@ CONTAINER_WORK_DIR = "/work" -CONTAINER_INPUT_DIR = "/work/input" -CONTAINER_OUTPUT_DIR = "/work/output" -CONTAINER_TASK_DIR = "/work/task_dir" +CONTAINER_AGENT_WORK_DIR = "/work/agent" + +# The evaluated agent shares the container with the trusted harness, but cannot +# traverse this root-owned directory. Hidden task, grader, reference, fixture, +# and result material is mounted below it rather than at agent-readable /work +# paths. +CONTAINER_GRADER_DIR = "/opt/coder-eval/grader" +CONTAINER_INPUT_DIR = f"{CONTAINER_GRADER_DIR}/input" +CONTAINER_OUTPUT_DIR = f"{CONTAINER_GRADER_DIR}/output" +CONTAINER_TASK_DIR = f"{CONTAINER_GRADER_DIR}/task_dir" +CONTAINER_PRIVATE_PLUGIN_DIR = f"{CONTAINER_GRADER_DIR}/plugins" + +# Manifest-verified skill projections are the only plugin trees exposed to the +# evaluated agent. +CONTAINER_AGENT_SKILLS_DIR = "/opt/coder-eval/agent-skills" + +AGENT_UID = 2000 +AGENT_GID = 2000 +AGENT_USERNAME = "agent" +AGENT_HOME = "/home/agent" +MOCKD_UID = 2100 +MOCKD_GID = 2100 +MOCKD_USERNAME = "mockd" +MOCK_RPC_GID = 2200 +MOCK_RPC_GROUP = "uip-rpc" + +CONTAINER_DROP_SHIM = "/usr/local/bin/coder_eval_drop_privilege.sh" +CONTAINER_CLAUDE_SHIM = "/usr/local/bin/coder_eval_claude_agent.sh" # Paths a task's WORKDIR must never collide with: the container root and every -# framework-owned mount under /work. Consumed by SandboxConfig's working_dir -# validator (models/sandbox.py) and re-asserted host-side in docker_runner. +# framework-owned public or private mount. Consumed by SandboxConfig's +# working_dir validator and re-asserted host-side in docker_runner. RESERVED_CONTAINER_DIRS = frozenset( - {"/", CONTAINER_WORK_DIR, CONTAINER_INPUT_DIR, CONTAINER_OUTPUT_DIR, CONTAINER_TASK_DIR} + { + "/", + CONTAINER_WORK_DIR, + CONTAINER_AGENT_WORK_DIR, + CONTAINER_GRADER_DIR, + CONTAINER_INPUT_DIR, + CONTAINER_OUTPUT_DIR, + CONTAINER_TASK_DIR, + CONTAINER_PRIVATE_PLUGIN_DIR, + CONTAINER_AGENT_SKILLS_DIR, + } ) diff --git a/src/coder_eval/models/sandbox.py b/src/coder_eval/models/sandbox.py index d2083d09..46ab04ec 100644 --- a/src/coder_eval/models/sandbox.py +++ b/src/coder_eval/models/sandbox.py @@ -195,6 +195,14 @@ class DockerDriverConfig(BaseModel): default="bridge", description="Container network. 'bridge' for tasks needing LLM/pkg access; 'none' for fully sealed runs.", ) + agent_isolation: bool = Field( + default=True, + description=( + "Run the evaluated agent under the image's dedicated unprivileged UID/GID and expose local plugins " + "only through manifest-verified bundles. Enabled by default. Set false only for temporary migration " + "of a trusted task; false is not a secure evaluation boundary." + ), + ) working_dir: str | None = Field( default=None, description=( @@ -398,6 +406,55 @@ def validate_tool_name(cls, v: str) -> str: return v +class ProtectedMockConfig(BaseModel): + """Fixture-backed CLI served across the protected mock Unix socket. + + ``fixture`` is read by the mockd identity, never by the evaluated agent. + The fixture schema maps exact argv lists to bounded stdout/stderr/exit-code + responses; it exposes no general file or search operation. + """ + + model_config = ConfigDict(extra="forbid") + + tool: str = Field(description="Bare executable name presented to the agent (for example, 'uip')") + fixture: str = Field(description="Path to the protected exact-command response fixture") + max_requests: int = Field(default=100, ge=1, le=10_000, description="Per-run request budget for this tool") + passthrough_argv_prefixes: list[list[str]] = Field( + default_factory=list, + max_length=16, + description=( + "Public argv prefixes that mockd may proxy to the real tool, for example [['docsai', 'ask']]. " + "All other invocations remain fixture-backed or receive the fixed default response." + ), + ) + + @field_validator("tool") + @classmethod + def validate_tool_name(cls, value: str) -> str: + if not value or value != value.strip() or value in {".", ".."} or "/" in value or "\\" in value: + raise ValueError("protected mock tool must be a non-empty bare executable name") + if value.lower().endswith((".cmd", ".bat", ".exe")): + raise ValueError("protected mock tool must not include a platform executable suffix") + return value + + @field_validator("passthrough_argv_prefixes") + @classmethod + def validate_passthrough_prefixes(cls, prefixes: list[list[str]]) -> list[list[str]]: + normalized: list[list[str]] = [] + seen: set[tuple[str, ...]] = set() + for prefix in prefixes: + if not prefix or len(prefix) > 8: + raise ValueError("protected mock passthrough prefixes must contain 1 to 8 argv tokens") + if any(not isinstance(token, str) or not token or len(token) > 256 for token in prefix): + raise ValueError("protected mock passthrough prefix tokens must be non-empty strings up to 256 chars") + key = tuple(prefix) + if key in seen: + raise ValueError("protected mock passthrough prefixes must be unique") + seen.add(key) + normalized.append(list(prefix)) + return normalized + + class SandboxConfig(BaseModel): """Configuration for the sandboxed execution environment. @@ -451,6 +508,15 @@ class SandboxConfig(BaseModel): ), ) + protected_mocks: list[ProtectedMockConfig] | None = MergeField( + strategy="replace", + default=None, + description=( + "Docker-only fixture-backed mock CLIs served by the isolated mockd UID over a Unix socket. " + "The agent receives a thin client; fixture bytes are never copied into its workspace." + ), + ) + record_cli: list[RecordedCli] | None = MergeField( strategy="replace", default=None, @@ -489,4 +555,14 @@ def validate_template_sources(self) -> SandboxConfig: """Validate template sources configuration.""" if self.template_sources: validate_template_sources_list(self.template_sources) + if self.protected_mocks: + if self.driver != "docker": + raise ValueError("sandbox.protected_mocks requires driver: docker") + tools = [mock.tool for mock in self.protected_mocks] + if len(tools) != len(set(tools)): + raise ValueError("sandbox.protected_mocks tool names must be unique") + recorded = {spec.tool for spec in self.record_cli or []} + overlap = sorted(recorded & set(tools)) + if overlap: + raise ValueError(f"protected_mocks and record_cli cannot both provide: {overlap}") return self diff --git a/src/coder_eval/orchestration/experiment.py b/src/coder_eval/orchestration/experiment.py index f75c585e..96c967d7 100644 --- a/src/coder_eval/orchestration/experiment.py +++ b/src/coder_eval/orchestration/experiment.py @@ -43,6 +43,7 @@ expand_dataset, load_task, resolve_agent_system_prompt, + resolve_protected_mock_paths, resolve_template_source_paths, resolve_variant_initial_prompt_file, ) @@ -531,6 +532,8 @@ def resolve_task_files( # Resolve relative template_sources paths if task.sandbox.template_sources: resolve_template_source_paths(task.sandbox.template_sources, exp_dir) + if task.sandbox.protected_mocks: + resolve_protected_mock_paths(task, exp_dir) def resolve_all_tasks( diff --git a/src/coder_eval/orchestration/task_loader.py b/src/coder_eval/orchestration/task_loader.py index bff0c13a..9f1162ec 100644 --- a/src/coder_eval/orchestration/task_loader.py +++ b/src/coder_eval/orchestration/task_loader.py @@ -61,6 +61,7 @@ def load_task(task_file: Path) -> tuple[TaskDefinition, str]: task = TaskDefinition(**task_data) # Resolve relative template paths task = resolve_template_paths(task, task_file.parent) + task = resolve_protected_mock_paths(task, task_file.parent) task = resolve_initial_prompt_file(task, task_file.parent) task = resolve_system_prompt_files(task, task_file.parent) task = resolve_dockerfile_path(task, task_file.parent) @@ -140,6 +141,19 @@ def resolve_template_paths(task: TaskDefinition, base_dir: Path) -> TaskDefiniti return task +def resolve_protected_mock_paths(task: TaskDefinition, base_dir: Path) -> TaskDefinition: + """Resolve and validate protected mock fixtures relative to the task YAML.""" + + for mock in task.sandbox.protected_mocks or []: + fixture = Path(os.path.expandvars(mock.fixture)) + if not fixture.is_absolute(): + fixture = (base_dir / fixture).resolve() + if not fixture.is_file(): + raise FileNotFoundError(f"Protected mock fixture not found: {fixture}") + mock.fixture = str(fixture) + return task + + def resolve_dockerfile_path(task: TaskDefinition, base_dir: Path) -> TaskDefinition: """Resolve ``sandbox.docker.dockerfile_path`` to an absolute path, in place. diff --git a/src/coder_eval/orchestrator.py b/src/coder_eval/orchestrator.py index 8640dd7f..36eee20d 100644 --- a/src/coder_eval/orchestrator.py +++ b/src/coder_eval/orchestrator.py @@ -473,6 +473,10 @@ async def run(self) -> EvaluationResult: # the run as FinalStatus.ERROR; _run_post_run_commands and # _cleanup still execute via the finally block. await self._run_pre_run_commands() + # Trusted pre-run commands may create root-owned files. Re-grant + # only the disposable sandbox before the first model turn; + # hidden harness mounts live outside this tree. + await self._grant_current_sandbox_to_agent() # Enforce task-level timeout via an OS-thread watchdog that # SIGKILLs the in-flight CLI subprocess AND cancels this @@ -597,6 +601,11 @@ def _kill_agent_subprocess_sync() -> None: # awaits below run normally after the CancelledError is caught. teardown_interrupt: BaseException | None = None try: + # Protected Docker runs stop the SDK process and kill any + # same-UID descendants before post-run commands, capture, + # or task.json publication. Background shells must not + # observe trusted finalization or keep using the mock RPC. + await self._stop_isolated_agent_processes() # BEFORE post-run/cleanup: needs the live sandbox to resolve # the agent-aligned `uip`, and post-task tool state on disk. self._refresh_runtime_tool_versions() @@ -1071,6 +1080,10 @@ async def _setup_sandbox() -> Any: assert self.result is not None, "Result not initialized" self.result.sandbox_path = str(sandbox_dir) + # Root prepares templates and dependencies, then hands the generated + # workspace—not any raw task/plugin/reference source—to the agent UID. + await self._grant_current_sandbox_to_agent() + # Determine API routing from settings.api_backend enum self.route = resolve_route(settings) self.eval_route = resolve_evaluation_route(settings, self.route) @@ -2260,6 +2273,27 @@ async def _run_post_run_commands(self) -> None: return await self._run_command_list(self.task.post_run, self.result.post_run_results, "post_run") + async def _stop_isolated_agent_processes(self) -> None: + """Stop the SDK and residual same-UID children before finalization.""" + + from .isolation.agent_identity import agent_isolation_enabled, terminate_agent_processes + + if not agent_isolation_enabled(): + return + if self.agent is not None: + with suppress(Exception): + await self.agent.stop() + await asyncio.to_thread(terminate_agent_processes) + + async def _grant_current_sandbox_to_agent(self) -> None: + """Transfer only the generated sandbox tree to the agent identity.""" + + if self.sandbox is None or self.sandbox.sandbox_dir is None: + return + from .isolation.agent_identity import grant_agent_workspace + + await asyncio.to_thread(grant_agent_workspace, self.sandbox.sandbox_dir) + async def _cleanup(self) -> None: """Clean up all resources.""" # Stop agent diff --git a/src/coder_eval/plugin_bundle.py b/src/coder_eval/plugin_bundle.py new file mode 100644 index 00000000..4d6073c8 --- /dev/null +++ b/src/coder_eval/plugin_bundle.py @@ -0,0 +1,257 @@ +"""Build a manifest-verified, agent-visible projection of a local plugin. + +Local skill repositories commonly contain both public instructions and hidden +graders, references, fixtures, or resolution notes. Docker evaluations must +never mount that complete tree at an agent-readable path. This module copies +only the plugin discovery surface and records every copied file in a digest +manifest. Any violation fails closed; callers must not fall back to the raw +source path. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import shutil +from dataclasses import dataclass +from fnmatch import fnmatchcase +from pathlib import Path + + +PLUGIN_AGENT_ALLOWED_SUBDIRS = frozenset({"skills", "commands", "agents", ".claude-plugin", "hooks"}) +HIDDEN_MATERIAL_FILE_PATTERNS: tuple[str, ...] = ("resolution.md", "check_*.py") +MANIFEST_SUFFIX = ".manifest.json" + + +class PluginBundleError(RuntimeError): + """The sanitized plugin bundle could not be built or verified safely.""" + + +@dataclass(frozen=True) +class BundleManifest: + """Content inventory for one sanitized plugin bundle.""" + + source: str + files: dict[str, str] + symlinks: dict[str, str] + digest: str + + +def _hash_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _manifest_digest(files: dict[str, str], symlinks: dict[str, str]) -> str: + payload = json.dumps({"files": files, "symlinks": symlinks}, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def _is_hidden_material(path: Path) -> bool: + name = path.name.lower() + return any(fnmatchcase(name, pattern) for pattern in HIDDEN_MATERIAL_FILE_PATTERNS) + + +def _validate_symlink(link: Path, source_root: Path) -> str: + """Return a safe relative link target or raise. + + Absolute links are rejected even when they currently resolve inside the + source tree: recreating one in the bundle would point back to the raw host + checkout. Relative links must be unbroken and resolve to either the source + root or one of the explicitly included top-level subtrees. Recreating the + same relative target therefore cannot make excluded source content visible. + """ + + raw_target = os.readlink(link) + if Path(raw_target).is_absolute(): + raise PluginBundleError(f"absolute plugin symlink is not allowed: {link} -> {raw_target!r}") + + try: + root = source_root.resolve(strict=True) + resolved = link.resolve(strict=True) + except (OSError, RuntimeError) as exc: + raise PluginBundleError(f"broken or looping plugin symlink is not allowed: {link} -> {raw_target!r}") from exc + + if resolved != root and root not in resolved.parents: + raise PluginBundleError(f"plugin symlink escapes its source root: {link} -> {raw_target!r}") + + if resolved != root: + relative_target = resolved.relative_to(root) + if not relative_target.parts or relative_target.parts[0] not in PLUGIN_AGENT_ALLOWED_SUBDIRS: + raise PluginBundleError( + f"plugin symlink targets excluded content: {link} -> {raw_target!r} ({relative_target.as_posix()})" + ) + return raw_target + + +def build_manifest(source: Path) -> BundleManifest: + """Inventory the allowed plugin projection and fail on hidden material.""" + + try: + source = source.resolve(strict=True) + except OSError as exc: + raise PluginBundleError(f"plugin source does not exist: {source}") from exc + if not source.is_dir(): + raise PluginBundleError(f"plugin source is not a directory: {source}") + + files: dict[str, str] = {} + symlinks: dict[str, str] = {} + + def record(path: Path) -> None: + relative = path.relative_to(source).as_posix() + if path.is_symlink(): + symlinks[relative] = _validate_symlink(path, source) + return + if _is_hidden_material(path): + patterns = ", ".join(HIDDEN_MATERIAL_FILE_PATTERNS) + raise PluginBundleError( + f"hidden grading material appears inside an agent-visible plugin subtree: {relative} " + + f"(forbidden patterns: {patterns})" + ) + if path.is_file(): + files[relative] = _hash_file(path) + + for name in sorted(PLUGIN_AGENT_ALLOWED_SUBDIRS): + top = source / name + if top.is_symlink() or top.is_file(): + record(top) + continue + if not top.is_dir(): + continue + for root_name, dirnames, filenames in os.walk(top, followlinks=False): + root = Path(root_name) + for dirname in sorted(dirnames): + child = root / dirname + if child.is_symlink(): + record(child) + for filename in sorted(filenames): + record(root / filename) + + return BundleManifest( + source=str(source), + files=dict(sorted(files.items())), + symlinks=dict(sorted(symlinks.items())), + digest=_manifest_digest(files, symlinks), + ) + + +def manifest_path_for(bundle_dir: Path) -> Path: + """Keep the inventory beside, rather than inside, the public bundle.""" + + return bundle_dir.with_name(bundle_dir.name + MANIFEST_SUFFIX) + + +def stage_bundle(source: Path, bundle_dir: Path) -> BundleManifest: + """Create and self-verify a sanitized plugin bundle. + + ``bundle_dir`` must not already contain data. This prevents a prior task or + failed attempt from leaving undeclared files in a newly staged projection. + """ + + if bundle_dir.exists() and any(bundle_dir.iterdir()): + raise PluginBundleError(f"plugin bundle destination is not empty: {bundle_dir}") + + manifest = build_manifest(source) + bundle_dir.mkdir(parents=True, exist_ok=True) + source_root = Path(manifest.source) + try: + for relative in manifest.files: + destination = bundle_dir / relative + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source_root / relative, destination) + for relative, target in manifest.symlinks.items(): + original = source_root / relative + destination = bundle_dir / relative + destination.parent.mkdir(parents=True, exist_ok=True) + os.symlink(target, destination, target_is_directory=original.is_dir()) + except OSError as exc: + raise PluginBundleError(f"failed to copy sanitized plugin bundle {source_root}: {exc}") from exc + + # The bundle is disposable public staging. Normalize away restrictive + # source modes so the unrelated agent UID can read it through a read-only + # bind mount; never apply these changes to ``source_root``. + try: + bundle_dir.chmod(0o555) + for root_name, dirnames, filenames in os.walk(bundle_dir, followlinks=False): + root = Path(root_name) + for dirname in dirnames: + child = root / dirname + if not child.is_symlink(): + child.chmod(0o555) + for filename in filenames: + child = root / filename + if not child.is_symlink(): + child.chmod(0o444) + except OSError as exc: + raise PluginBundleError(f"failed to normalize public bundle permissions at {bundle_dir}: {exc}") from exc + + manifest_path_for(bundle_dir).write_text( + json.dumps( + { + "source": manifest.source, + "files": manifest.files, + "symlinks": manifest.symlinks, + "digest": manifest.digest, + }, + indent=2, + sort_keys=True, + ), + encoding="utf-8", + ) + return verify_bundle(bundle_dir) + + +def verify_bundle(bundle_dir: Path) -> BundleManifest: + """Verify manifest integrity and both directions of the staged inventory.""" + + manifest_path = manifest_path_for(bundle_dir) + try: + raw = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest = BundleManifest( + source=str(raw["source"]), + files={str(k): str(v) for k, v in raw["files"].items()}, + symlinks={str(k): str(v) for k, v in raw["symlinks"].items()}, + digest=str(raw["digest"]), + ) + except (OSError, KeyError, TypeError, ValueError, json.JSONDecodeError) as exc: + raise PluginBundleError(f"plugin bundle manifest is unreadable: {manifest_path}: {exc}") from exc + + if _manifest_digest(manifest.files, manifest.symlinks) != manifest.digest: + raise PluginBundleError(f"plugin bundle manifest digest is invalid: {manifest_path}") + + actual_files: dict[str, str] = {} + actual_symlinks: dict[str, str] = {} + for root_name, dirnames, filenames in os.walk(bundle_dir, followlinks=False): + root = Path(root_name) + for dirname in dirnames: + child = root / dirname + if child.is_symlink(): + actual_symlinks[child.relative_to(bundle_dir).as_posix()] = os.readlink(child) + for filename in filenames: + child = root / filename + relative = child.relative_to(bundle_dir).as_posix() + if child.is_symlink(): + actual_symlinks[relative] = os.readlink(child) + else: + actual_files[relative] = _hash_file(child) + + actual_files = dict(sorted(actual_files.items())) + actual_symlinks = dict(sorted(actual_symlinks.items())) + if actual_files != manifest.files or actual_symlinks != manifest.symlinks: + declared_paths = set(manifest.files) | set(manifest.symlinks) + actual_paths = set(actual_files) | set(actual_symlinks) + missing = sorted(declared_paths - actual_paths)[:5] + added = sorted(actual_paths - declared_paths)[:5] + changed = sorted( + key for key in set(actual_files) & set(manifest.files) if actual_files[key] != manifest.files[key] + )[:5] + raise PluginBundleError( + f"plugin bundle differs from its manifest: {bundle_dir} " + + f"(missing={missing}, added={added}, changed={changed}, " + + f"symlink_drift={actual_symlinks != manifest.symlinks})" + ) + return manifest diff --git a/src/coder_eval/protected_mock/__init__.py b/src/coder_eval/protected_mock/__init__.py new file mode 100644 index 00000000..033cd467 --- /dev/null +++ b/src/coder_eval/protected_mock/__init__.py @@ -0,0 +1 @@ +"""Protected fixture-backed CLI service used by Docker agent isolation.""" diff --git a/src/coder_eval/protected_mock/client.py b/src/coder_eval/protected_mock/client.py new file mode 100644 index 00000000..e18e5506 --- /dev/null +++ b/src/coder_eval/protected_mock/client.py @@ -0,0 +1,105 @@ +"""Thin, agent-visible client for the protected exact-command mock service.""" + +from __future__ import annotations + +import json +import os +import socket +import sys +import time +from pathlib import Path + +from .protocol import ( + CLIENT_TIMEOUT_SECONDS, + MAX_REQUEST_BYTES, + MAX_RESPONSE_BYTES, + PROTOCOL_VERSION, + SOCKET_PATH, +) + + +def _record(tool: str, argv: list[str], exit_code: int) -> None: + """Best-effort compatibility with the existing cli_called JSONL schema.""" + + raw_path = os.environ.get("CODER_EVAL_MOCK_CALL_LOG") + if not raw_path: + return + entry = {"ts": round(time.time(), 3), "tool": tool, "argv": argv, "exit": exit_code} + try: + with Path(raw_path).open("a", encoding="utf-8", newline="\n") as stream: + stream.write(json.dumps(entry, ensure_ascii=True, separators=(",", ":")) + "\n") + except OSError as exc: + sys.stderr.write(f"protected mock client: invocation log failed: {exc!r}\n") + + +def _receive_line(connection: socket.socket) -> bytes: + chunks: list[bytes] = [] + total = 0 + while True: + chunk = connection.recv(min(65536, MAX_RESPONSE_BYTES + 1 - total)) + if not chunk: + break + chunks.append(chunk) + total += len(chunk) + if total > MAX_RESPONSE_BYTES: + raise RuntimeError("response exceeded size limit") + if b"\n" in chunk: + break + return b"".join(chunks).split(b"\n", 1)[0] + + +def invoke(tool: str, argv: list[str]) -> int: + request = ( + json.dumps( + {"version": PROTOCOL_VERSION, "tool": tool, "argv": argv}, + ensure_ascii=True, + separators=(",", ":"), + ).encode("utf-8") + + b"\n" + ) + if len(request) > MAX_REQUEST_BYTES: + sys.stderr.write("protected mock client: request exceeds size limit\n") + _record(tool, argv, 125) + return 125 + + try: + af_unix = getattr(socket, "AF_UNIX", None) + if af_unix is None: + raise RuntimeError("Unix-domain sockets are unavailable") + with socket.socket(af_unix, socket.SOCK_STREAM) as connection: + connection.settimeout(CLIENT_TIMEOUT_SECONDS) + connection.connect(SOCKET_PATH) + connection.sendall(request) + raw_response = _receive_line(connection) + response = json.loads(raw_response.decode("utf-8")) + if not isinstance(response, dict) or response.get("version") != PROTOCOL_VERSION: + raise RuntimeError("invalid response envelope") + exit_code = response.get("exit_code") + stdout = response.get("stdout") + stderr = response.get("stderr") + if not isinstance(exit_code, int) or not 0 <= exit_code <= 255: + raise RuntimeError("invalid response exit_code") + if not isinstance(stdout, str) or not isinstance(stderr, str): + raise RuntimeError("invalid response streams") + except (OSError, UnicodeError, ValueError, RuntimeError) as exc: + sys.stderr.write(f"protected mock client: service unavailable or invalid response: {exc}\n") + _record(tool, argv, 125) + return 125 + + if stdout: + sys.stdout.write(stdout) + if stderr: + sys.stderr.write(stderr) + _record(tool, argv, exit_code) + return exit_code + + +def main() -> int: + if len(sys.argv) < 2: + sys.stderr.write("protected mock client: missing tool name\n") + return 64 + return invoke(sys.argv[1], sys.argv[2:]) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/coder_eval/protected_mock/protocol.py b/src/coder_eval/protected_mock/protocol.py new file mode 100644 index 00000000..81d7c46c --- /dev/null +++ b/src/coder_eval/protected_mock/protocol.py @@ -0,0 +1,13 @@ +"""Constants shared by the protected mock client, server, and runtime.""" + +from __future__ import annotations + + +PROTOCOL_VERSION = 1 +SOCKET_PATH = "/run/coder-eval/uip.sock" +CLIENT_EXECUTABLE = "/usr/local/bin/coder_eval_mock_client" +SERVER_LAUNCHER = "/usr/local/bin/coder_eval_mockd.sh" +CONTAINER_FIXTURE_DIR = "/opt/coder-eval/mock/fixtures" +MAX_REQUEST_BYTES = 64 * 1024 +MAX_RESPONSE_BYTES = 1024 * 1024 +CLIENT_TIMEOUT_SECONDS = 5.0 diff --git a/src/coder_eval/protected_mock/runtime.py b/src/coder_eval/protected_mock/runtime.py new file mode 100644 index 00000000..f67c850b --- /dev/null +++ b/src/coder_eval/protected_mock/runtime.py @@ -0,0 +1,47 @@ +"""Root-harness lifecycle wrapper for the protected mockd subprocess.""" + +from __future__ import annotations + +import contextlib +import os +import subprocess +import sys +import time +from collections.abc import Iterator +from pathlib import Path + +from .protocol import SERVER_LAUNCHER, SOCKET_PATH + + +@contextlib.contextmanager +def running_mock_server(config_path: Path | None) -> Iterator[None]: + if config_path is None: + yield + return + + process = subprocess.Popen( + [SERVER_LAUNCHER, sys.executable, "-m", "coder_eval.protected_mock.server", "--config", str(config_path)], + stdin=subprocess.DEVNULL, + ) + socket_path = Path(SOCKET_PATH) + try: + deadline = time.monotonic() + 5 + while time.monotonic() < deadline: + if process.poll() is not None: + raise RuntimeError(f"protected mockd exited during startup with code {process.returncode}") + if socket_path.exists(): + break + time.sleep(0.02) + else: + raise RuntimeError("protected mockd did not create its socket within 5 seconds") + yield + finally: + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=3) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=3) + with contextlib.suppress(OSError): + os.unlink(socket_path) diff --git a/src/coder_eval/protected_mock/server.py b/src/coder_eval/protected_mock/server.py new file mode 100644 index 00000000..2fbc1102 --- /dev/null +++ b/src/coder_eval/protected_mock/server.py @@ -0,0 +1,314 @@ +"""mockd: exact-command fixture service running as the private mock UID.""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import socket +import socketserver +import struct +import subprocess +import threading +from dataclasses import dataclass +from pathlib import Path +from typing import Any, cast + +from coder_eval.models import AGENT_UID, MOCK_RPC_GID + +from .protocol import MAX_REQUEST_BYTES, MAX_RESPONSE_BYTES, PROTOCOL_VERSION, SOCKET_PATH + + +@dataclass(frozen=True) +class CommandResponse: + exit_code: int + stdout: str + stderr: str + + +@dataclass +class ToolState: + responses: dict[tuple[str, ...], CommandResponse] + normalized_responses: dict[tuple[str, ...], CommandResponse] + default: CommandResponse + remaining: int + passthrough_prefixes: tuple[tuple[str, ...], ...] + passthrough_executable: str | None + passthrough_cache: dict[tuple[str, ...], CommandResponse] + + +PASSTHROUGH_TIMEOUT_SECONDS = 60 +_NOISE_VALUE_FLAGS = frozenset({"--output"}) + + +def _normalized_argv(argv: list[str]) -> tuple[str, ...]: + """Canonical finite-command key: flag form/order agnostic, never subset matching.""" + + expanded: list[str] = [] + for raw in argv: + if raw.startswith("-") and "=" in raw: + flag, value = raw.split("=", 1) + expanded.append(flag) + if value: + expanded.append(value) + else: + expanded.append(raw) + + cleaned: list[str] = [] + skip_next = False + for token in expanded: + if skip_next: + skip_next = False + continue + if token in _NOISE_VALUE_FLAGS: + skip_next = True + continue + cleaned.append(token) + return tuple(sorted(cleaned)) + + +def _response(raw: object, *, context: str) -> CommandResponse: + if not isinstance(raw, dict): + raise ValueError(f"{context} must be an object") + exit_code = raw.get("exit_code", 0) + stdout = raw.get("stdout", "") + stderr = raw.get("stderr", "") + if not isinstance(exit_code, int) or not 0 <= exit_code <= 255: + raise ValueError(f"{context}.exit_code must be an integer from 0 to 255") + if not isinstance(stdout, str) or not isinstance(stderr, str): + raise ValueError(f"{context} stdout/stderr must be strings") + encoded_size = len(stdout.encode("utf-8")) + len(stderr.encode("utf-8")) + if encoded_size > MAX_RESPONSE_BYTES // 2: + raise ValueError(f"{context} response exceeds the configured size limit") + return CommandResponse(exit_code=exit_code, stdout=stdout, stderr=stderr) + + +def _load_tool( + tool: str, + fixture_path: Path, + max_requests: int, + passthrough_prefixes: list[list[str]], +) -> ToolState: + raw = json.loads(fixture_path.read_text(encoding="utf-8")) + if not isinstance(raw, dict) or raw.get("version") != PROTOCOL_VERSION: + raise ValueError(f"fixture {fixture_path} must declare version {PROTOCOL_VERSION}") + entries = raw.get("responses") + if not isinstance(entries, list): + raise ValueError(f"fixture {fixture_path} responses must be a list") + responses: dict[tuple[str, ...], CommandResponse] = {} + normalized_responses: dict[tuple[str, ...], CommandResponse] = {} + for index, entry in enumerate(entries): + if not isinstance(entry, dict): + raise ValueError(f"fixture {fixture_path} response {index} must be an object") + argv = entry.get("argv") + if not isinstance(argv, list) or not all(isinstance(item, str) for item in argv): + raise ValueError(f"fixture {fixture_path} response {index}.argv must be a string list") + key = tuple(argv) + match_mode = entry.get("match_mode", "exact") + if match_mode not in {"exact", "normalized"}: + raise ValueError(f"fixture {fixture_path} response {index}.match_mode must be exact or normalized") + destination = responses if match_mode == "exact" else normalized_responses + command_key = key if match_mode == "exact" else _normalized_argv(argv) + if command_key in destination: + raise ValueError(f"fixture {fixture_path} contains duplicate argv {argv!r} for {match_mode} matching") + destination[command_key] = _response(entry, context=f"response {index}") + default = _response( + raw.get( + "default", + {"exit_code": 2, "stderr": "protected mock: command is not configured for this scenario\n"}, + ), + context="default", + ) + executable = shutil.which(tool) if passthrough_prefixes else None + if passthrough_prefixes and executable is None: + raise ValueError(f"protected mock passthrough tool is not installed: {tool}") + return ToolState( + responses=responses, + normalized_responses=normalized_responses, + default=default, + remaining=max_requests, + passthrough_prefixes=tuple(tuple(prefix) for prefix in passthrough_prefixes), + passthrough_executable=executable, + passthrough_cache={}, + ) + + +def load_config(config_path: Path) -> dict[str, ToolState]: + raw = json.loads(config_path.read_text(encoding="utf-8")) + if not isinstance(raw, dict) or raw.get("version") != PROTOCOL_VERSION: + raise ValueError(f"mock config must declare version {PROTOCOL_VERSION}") + tools = raw.get("tools") + if not isinstance(tools, list) or not tools: + raise ValueError("mock config tools must be a non-empty list") + loaded: dict[str, ToolState] = {} + for entry in tools: + if not isinstance(entry, dict): + raise ValueError("mock config tool entries must be objects") + tool = entry.get("tool") + fixture = entry.get("fixture") + max_requests = entry.get("max_requests") + passthrough_prefixes = entry.get("passthrough_argv_prefixes", []) + if not isinstance(tool, str) or not tool or tool in loaded: + raise ValueError("mock config tools must have unique non-empty names") + if not isinstance(fixture, str) or not isinstance(max_requests, int) or max_requests < 1: + raise ValueError(f"mock config entry for {tool!r} has invalid fixture or max_requests") + if not isinstance(passthrough_prefixes, list) or not all( + isinstance(prefix, list) and prefix and all(isinstance(token, str) and token for token in prefix) + for prefix in passthrough_prefixes + ): + raise ValueError(f"mock config entry for {tool!r} has invalid passthrough prefixes") + loaded[tool] = _load_tool(tool, Path(fixture), max_requests, passthrough_prefixes) + return loaded + + +_UnixStreamServer: Any = getattr(socketserver, "UnixStreamServer", object) + + +class ProtectedMockServer(socketserver.ThreadingMixIn, _UnixStreamServer): + daemon_threads = True + + def __init__(self, path: str, tools: dict[str, ToolState]) -> None: + self.tools = tools + self.budget_lock = threading.Lock() + self.passthrough_lock = threading.Lock() + super().__init__(path, ProtectedMockHandler) # pyright: ignore[reportCallIssue] + + def dispatch(self, tool: str, argv: list[str]) -> CommandResponse: + state = self.tools.get(tool) + if state is None: + return CommandResponse(127, "", "protected mock: unknown tool\n") + with self.budget_lock: + if state.remaining <= 0: + return CommandResponse(75, "", "protected mock: request budget exhausted\n") + state.remaining -= 1 + response = state.responses.get(tuple(argv)) + if response is None: + response = state.normalized_responses.get(_normalized_argv(argv)) + if response is not None: + return response + if any(tuple(argv[: len(prefix)]) == prefix for prefix in state.passthrough_prefixes): + return self._passthrough(state, argv) + return state.default + + def _passthrough(self, state: ToolState, argv: list[str]) -> CommandResponse: + key = tuple(argv) + with self.passthrough_lock: + cached = state.passthrough_cache.get(key) + if cached is not None: + return cached + if state.passthrough_executable is None: + return CommandResponse(69, "", "protected mock: passthrough is unavailable\n") + try: + result = subprocess.run( + [state.passthrough_executable, *argv], + stdin=subprocess.DEVNULL, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + check=False, + timeout=PASSTHROUGH_TIMEOUT_SECONDS, + ) + exit_code = result.returncode if 0 <= result.returncode <= 255 else 70 + response = CommandResponse(exit_code, result.stdout, result.stderr) + except (OSError, subprocess.SubprocessError): + response = CommandResponse(70, "", "protected mock: passthrough failed\n") + encoded_size = len(response.stdout.encode("utf-8")) + len(response.stderr.encode("utf-8")) + if encoded_size > MAX_RESPONSE_BYTES // 2: + response = CommandResponse(70, "", "protected mock: passthrough response exceeds size limit\n") + state.passthrough_cache[key] = response + return response + + +class ProtectedMockHandler(socketserver.StreamRequestHandler): + def handle(self) -> None: + if self._peer_uid() not in {0, AGENT_UID}: + self._write(CommandResponse(77, "", "protected mock: caller identity rejected\n")) + return + line = self.rfile.readline(MAX_REQUEST_BYTES + 1) + if len(line) > MAX_REQUEST_BYTES or not line.endswith(b"\n"): + self._write(CommandResponse(64, "", "protected mock: invalid request size\n")) + return + try: + request: Any = json.loads(line.decode("utf-8")) + if not isinstance(request, dict) or request.get("version") != PROTOCOL_VERSION: + raise ValueError + tool = request.get("tool") + argv = request.get("argv") + if not isinstance(tool, str) or not isinstance(argv, list): + raise ValueError + if not all(isinstance(item, str) for item in argv): + raise ValueError + except (UnicodeError, ValueError): + self._write(CommandResponse(64, "", "protected mock: invalid request\n")) + return + server = cast(ProtectedMockServer, self.server) + self._write(server.dispatch(tool, argv)) + + def _peer_uid(self) -> int: + peer_cred = getattr(socket, "SO_PEERCRED", None) + if peer_cred is None: + raise RuntimeError("SO_PEERCRED is required for protected mock caller validation") + credentials = self.request.getsockopt(socket.SOL_SOCKET, peer_cred, struct.calcsize("3i")) + _pid, uid, _gid = struct.unpack("3i", credentials) + return uid + + def _write(self, response: CommandResponse) -> None: + payload = ( + json.dumps( + { + "version": PROTOCOL_VERSION, + "exit_code": response.exit_code, + "stdout": response.stdout, + "stderr": response.stderr, + }, + ensure_ascii=True, + separators=(",", ":"), + ).encode("utf-8") + + b"\n" + ) + if len(payload) > MAX_RESPONSE_BYTES: + payload = ( + json.dumps( + { + "version": PROTOCOL_VERSION, + "exit_code": 70, + "stdout": "", + "stderr": "protected mock: response exceeds size limit\n", + }, + separators=(",", ":"), + ).encode("utf-8") + + b"\n" + ) + self.wfile.write(payload) + + +def serve(config_path: Path) -> None: + socket_path = Path(SOCKET_PATH) + socket_path.parent.mkdir(parents=True, exist_ok=True) + socket_path.unlink(missing_ok=True) + tools = load_config(config_path) + server = ProtectedMockServer(str(socket_path), tools) + try: + chown = getattr(os, "chown", None) + geteuid = getattr(os, "geteuid", None) + if chown is None or geteuid is None: + raise RuntimeError("mockd requires Linux chown/geteuid support") + chown(socket_path, geteuid(), MOCK_RPC_GID) + socket_path.chmod(0o660) + server.serve_forever(poll_interval=0.2) + finally: + server.server_close() + socket_path.unlink(missing_ok=True) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--config", required=True, type=Path) + args = parser.parse_args() + serve(args.config) + + +if __name__ == "__main__": + main() diff --git a/src/coder_eval/sandbox.py b/src/coder_eval/sandbox.py index 2748443f..cf161232 100644 --- a/src/coder_eval/sandbox.py +++ b/src/coder_eval/sandbox.py @@ -4,6 +4,7 @@ import json import logging import os +import shlex import shutil import subprocess import sys @@ -210,6 +211,7 @@ def _setup_tempdir(self, target_dir: Path | None = None) -> Path: # Generate recording shims for `record_cli` tools (before the +x pass # below, which also covers them) self._generate_cli_recorders() + self._generate_protected_mock_clients() # Mark mock binaries executable so the agent's PATH can shadow real CLIs self._prepare_mock_path_dirs() @@ -458,7 +460,7 @@ def resolved_mock_path_dirs(self) -> list[Path]: # generate a shim whose name a user mock dir already provides, so this # order can never silently shadow a task's own mock — it only fixes which # directory wins for names the harness itself owns. - if self.config.record_cli: + if self.config.record_cli or self.config.protected_mocks: generated = self._resolve_within_sandbox(RECORD_CLI_DIR, field="record_cli directory") if generated.is_dir(): resolved.append(generated) @@ -559,6 +561,41 @@ def _generate_cli_recorders(self) -> None: + ", ".join(f"{s.tool}(exit {s.exit_code})" for s in self.config.record_cli) ) + def _generate_protected_mock_clients(self) -> None: + """Generate data-free wrappers for fixture-backed mockd tools.""" + + if not self.config.protected_mocks: + return + assert self.sandbox_dir is not None, "Sandbox directory not initialized" + + from coder_eval.protected_mock.protocol import CLIENT_EXECUTABLE + + client_dir = self._resolve_within_sandbox(RECORD_CLI_DIR, field="protected mock client directory") + client_dir.mkdir(parents=True, exist_ok=True) + log_path = self.sandbox_dir / RECORD_CLI_LOG + log_path.touch(exist_ok=True) + + for spec in self.config.protected_mocks: + wrapper = client_dir / spec.tool + if wrapper.exists(): + raise RuntimeError( + f"protected mock client would overwrite {wrapper}; remove the colliding record_cli or mock" + ) + wrapper.write_text( + "#!/bin/sh\n" + + f"CODER_EVAL_MOCK_CALL_LOG={shlex.quote(str(log_path))} " + + f'exec {shlex.quote(CLIENT_EXECUTABLE)} {shlex.quote(spec.tool)} "$@"\n', + encoding="utf-8", + newline="\n", + ) + wrapper.chmod(0o555) + + logger.info( + "Generated protected mock client(s) in %s: %s", + RECORD_CLI_DIR, + ", ".join(spec.tool for spec in self.config.protected_mocks), + ) + def _apply_starter_files_source(self, source: StarterFilesSource) -> None: """Create inline starter files in sandbox with overwrite tracking. diff --git a/src/coder_eval/utils.py b/src/coder_eval/utils.py index 1624dae6..4a8800f7 100644 --- a/src/coder_eval/utils.py +++ b/src/coder_eval/utils.py @@ -86,6 +86,40 @@ def process_plugins( return processed +AGENT_ENV_SCRUB_VARS: tuple[str, ...] = ( + "SKILLS_REPO_PATH", + "TASK_DIR", + # The evaluator's Bedrock credential. No agent needs to INHERIT it: the Claude + # backend sets it explicitly from a resolved BedrockRoute (and blanks it on the + # LiteLLM route), Codex authenticates via CODEX_API_KEY, and Antigravity does not + # use Bedrock. Left inherited it reaches the dropped agent process, where it is + # readable through that process's own environment -- the UID barrier stops + # filesystem access to grading material but cannot hide an agent's own env. + # Scrubbing it also stops an inherited token from silently steering a DirectRoute + # run onto Bedrock (the CLI auto-selects on `process.env.AWS_BEARER_TOKEN_BEDROCK`). + "AWS_BEARER_TOKEN_BEDROCK", +) +AGENT_ENV_SCRUB_PREFIXES: tuple[str, ...] = ("CODER_EVAL_",) +AGENT_ENV_PASSTHROUGH_VARS: tuple[str, ...] = ("CODER_EVAL_AGENT_ALLOW_RPC",) + + +def scrub_agent_env_overrides() -> dict[str, str]: + """Mask harness-only variables in SDK subprocess environments. + + Claude and Codex merge their explicit environment over ``os.environ``; + empty-string overrides are therefore the only concurrency-safe removal + mechanism. Antigravity has no environment seam and removes the same names + during its serialized spawn window. + """ + + return { + name: "" + for name in os.environ + if name in AGENT_ENV_SCRUB_VARS + or (name.startswith(AGENT_ENV_SCRUB_PREFIXES) and name not in AGENT_ENV_PASSTHROUGH_VARS) + } + + SKIP = object() # Sentinel marking values that serialize_value should drop from the result. diff --git a/tests/test_codex_agent.py b/tests/test_codex_agent.py index 95ee117f..2123fd13 100644 --- a/tests/test_codex_agent.py +++ b/tests/test_codex_agent.py @@ -18,6 +18,7 @@ CodexAgent, ) from coder_eval.models import AgentConfig, AgentKind, parse_agent_config +from coder_eval.utils import AGENT_ENV_SCRUB_VARS class TestCodexAgentInitialization: @@ -128,6 +129,21 @@ def test_sandbox_is_full_access(self, monkeypatch, mode, in_container, os_name): class TestCodexEnvironmentConfiguration: """Test _build_codex_env: only CODEX_API_KEY travels via env.""" + @pytest.fixture(autouse=True) + def _no_ambient_scrub_vars(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Keep ambient evaluator credentials out of the exact-equality assertions. + + ``_build_codex_env`` starts from ``scrub_agent_env_overrides()``, so any + scrubbed name that the host happens to export (a developer's or CI's + ``AWS_BEARER_TOKEN_BEDROCK``, say) shows up as an extra masking entry and + breaks assertions that are only about what codex itself contributes. The + masking behavior has its own coverage in + ``tests/test_docker_identity_isolation.py``. + """ + + for name in AGENT_ENV_SCRUB_VARS: + monkeypatch.delenv(name, raising=False) + def test_build_codex_env_returns_none_without_key(self, monkeypatch): """No CODEX_API_KEY -> None (base URL alone is not enough).""" monkeypatch.delenv("CODEX_API_KEY", raising=False) diff --git a/tests/test_docker_build_failure.py b/tests/test_docker_build_failure.py index 32f28e0d..31c7d024 100644 --- a/tests/test_docker_build_failure.py +++ b/tests/test_docker_build_failure.py @@ -33,6 +33,11 @@ def _make_runner(run_dir: Path) -> DockerRunner: task_id="suri", description="t", initial_prompt="do", + # A concrete agent type is required now that docker.agent_isolation + # defaults to true: the isolation gate rejects a type with no verified + # UID-drop launch seam before the build runs, and this test covers + # build-failure observability rather than that gate. + agent={"type": "claude-code"}, sandbox=SandboxConfig(driver="docker", docker=DockerDriverConfig(image="x:1", dockerfile_path="/df")), success_criteria=[FileExistsCriterion(description="c", path="out.txt")], ) diff --git a/tests/test_docker_identity_isolation.py b/tests/test_docker_identity_isolation.py new file mode 100644 index 00000000..8b5247a7 --- /dev/null +++ b/tests/test_docker_identity_isolation.py @@ -0,0 +1,154 @@ +"""Drift guards for the Linux UID/GID agent boundary.""" + +from __future__ import annotations + +import signal +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from coder_eval.agents.codex_agent import CodexAgent +from coder_eval.isolation.docker_runner import DockerRunError, _preflight_agent_isolation_image +from coder_eval.models import ( + AGENT_GID, + AGENT_HOME, + AGENT_UID, + MOCK_RPC_GID, + MOCKD_GID, + MOCKD_UID, + AgentKind, + DockerDriverConfig, + parse_agent_config, +) +from coder_eval.utils import scrub_agent_env_overrides + + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +def test_image_identity_literals_and_capability_label_match_models() -> None: + dockerfile = (REPO_ROOT / "docker" / "Dockerfile").read_text(encoding="utf-8") + assert f"ARG AGENT_UID={AGENT_UID}" in dockerfile + assert f"ARG AGENT_GID={AGENT_GID}" in dockerfile + assert f"ARG MOCKD_UID={MOCKD_UID}" in dockerfile + assert f"ARG MOCKD_GID={MOCKD_GID}" in dockerfile + assert f"ARG MOCK_RPC_GID={MOCK_RPC_GID}" in dockerfile + assert 'LABEL org.coder-eval.agent-isolation="uid-gid-v1"' in dockerfile + assert "USER agent" not in dockerfile + assert DockerDriverConfig().agent_isolation is True + + +@pytest.mark.parametrize("script_name", ["coder_eval_drop_privilege.sh", "coder_eval_mockd.sh"]) +def test_privilege_launchers_clear_capabilities_and_set_no_new_privs(script_name: str) -> None: + script = (REPO_ROOT / "docker" / script_name).read_text(encoding="utf-8") + assert "--inh-caps=-all" in script + assert "--ambient-caps=-all" in script + assert "--bounding-set=-all" in script + assert "--no-new-privs" in script + if script_name == "coder_eval_drop_privilege.sh": + assert "--clear-groups" in script + assert "CODER_EVAL_AGENT_ALLOW_RPC" in script + else: + assert "--groups=uip-rpc" in script + + +def test_agent_launcher_targets_only_agent_identity() -> None: + script = (REPO_ROOT / "docker" / "coder_eval_drop_privilege.sh").read_text(encoding="utf-8") + assert "--reuid=agent" in script + assert "--regid=agent" in script + assert "mockd" not in script + + +def test_agent_environment_scrubs_only_present_harness_paths(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("SKILLS_REPO_PATH", "/private/skills") + monkeypatch.setenv("TASK_DIR", "/private/task") + monkeypatch.setenv("CODER_EVAL_AGENT_ISOLATION", "1") + monkeypatch.setenv("CODER_EVAL_AGENT_ALLOW_RPC", "1") + monkeypatch.setenv("ANTHROPIC_API_KEY", "needed-by-agent") + # Keep the exact-equality assertion below valid on a host that exports it. + monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) + + overrides = scrub_agent_env_overrides() + + assert overrides == { + "SKILLS_REPO_PATH": "", + "TASK_DIR": "", + "CODER_EVAL_AGENT_ISOLATION": "", + } + assert "ANTHROPIC_API_KEY" not in overrides + assert "CODER_EVAL_AGENT_ALLOW_RPC" not in overrides + + +def test_agent_environment_scrubs_inherited_bedrock_credential(monkeypatch: pytest.MonkeyPatch) -> None: + """The evaluated agent must not inherit the evaluator's Bedrock token. + + The UID barrier blocks filesystem access to grading material but cannot hide a + process's own environment, so a credential left there is readable by the agent + itself. Claude re-sets this explicitly from a resolved BedrockRoute, so masking + the inherited value costs the Bedrock path nothing. + """ + + monkeypatch.setenv("AWS_BEARER_TOKEN_BEDROCK", "evaluator-only-secret") + monkeypatch.setenv("AWS_REGION", "us-east-1") + + overrides = scrub_agent_env_overrides() + + assert overrides["AWS_BEARER_TOKEN_BEDROCK"] == "" + # The region is not a credential and stays inherited. + assert "AWS_REGION" not in overrides + + +def test_isolated_codex_profiles_never_restore_root_harness_home(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("HOME", "/root") + monkeypatch.setenv("ZDOTDIR", "/root/private-zdot") + monkeypatch.setenv("CODER_EVAL_AGENT_ISOLATION", "1") + monkeypatch.setattr(CodexAgent, "_login_shell_profiles_supported", staticmethod(lambda: True)) + agent = CodexAgent(parse_agent_config(type=AgentKind.CODEX)) + agent._env_path_prepend = ["/work/agent/cli_mocks"] + + agent._setup_login_shell_home() + try: + assert agent._login_shell_home is not None + for profile in (".bash_profile", ".profile", ".zshenv", ".zprofile", ".zshrc"): + content = (agent._login_shell_home / profile).read_text(encoding="utf-8") + assert f"export HOME={AGENT_HOME}" in content + assert "/root" not in content + finally: + agent._cleanup_login_shell_home() + + +def test_agent_teardown_rescans_until_uid_has_no_processes(monkeypatch: pytest.MonkeyPatch) -> None: + from coder_eval.isolation import agent_identity + + scans = iter([[41, 42], [42], []]) + signals: list[tuple[int, signal.Signals]] = [] + monkeypatch.setenv("CODER_EVAL_AGENT_ISOLATION", "1") + monkeypatch.setattr(agent_identity, "require_isolation_runtime", lambda: None) + monkeypatch.setattr(agent_identity, "_agent_pids", lambda: next(scans)) + monkeypatch.setattr( + agent_identity, + "_signal_agent_pids", + lambda pids, sig: signals.extend((pid, sig) for pid in pids), + ) + monkeypatch.setattr(agent_identity.time, "sleep", lambda _seconds: None) + + agent_identity.terminate_agent_processes() + + expected_kill = getattr(signal, "SIGKILL", signal.SIGTERM) + assert signals == [(41, signal.SIGTERM), (42, signal.SIGTERM), (42, expected_kill)] + + +def test_isolation_image_label_preflight_accepts_only_declared_capability(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + "coder_eval.isolation.docker_runner.subprocess.run", + lambda *args, **kwargs: SimpleNamespace(stdout="uid-gid-v1\n"), + ) + _preflight_agent_isolation_image("image:good") + + monkeypatch.setattr( + "coder_eval.isolation.docker_runner.subprocess.run", + lambda *args, **kwargs: SimpleNamespace(stdout="\n"), + ) + with pytest.raises(DockerRunError, match="does not declare"): + _preflight_agent_isolation_image("image:old") diff --git a/tests/test_docker_runner_mounts.py b/tests/test_docker_runner_mounts.py index 638d1b69..2788d9d3 100644 --- a/tests/test_docker_runner_mounts.py +++ b/tests/test_docker_runner_mounts.py @@ -4,8 +4,8 @@ omitted, rejection of destinations that shadow framework-owned mounts (``/work``, ``/``), and ``~`` / ``$VAR`` expansion on the source side. -Also covers user/output directory fixes: --user flag on POSIX, output -directory mounted to /work/output, and --output argument using container path. +Also covers protected output and agent-home mounts plus the container-side +``--output`` argument. """ from __future__ import annotations @@ -32,7 +32,7 @@ _sanitize_container_name_component, _validate_extra_mount, ) -from coder_eval.models import FileExistsCriterion, SandboxConfig, TaskDefinition +from coder_eval.models import AGENT_HOME, FileExistsCriterion, SandboxConfig, TaskDefinition # DockerRunner targets Linux containers from POSIX hosts. On Windows the test @@ -137,7 +137,7 @@ def _make_runner(self, run_dir: Path | None = None) -> DockerRunner: return DockerRunner(rt) def test_output_mounted_to_container_output_dir(self): - """Output directory should be mounted to CONTAINER_OUTPUT_DIR (/work/output).""" + """Output directory should be mounted below the protected grader root.""" runner = self._make_runner() with tempfile.TemporaryDirectory() as tmpdir: @@ -380,8 +380,8 @@ class TestClaudeHomeRWCopyMount: ``_prepare_host_mounts`` copies the host ``~/.claude`` (minus heavy per-session state) into a tmp dir and records it on ``_claude_mount_src``; ``_build_argv`` then mounts that copy read-WRITE at - the symmetric ``$HOME/.claude`` path. The old two-layer (``:ro`` parent + - ``session-env`` RW child) scheme is gone. + the dropped agent's dedicated ``/home/agent/.claude`` path. The old + symmetric-host-home and two-layer schemes are gone. """ def _make_runner(self) -> DockerRunner: @@ -432,7 +432,7 @@ def fake_home(self, tmp_path, monkeypatch): monkeypatch.delenv("CODER_EVAL_NO_CLAUDE_MOUNT", raising=False) return home - def test_rw_copy_mounted_at_symmetric_path(self, fake_home, tmp_path): + def test_rw_copy_mounted_at_agent_home(self, fake_home, tmp_path): runner = self._make_runner() staging = tmp_path / "staging" staging.mkdir() @@ -450,10 +450,11 @@ def test_rw_copy_mounted_at_symmetric_path(self, fake_home, tmp_path): mounts = self._volume_mounts(argv) host_claude = fake_home / ".claude" - # Exactly one claude mount: the copy → symmetric path, read-WRITE (no :ro). - assert f"{copy}:{host_claude}" in mounts - claude_mounts = [m for m in mounts if m.endswith(str(host_claude)) or f":{host_claude}" in m] - assert claude_mounts == [f"{copy}:{host_claude}"] + agent_claude = Path(AGENT_HOME) / ".claude" + # Exactly one Claude mount: disposable copy → agent HOME, read-WRITE. + assert f"{copy}:{agent_claude}" in mounts + claude_mounts = [m for m in mounts if m.endswith(str(agent_claude))] + assert claude_mounts == [f"{copy}:{agent_claude}"] # The retired two-layer scheme leaves no trace. assert f"{host_claude}:{host_claude}:ro" not in mounts assert not any("session-env" in m for m in mounts) @@ -674,4 +675,4 @@ def test_argv_reserved_workspace_raises(self): def test_container_paths_reexported_from_docker_runner(self): # Existing importers read CONTAINER_OUTPUT_DIR from docker_runner; keep that working. - assert CONTAINER_OUTPUT_DIR == "/work/output" + assert CONTAINER_OUTPUT_DIR == "/opt/coder-eval/grader/output" diff --git a/tests/test_plugin_bundle.py b/tests/test_plugin_bundle.py new file mode 100644 index 00000000..c3164a55 --- /dev/null +++ b/tests/test_plugin_bundle.py @@ -0,0 +1,203 @@ +"""Security tests for manifest-verified agent plugin projections.""" + +from __future__ import annotations + +import json +import os +import sys +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +from coder_eval.isolation.docker_runner import DockerRunner +from coder_eval.models import ( + CONTAINER_AGENT_SKILLS_DIR, + CONTAINER_PRIVATE_PLUGIN_DIR, + AgentKind, + ClaudeCodeAgentConfig, + DockerDriverConfig, + FileExistsCriterion, + RunCommandCriterion, + SandboxConfig, + TaskDefinition, +) +from coder_eval.plugin_bundle import PluginBundleError, build_manifest, stage_bundle, verify_bundle + + +def _write(path: Path, text: str = "data") -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return path + + +def _symlink_or_skip(target: str, link: Path, *, is_dir: bool = False) -> None: + try: + os.symlink(target, link, target_is_directory=is_dir) + except (OSError, NotImplementedError) as exc: + pytest.skip(f"symlinks unavailable: {exc}") + + +# Windows stores a relative reparse-point target verbatim and resolves it with +# backslash-only NT parsing, so a POSIX-separated target like "../shared/x.md" +# dangles there. Plugin bundles are only consumed by the POSIX-only DockerRunner. +posix_relative_symlink = pytest.mark.skipif( + sys.platform == "win32", + reason="relative POSIX symlink targets do not resolve on Windows; plugin bundles are POSIX-only", +) + + +def test_bundle_includes_only_plugin_discovery_subtrees(tmp_path: Path) -> None: + source = tmp_path / "skills-repo" + _write(source / "skills" / "demo" / "SKILL.md", "public") + _write(source / "commands" / "run.md", "public command") + _write(source / "graders" / "secret.py", "answer") + _write(source / "tests" / "fixtures" / "golden.json", "answer") + + bundle = tmp_path / "bundle" + manifest = stage_bundle(source, bundle) + + assert (bundle / "skills" / "demo" / "SKILL.md").read_text(encoding="utf-8") == "public" + assert (bundle / "commands" / "run.md").is_file() + assert not (bundle / "graders").exists() + assert not (bundle / "tests").exists() + assert set(manifest.files) == {"commands/run.md", "skills/demo/SKILL.md"} + + +@pytest.mark.parametrize("name", ["RESOLUTION.md", "check_answer.py", "CHECK_X.PY"]) +def test_bundle_fails_closed_on_answer_key_name_inside_allowed_tree(tmp_path: Path, name: str) -> None: + source = tmp_path / "plugin" + _write(source / "skills" / "demo" / name, "hidden") + + with pytest.raises(PluginBundleError, match="hidden grading material"): + build_manifest(source) + + +def test_bundle_rejects_absolute_symlink_back_to_source(tmp_path: Path) -> None: + source = tmp_path / "plugin" + secret = _write(source / "skills" / "demo" / "secret.txt") + link = source / "skills" / "demo" / "alias.txt" + _symlink_or_skip(str(secret.resolve()), link) + + with pytest.raises(PluginBundleError, match="absolute plugin symlink"): + build_manifest(source) + + +@posix_relative_symlink +def test_bundle_rejects_relative_symlink_to_excluded_tree(tmp_path: Path) -> None: + source = tmp_path / "plugin" + _write(source / "fixtures" / "answer.json", "hidden") + link = source / "skills" / "answer.json" + link.parent.mkdir(parents=True) + _symlink_or_skip("../fixtures/answer.json", link) + + with pytest.raises(PluginBundleError, match="excluded content"): + build_manifest(source) + + +@posix_relative_symlink +def test_bundle_preserves_safe_relative_symlink(tmp_path: Path) -> None: + source = tmp_path / "plugin" + _write(source / "skills" / "shared" / "guide.md", "public") + link = source / "skills" / "demo" / "guide.md" + link.parent.mkdir(parents=True) + _symlink_or_skip("../shared/guide.md", link) + + bundle = tmp_path / "bundle" + stage_bundle(source, bundle) + + staged_link = bundle / "skills" / "demo" / "guide.md" + assert staged_link.is_symlink() + assert os.readlink(staged_link) == "../shared/guide.md" + assert staged_link.read_text(encoding="utf-8") == "public" + + +@pytest.mark.parametrize("mutation", ["change", "add", "delete"]) +def test_bundle_verification_detects_drift(tmp_path: Path, mutation: str) -> None: + source = tmp_path / "plugin" + _write(source / "skills" / "demo" / "SKILL.md", "public") + bundle = tmp_path / "bundle" + stage_bundle(source, bundle) + + staged = bundle / "skills" / "demo" / "SKILL.md" + # stage_bundle hardens the projection (dirs 0o555, files 0o444). Adding or + # removing an entry needs write permission on the directory, not just the + # file, so relax both before simulating drift. + staged.parent.chmod(0o755) + staged.chmod(0o644) + if mutation == "change": + staged.write_text("tampered", encoding="utf-8") + elif mutation == "add": + _write(bundle / "skills" / "demo" / "undeclared.txt") + else: + staged.unlink() + + with pytest.raises(PluginBundleError, match="differs from its manifest"): + verify_bundle(bundle) + + +def test_docker_runner_rewrites_plugin_and_never_mounts_raw_at_original_path(tmp_path: Path) -> None: + plugin = tmp_path / "skills-repo" + _write(plugin / "skills" / "demo" / "SKILL.md", "public") + _write(plugin / "tests" / "fixtures" / "golden.json", "hidden") + task_dir = tmp_path / "task" + task_dir.mkdir() + + task = TaskDefinition( + task_id="isolation", + description="test", + initial_prompt="use the demo skill", + agent=ClaudeCodeAgentConfig( + type=AgentKind.CLAUDE_CODE, + plugins=[{"type": "local", "path": str(plugin)}], + ), + sandbox=SandboxConfig(driver="docker", docker=DockerDriverConfig(agent_isolation=True)), + success_criteria=[FileExistsCriterion(description="done", path="done.txt")], + ) + rt = MagicMock() + rt.task = task + rt.task_file = task_dir / "task.yaml" + rt.run_dir = tmp_path / "run" + runner = DockerRunner(rt) + + staging = tmp_path / "staging" + staging.mkdir() + runner._prepare_isolated_sources(staging) + payload = runner._rewrite_task_paths(task.model_dump(mode="json")) + + agent = payload["agent"] + assert isinstance(agent, dict) + plugins = agent["plugins"] + assert isinstance(plugins, list) + assert plugins[0]["path"] == f"{CONTAINER_AGENT_SKILLS_DIR}/plugin-0" + assert str(plugin) not in json.dumps(payload) + + input_dir = staging / "input" + output_dir = staging / "output" + input_dir.mkdir() + output_dir.mkdir() + argv = runner._build_argv(input_dir, output_dir, container_name="isolation", image="test-image") + mounts = [argv[index + 1] for index, value in enumerate(argv) if value == "-v"] + + assert "--init" in argv + pids_index = argv.index("--pids-limit") + assert argv[pids_index + 1] == "512" + assert f"{plugin.resolve()}:{CONTAINER_PRIVATE_PLUGIN_DIR}/plugin-0:ro" in mounts + assert not any(mount.startswith(f"{plugin.resolve()}:{plugin.resolve()}") for mount in mounts) + assert any(mount.endswith(f":{CONTAINER_AGENT_SKILLS_DIR}/plugin-0:ro") for mount in mounts) + + +def test_isolation_rejects_dynamic_privileged_criterion(tmp_path: Path) -> None: + task = TaskDefinition( + task_id="unsafe-grader", + description="test", + initial_prompt="work", + agent=ClaudeCodeAgentConfig(type=AgentKind.CLAUDE_CODE), + sandbox=SandboxConfig(driver="docker", docker=DockerDriverConfig(agent_isolation=True)), + success_criteria=[RunCommandCriterion(description="unsafe", command="python check.py")], + ) + rt = MagicMock(task=task, task_file=tmp_path / "task.yaml", run_dir=tmp_path / "run") + runner = DockerRunner(rt) + + with pytest.raises(RuntimeError, match="dynamic criteria"): + runner._validate_agent_isolation_compatibility() diff --git a/tests/test_protected_mock.py b/tests/test_protected_mock.py new file mode 100644 index 00000000..318a489a --- /dev/null +++ b/tests/test_protected_mock.py @@ -0,0 +1,277 @@ +"""Tests for the protected exact-command mock protocol and thin wrappers.""" + +from __future__ import annotations + +import json +import subprocess +import threading +from pathlib import Path +from unittest.mock import MagicMock + +import pytest +from pydantic import ValidationError + +from coder_eval.isolation.docker_runner import DockerRunner +from coder_eval.models import ( + AgentKind, + ClaudeCodeAgentConfig, + DockerDriverConfig, + FileExistsCriterion, + ProtectedMockConfig, + SandboxConfig, + TaskDefinition, +) +from coder_eval.protected_mock.protocol import CLIENT_EXECUTABLE +from coder_eval.protected_mock.server import ProtectedMockServer, load_config +from coder_eval.sandbox import Sandbox + + +def _fixture(path: Path) -> Path: + path.write_text( + json.dumps( + { + "version": 1, + "responses": [ + { + "argv": ["rpa", "get-errors", "--output", "json"], + "exit_code": 0, + "stdout": '{"errors":[]}\n', + } + ], + "default": {"exit_code": 2, "stderr": "not configured\n"}, + } + ), + encoding="utf-8", + ) + return path + + +def test_protected_mocks_require_docker_driver(tmp_path: Path) -> None: + fixture = _fixture(tmp_path / "uip.json") + with pytest.raises(ValidationError, match="requires driver: docker"): + SandboxConfig( + driver="tempdir", + protected_mocks=[ProtectedMockConfig(tool="uip", fixture=str(fixture))], + ) + + +def test_protected_mock_names_are_unique_and_do_not_collide_with_recorders(tmp_path: Path) -> None: + fixture = _fixture(tmp_path / "uip.json") + spec = {"tool": "uip", "fixture": str(fixture)} + with pytest.raises(ValidationError, match="must be unique"): + SandboxConfig(driver="docker", protected_mocks=[spec, spec]) # type: ignore[list-item] + with pytest.raises(ValidationError, match="cannot both provide"): + SandboxConfig( + driver="docker", + protected_mocks=[spec], # type: ignore[list-item] + record_cli=[{"tool": "uip"}], # type: ignore[list-item] + ) + + +def test_fixture_service_matches_exact_argv_and_enforces_budget(tmp_path: Path) -> None: + fixture = _fixture(tmp_path / "uip.json") + config = tmp_path / "config.json" + config.write_text( + json.dumps( + { + "version": 1, + "tools": [{"tool": "uip", "fixture": str(fixture), "max_requests": 2}], + } + ), + encoding="utf-8", + ) + tools = load_config(config) + fake_server = MagicMock() + fake_server.tools = tools + fake_server.budget_lock = threading.Lock() + + expected = ProtectedMockServer.dispatch(fake_server, "uip", ["rpa", "get-errors", "--output", "json"]) + assert expected.exit_code == 0 + assert expected.stdout == '{"errors":[]}\n' + + # There is no generic file-read endpoint: an arbitrary path-bearing argv is + # merely an unmatched CLI command and receives the fixture's fixed default. + unmatched = ProtectedMockServer.dispatch(fake_server, "uip", ["read", "/etc/passwd"]) + assert unmatched.exit_code == 2 + assert unmatched.stderr == "not configured\n" + + exhausted = ProtectedMockServer.dispatch(fake_server, "uip", ["rpa", "get-errors", "--output", "json"]) + assert exhausted.exit_code == 75 + assert "budget exhausted" in exhausted.stderr + + +def test_fixture_rejects_duplicate_argv(tmp_path: Path) -> None: + fixture = tmp_path / "bad.json" + response = {"argv": ["same"], "exit_code": 0} + fixture.write_text(json.dumps({"version": 1, "responses": [response, response]}), encoding="utf-8") + config = tmp_path / "config.json" + config.write_text( + json.dumps({"version": 1, "tools": [{"tool": "uip", "fixture": str(fixture), "max_requests": 1}]}), + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="duplicate argv"): + load_config(config) + + +def test_normalized_fixture_matching_remains_finite(tmp_path: Path) -> None: + fixture = tmp_path / "normalized.json" + fixture.write_text( + json.dumps( + { + "version": 1, + "responses": [ + { + "argv": ["rpa", "get-errors", "--job-id", "42"], + "match_mode": "normalized", + "exit_code": 0, + "stdout": "configured\n", + } + ], + "default": {"exit_code": 2, "stderr": "not configured\n"}, + } + ), + encoding="utf-8", + ) + config = tmp_path / "config.json" + config.write_text( + json.dumps({"version": 1, "tools": [{"tool": "uip", "fixture": str(fixture), "max_requests": 2}]}), + encoding="utf-8", + ) + fake_server = MagicMock() + fake_server.tools = load_config(config) + fake_server.budget_lock = threading.Lock() + + matched = ProtectedMockServer.dispatch( + fake_server, + "uip", + ["--job-id=42", "get-errors", "rpa", "--output", "json"], + ) + assert matched.stdout == "configured\n" + + extra_argument = ProtectedMockServer.dispatch( + fake_server, + "uip", + ["rpa", "get-errors", "--job-id", "42", "--include-secrets"], + ) + assert extra_argument.exit_code == 2 + + +def test_passthrough_is_prefix_limited_and_cached(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + fixture = _fixture(tmp_path / "uip.json") + config = tmp_path / "config.json" + config.write_text( + json.dumps( + { + "version": 1, + "tools": [ + { + "tool": "uip", + "fixture": str(fixture), + "max_requests": 3, + "passthrough_argv_prefixes": [["docsai", "ask"]], + } + ], + } + ), + encoding="utf-8", + ) + monkeypatch.setattr("coder_eval.protected_mock.server.shutil.which", lambda _tool: "/usr/local/bin/uip") + run = MagicMock(return_value=subprocess.CompletedProcess([], 0, "answer\n", "")) + monkeypatch.setattr("coder_eval.protected_mock.server.subprocess.run", run) + fake_server = MagicMock() + fake_server.tools = load_config(config) + fake_server.budget_lock = threading.Lock() + fake_server.passthrough_lock = threading.Lock() + fake_server._passthrough.side_effect = lambda state, argv: ProtectedMockServer._passthrough( + fake_server, state, argv + ) + + argv = ["docsai", "ask", "what failed?"] + first = ProtectedMockServer.dispatch(fake_server, "uip", argv) + second = ProtectedMockServer.dispatch(fake_server, "uip", argv) + blocked = ProtectedMockServer.dispatch(fake_server, "uip", ["auth", "token"]) + + assert first.stdout == second.stdout == "answer\n" + assert blocked.exit_code == 2 + run.assert_called_once() + assert run.call_args.args[0] == ["/usr/local/bin/uip", *argv] + assert run.call_args.kwargs["stdin"] is subprocess.DEVNULL + + +def test_passthrough_prefixes_are_validated() -> None: + with pytest.raises(ValidationError, match="must be unique"): + ProtectedMockConfig( + tool="uip", + fixture="fixture.json", + passthrough_argv_prefixes=[["docsai", "ask"], ["docsai", "ask"]], + ) + with pytest.raises(ValidationError, match="1 to 8"): + ProtectedMockConfig(tool="uip", fixture="fixture.json", passthrough_argv_prefixes=[[]]) + + +def test_sandbox_generates_data_free_client_wrapper(tmp_path: Path) -> None: + fixture = _fixture(tmp_path / "uip.json") + config = SandboxConfig( + driver="docker", + python=None, + protected_mocks=[ProtectedMockConfig(tool="uip", fixture=str(fixture))], + ) + # Mirrors _run-task-internal: validate protected_mocks while the authored + # driver is docker, then execute the already-containerized sandbox locally. + config = config.model_copy(update={"driver": "tempdir"}) + sandbox = Sandbox(config, task_id="protected-client") + workspace = tmp_path / "workspace" + try: + sandbox.setup(workspace) + wrapper = workspace / "cli_mocks" / "uip" + text = wrapper.read_text(encoding="utf-8") + assert CLIENT_EXECUTABLE in text + assert str(fixture) not in text + assert '{"errors":[]}' not in text + assert sandbox.resolved_mock_path_dirs == [(workspace / "cli_mocks").resolve()] + assert (workspace / "cli_mocks" / "calls.jsonl").is_file() + finally: + sandbox.cleanup() + + +def test_docker_stages_fixture_copy_only_under_mockd_parent(tmp_path: Path) -> None: + fixture = _fixture(tmp_path / "uip.json") + task_dir = tmp_path / "task" + task_dir.mkdir() + task = TaskDefinition( + task_id="protected-mock", + description="test", + initial_prompt="run uip", + agent=ClaudeCodeAgentConfig(type=AgentKind.CLAUDE_CODE), + sandbox=SandboxConfig( + driver="docker", + docker=DockerDriverConfig(agent_isolation=True), + protected_mocks=[ + ProtectedMockConfig( + tool="uip", + fixture=str(fixture), + max_requests=3, + passthrough_argv_prefixes=[["docsai", "ask"]], + ) + ], + ), + success_criteria=[FileExistsCriterion(description="done", path="done.txt")], + ) + rt = MagicMock(task=task, task_file=task_dir / "task.yaml", run_dir=tmp_path / "run") + runner = DockerRunner(rt) + staging = tmp_path / "staging" + staging.mkdir() + + runner._prepare_isolated_sources(staging) + payload = runner._rewrite_task_paths(task.model_dump(mode="json")) + + assert runner._mock_fixture_mount == staging / "protected-mock-fixtures" + assert (runner._mock_fixture_mount / "fixture-0.json").read_text(encoding="utf-8") == fixture.read_text( + encoding="utf-8" + ) + assert str(fixture) not in json.dumps(payload) + protected = payload["sandbox"]["protected_mocks"] # type: ignore[index] + assert protected[0]["fixture"] == "/opt/coder-eval/mock/fixtures/fixture-0.json" # type: ignore[index] + config = json.loads((runner._mock_fixture_mount / "mock-config.json").read_text(encoding="utf-8")) + assert config["tools"][0]["passthrough_argv_prefixes"] == [["docsai", "ask"]]