diff --git a/.github/workflows/plugin-image.yml b/.github/workflows/plugin-image.yml new file mode 100644 index 0000000..7ca9c6d --- /dev/null +++ b/.github/workflows/plugin-image.yml @@ -0,0 +1,69 @@ +name: Plugin image + +# Builds the Phase B2 plugin sandbox image and runs the live container +# backend test against the digest-pinned reference — so the container path +# is exercised in CI, not only on maintainer machines. +# +# No registry publication here: the image is pushed to a job-local registry +# and its digest is consumed in the same job. Publishing a pinned image for +# hosted fleets (GHCR) is a separate commercial decision; see +# docs/ISOLATION_MODEL.md. + +on: + push: + branches: [main] + paths: + - "docker/plugin.Dockerfile" + - "src/or_audit/eval/plugins.py" + - "src/or_audit/eval/plugin_host.py" + - "src/or_audit/eval/contracts.py" + - "tests/test_eval_plugin_host.py" + - ".github/workflows/plugin-image.yml" + pull_request: + paths: + - "docker/plugin.Dockerfile" + - "src/or_audit/eval/plugins.py" + - "src/or_audit/eval/plugin_host.py" + - "src/or_audit/eval/contracts.py" + - "tests/test_eval_plugin_host.py" + - ".github/workflows/plugin-image.yml" + workflow_dispatch: + +concurrency: + group: plugin-image-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + build-and-test: + name: Build sandbox image, test live backend + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - name: Start job-local registry + run: docker run -d -p 5000:5000 --name ci-registry registry:2 + - name: Build and push sandbox image + run: | + docker build --load -f docker/plugin.Dockerfile -t localhost:5000/surgeval-plugin:ci . + docker push localhost:5000/surgeval-plugin:ci + - uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5 + with: + enable-cache: true + - name: Install + run: uv sync --all-extras + - name: Resolve digest-pinned reference + id: ref + run: | + DIGEST=$(docker inspect --format='{{index .RepoDigests 0}}' localhost:5000/surgeval-plugin:ci) + case "$DIGEST" in + localhost:5000/surgeval-plugin@sha256:*) ;; + *) echo "Unexpected RepoDigest: $DIGEST" >&2; exit 1 ;; + esac + echo "value=$DIGEST" >> "$GITHUB_OUTPUT" + echo "Sandbox image: $DIGEST" >> "$GITHUB_STEP_SUMMARY" + - name: Live container backend test + env: + SURGEVAL_TEST_PLUGIN_IMAGE: ${{ steps.ref.outputs.value }} + run: uv run pytest tests/test_eval_plugin_host.py -q diff --git a/docs/ISOLATION_MODEL.md b/docs/ISOLATION_MODEL.md index 9592f42..c18c40a 100644 --- a/docs/ISOLATION_MODEL.md +++ b/docs/ISOLATION_MODEL.md @@ -11,12 +11,13 @@ marked enforced; everything else is an explicit gap with an owner. T0 protects against accidents and honest-agent label leakage, not malice. - **T1 — untrusted submission**: containerized plugin execution exists (`RuntimeDescriptor` kind `container` → digest-pinned image, `--network - none`, read-only package mount, tmpfs, memory/CPU/pids caps, separate - containers per runtime). Verified locally against a digest-pinned - registry image; image publication for CI/hosted fleets is still pending, - so CI exercises command construction only. Filesystem/network/resource - boundaries hold where the backend runs; hosted attestation (B5) still - pending for cross-lab trust. + none`, read-only package mount, non-root, dropped capabilities, + memory/CPU/pids caps from the digest-covered identity). CI builds the + sandbox image and runs the live backend test digest-pinned + (`.github/workflows/plugin-image.yml`); registry publication for hosted + fleets is still pending. Hosted attestation (B5) still pending for + cross-lab trust; the OSS verification contract is + `or_audit.eval.attestation`. 1. **Oracle routing**: labels travel only to the verifier context, never in agent payloads (`runner.py`, `plugins.py`). A protocol-conformant agent diff --git a/src/or_audit/eval/attestation.py b/src/or_audit/eval/attestation.py new file mode 100644 index 0000000..77e31ac --- /dev/null +++ b/src/or_audit/eval/attestation.py @@ -0,0 +1,76 @@ +"""Executor attestation for hosted evaluation runs (Phase B5, OSS side). + +Local job heads are unkeyed digests: tamper-evident, re-stampable. Cross-lab +trust needs the executor to stamp what it observed with a key the submitter +does not hold. This module defines the stamp and its verification; minting +happens in the hosted executor (private cloud tree), which holds the +operator secret. Nothing here mints without a secret, and nothing verifies +without the same one. +""" + +from __future__ import annotations + +import hashlib +import hmac +import json +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field + + +def _canonical(payload: dict[str, Any]) -> bytes: + return json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8") + + +class ExecutorAttestation(BaseModel): + """HMAC stamp over observed execution provenance.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + executor_id: str = Field(min_length=1, max_length=128) + artifact_head: str = Field(min_length=64, max_length=64, pattern="^[0-9a-f]{64}$") + backend: str = Field(min_length=1, max_length=32) + world_pin: str = Field(min_length=1, max_length=128) + nonce: str = Field(min_length=8, max_length=128) + mac: str = Field(min_length=64, max_length=64, pattern="^[0-9a-f]{64}$") + + def payload(self) -> dict[str, Any]: + return { + "executor_id": self.executor_id, + "artifact_head": self.artifact_head, + "backend": self.backend, + "world_pin": self.world_pin, + "nonce": self.nonce, + } + + +def attest( + *, + executor_id: str, + artifact_head: str, + backend: str, + world_pin: str, + nonce: str, + secret: bytes, +) -> ExecutorAttestation: + """Mint an attestation. Called by the hosted executor, never by evaluated code.""" + if not secret: + raise ValueError("attestation needs a non-empty operator secret") + stamp = ExecutorAttestation( + executor_id=executor_id, + artifact_head=artifact_head, + backend=backend, + world_pin=world_pin, + nonce=nonce, + mac="0" * 64, + ) + mac = hmac.new(secret, _canonical(stamp.payload()), hashlib.sha256).hexdigest() + return stamp.model_copy(update={"mac": mac}) + + +def verify(attestation: ExecutorAttestation, *, secret: bytes) -> bool: + """Check the stamp against the operator secret (constant-time).""" + if not secret: + return False + expected = hmac.new(secret, _canonical(attestation.payload()), hashlib.sha256).hexdigest() + return hmac.compare_digest(expected, attestation.mac) diff --git a/src/or_audit/eval/contracts.py b/src/or_audit/eval/contracts.py index 4a87089..4b6530d 100644 --- a/src/or_audit/eval/contracts.py +++ b/src/or_audit/eval/contracts.py @@ -240,6 +240,11 @@ class RuntimeDescriptor(_Frozen): #: Digest of the sandbox policy the intake ran under, so a run cannot claim #: an intake that happened under weaker isolation. sandbox_policy_digest: str = "" + #: Container resource envelope, part of the digest-covered identity so a + #: run cannot claim different limits than it executed under (B4). + container_memory: str = "2g" + container_cpus: str = "2.0" + container_pids_limit: str = "256" @model_validator(mode="after") def _identity_is_pinned(self) -> Self: diff --git a/src/or_audit/eval/plugins.py b/src/or_audit/eval/plugins.py index 6126a55..217de6e 100644 --- a/src/or_audit/eval/plugins.py +++ b/src/or_audit/eval/plugins.py @@ -88,6 +88,8 @@ def load_entrypoint(root: Path, entrypoint: str, *, label: str) -> Callable[..., #: the device-selection variables needed for GPU discovery are named. #: HOME is never inherited (it points at ~/.aws, ~/.huggingface, shell #: history): each runtime gets a fresh empty directory instead (B1 tier T0). +#: DOCKER_HOST is not inherited either: it selects a Docker daemon, which is +#: a host-tool concern handled per-command, never plugin input. _PLUGIN_ENV_ALLOW = frozenset( { "PATH", @@ -103,16 +105,11 @@ def load_entrypoint(root: Path, entrypoint: str, *, label: str) -> Callable[..., "CUDA_CACHE_PATH", "NVIDIA_VISIBLE_DEVICES", "NVIDIA_DRIVER_CAPABILITIES", - "DOCKER_HOST", } ) #: In-container mount point for the evaluated package (read-only). _CONTAINER_PKG_DIR = "/pkg" -#: Default resource envelope for containerized plugin children (B4). -_CONTAINER_MEMORY = "2g" -_CONTAINER_CPUS = "2.0" -_CONTAINER_PIDS_LIMIT = "256" #: Uppercase twin for Windows, where environment keys are case-insensitive #: but stored mixed-case (`Path`, `SystemRoot`). @@ -135,6 +132,19 @@ def _private_plugin_home() -> Path: return Path(tempfile.mkdtemp(prefix="surgeval-plugin-")) +def _host_tool_env(command: tuple[str, ...]) -> dict[str, str]: + """Host-tool variables for commands that are host tools, not plugins. + + The ``docker`` CLI driving a container spawn runs on the host and needs + daemon selection; evaluated plugin code inside the container never sees + it. Nothing else is passed through. + """ + if command[:1] == ("docker",): + host = os.environ.get("DOCKER_HOST") + return {"DOCKER_HOST": host} if host else {} + return {} + + #: Largest single plugin response accepted (8 MiB). Evidence transfer is one #: bounded JSON value per request, not a stream: an unbounded read lets a #: compromised or buggy child exhaust host memory before the timeout fires. @@ -172,6 +182,51 @@ def _readline_bounded(stream: Any, *, limit: int, timeout_sec: float) -> str | N return bytes(buf).decode("utf-8", errors="replace") +def _clamp_container_limits(descriptor: RuntimeDescriptor) -> tuple[str, str, str]: + """Executor-side bounds on task-declared container limits (B4). + + The descriptor carries the limits in its digest-covered identity, but the + executor refuses absurd ones: an untrusted package must not disable caps + (``--pids-limit -1``) or demand host-scale resources. + """ + memory = _parse_memory_mb(descriptor.container_memory) + if not 64 <= memory <= 16 * 1024: + raise TaskContractError( + f"container_memory {descriptor.container_memory!r} outside 64m..16g" + ) + try: + cpus = float(descriptor.container_cpus) + except ValueError as exc: + raise TaskContractError( + f"container_cpus {descriptor.container_cpus!r} is not a number" + ) from exc + if not 0.1 <= cpus <= 16.0: + raise TaskContractError(f"container_cpus {descriptor.container_cpus!r} outside 0.1..16") + try: + pids = int(descriptor.container_pids_limit) + except ValueError as exc: + raise TaskContractError( + f"container_pids_limit {descriptor.container_pids_limit!r} is not an integer" + ) from exc + if not 16 <= pids <= 4096: + raise TaskContractError( + f"container_pids_limit {descriptor.container_pids_limit!r} outside 16..4096" + ) + return descriptor.container_memory, descriptor.container_cpus, descriptor.container_pids_limit + + +def _parse_memory_mb(value: str) -> float: + """Parse a docker-style memory size to MiB.""" + text = value.strip().lower() + factors = {"g": 1024.0, "m": 1.0, "k": 1.0 / 1024} + factor = factors.get(text[-1:], 1.0 / (1024 * 1024)) if text else 0.0 + number = text[:-1] if text and text[-1:] in factors else text + try: + return float(number) * factor + except ValueError as exc: + raise TaskContractError(f"container_memory {value!r} is not a size") from exc + + class JsonSubprocessRuntime: """Persistent JSON-lines child with bounded request latency.""" @@ -184,6 +239,7 @@ def __init__(self, command: tuple[str, ...], *, cwd: Path, timeout_sec: float) - env["TMPDIR"] = str(self._plugin_home) env["TEMP"] = str(self._plugin_home) env["TMP"] = str(self._plugin_home) + env.update(_host_tool_env(command)) self._process = subprocess.Popen( command, cwd=cwd, @@ -210,7 +266,6 @@ def request(self, op: str, payload: dict[str, Any]) -> Any: process.stdin.flush() except BrokenPipeError as exc: raise TaskContractError(self._failure("plugin process exited before request")) from exc - line = _readline_bounded( process.stdout, limit=_MAX_RESPONSE_BYTES, timeout_sec=self._timeout_sec ) @@ -323,6 +378,7 @@ def _container_command( "pass the digest in image_digest, not in image" ) digest = descriptor.image_digest.removeprefix("sha256:") + memory, cpus, pids = _clamp_container_limits(descriptor) inner = [ "python", "-m", @@ -359,11 +415,11 @@ def _container_command( "--security-opt", "no-new-privileges", "--memory", - _CONTAINER_MEMORY, + memory, "--cpus", - _CONTAINER_CPUS, + cpus, "--pids-limit", - _CONTAINER_PIDS_LIMIT, + pids, "-v", f"{root.resolve()}:{_CONTAINER_PKG_DIR}:ro", f"{descriptor.image}@sha256:{digest}", diff --git a/tests/test_attestation.py b/tests/test_attestation.py new file mode 100644 index 0000000..7807008 --- /dev/null +++ b/tests/test_attestation.py @@ -0,0 +1,48 @@ +"""Executor attestation contract: minting needs the secret, forgery fails.""" + +from __future__ import annotations + +import pytest + +from or_audit.eval.attestation import attest, verify + + +def test_attest_verify_roundtrip() -> None: + stamp = attest( + executor_id="machine0-eu-1", + artifact_head="a" * 64, + backend="real", + world_pin="b" * 40, + nonce="nonce-1234", + secret=b"operator-secret", + ) + assert verify(stamp, secret=b"operator-secret") is True + assert verify(stamp, secret=b"wrong-secret") is False + assert verify(stamp, secret=b"") is False + + +def test_tampered_stamp_fails_verification() -> None: + stamp = attest( + executor_id="machine0-eu-1", + artifact_head="a" * 64, + backend="real", + world_pin="b" * 40, + nonce="nonce-1234", + secret=b"operator-secret", + ) + forged = stamp.model_copy(update={"backend": "synthetic-stub"}) + assert verify(forged, secret=b"operator-secret") is False + upgraded = stamp.model_copy(update={"artifact_head": "c" * 64}) + assert verify(upgraded, secret=b"operator-secret") is False + + +def test_minting_needs_a_secret() -> None: + with pytest.raises(ValueError, match="non-empty operator secret"): + attest( + executor_id="e", + artifact_head="a" * 64, + backend="real", + world_pin="b" * 40, + nonce="nonce-1234", + secret=b"", + ) diff --git a/tests/test_eval_plugin_host.py b/tests/test_eval_plugin_host.py index d8ccd69..32a75ef 100644 --- a/tests/test_eval_plugin_host.py +++ b/tests/test_eval_plugin_host.py @@ -461,3 +461,85 @@ def test_readline_bounded_expires_while_dribbling() -> None: assert _readline_bounded(stream, limit=1024, timeout_sec=0.05) is None finally: os_module.close(writer) + + +def test_docker_host_reaches_only_container_spawns(monkeypatch: pytest.MonkeyPatch) -> None: + from or_audit.eval.plugins import _host_tool_env, _scrubbed_plugin_env + + monkeypatch.setenv("DOCKER_HOST", "unix:///tmp/daemon.sock") + assert "DOCKER_HOST" not in _scrubbed_plugin_env() + assert _host_tool_env(("docker", "run")) == {"DOCKER_HOST": "unix:///tmp/daemon.sock"} + assert _host_tool_env(("python", "-m", "x")) == {} + + +def test_container_limits_come_from_descriptor_identity( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + import shutil + + from or_audit.eval.contracts import RuntimeDescriptor, RuntimeKind + from or_audit.eval.plugins import _runtime_command + + monkeypatch.setattr(shutil, "which", lambda name: "/usr/bin/docker") + descriptor = RuntimeDescriptor( + kind=RuntimeKind.CONTAINER, + image="registry.example/p", + image_digest="a" * 64, + container_memory="512m", + container_cpus="0.5", + container_pids_limit="64", + ) + command, _ = _runtime_command( + descriptor, role="predictor", root=tmp_path, entrypoint="spy.py:load_predictor" + ) + joined = " ".join(command) + assert "--memory 512m" in joined + assert "--cpus 0.5" in joined + assert "--pids-limit 64" in joined + assert "2g" not in joined + + +def test_entrypoint_and_weights_confined_to_package(tmp_path: Path) -> None: + from or_audit.eval.integrity import package_file + + outside = tmp_path / "outside.py" + outside.write_text("VALUE = 1\n", encoding="utf-8") + pkg = tmp_path / "pkg" + pkg.mkdir() + (pkg / "ok.py").write_text("def f() -> int:\n return 1\n", encoding="utf-8") + (pkg / "escape.py").symlink_to(outside) + with pytest.raises(TaskContractError, match="escapes package root"): + load_entrypoint(pkg, "../outside.py:f", label="policy") + with pytest.raises(TaskContractError, match="escapes package root"): + load_entrypoint(pkg, "escape.py:f", label="policy") + with pytest.raises(TaskContractError, match="escapes package root"): + load_entrypoint(pkg, "/etc/hosts:f", label="policy") + with pytest.raises(TaskContractError, match="missing"): + package_file(pkg, "absent.json", label="weights") + assert package_file(pkg, "ok.py", label="policy module").name == "ok.py" + + +def test_container_absurd_limits_are_refused( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + import shutil + + from or_audit.errors import TaskContractError + from or_audit.eval.contracts import RuntimeDescriptor, RuntimeKind + from or_audit.eval.plugins import _runtime_command + + monkeypatch.setattr(shutil, "which", lambda name: "/usr/bin/docker") + base: dict[str, object] = {"kind": RuntimeKind.CONTAINER, "image": "registry.example/p"} + + def refuses(extra: dict[str, object], pattern: str) -> None: + with pytest.raises(TaskContractError, match=pattern): + _runtime_command( + RuntimeDescriptor(**{**base, "image_digest": "a" * 64, **extra}), + role="predictor", + root=tmp_path, + entrypoint="spy.py:load_predictor", + ) + + refuses({"container_pids_limit": "-1"}, r"outside 16\.\.4096") + refuses({"container_memory": "999g"}, r"outside 64m\.\.16g") + refuses({"container_cpus": "0"}, r"outside 0\.1\.\.16")