diff --git a/.github/scripts/build_integration_review_pack.py b/.github/scripts/build_integration_review_pack.py index 9c1446cb0..d9d7db7df 100644 --- a/.github/scripts/build_integration_review_pack.py +++ b/.github/scripts/build_integration_review_pack.py @@ -760,6 +760,8 @@ def hardening_summary_md( cfg = { "network_mode": _cell_network_config(cell), "allowed_hosts": cell.raw.get("allowed_hosts"), + "blocked_urls": cell.raw.get("blocked_urls"), + "blocked_hosts": cell.raw.get("blocked_hosts"), } gate_id, status, detail = rubric_checks.network_hardening( cfg, verifier_or_sandbox_pr=verifier_or_sandbox_pr @@ -794,10 +796,11 @@ def hardening_summary_md( def _cell_network_config(cell: Cell) -> str | None: """Map the cell's EXPECTED network_mode (Q3) to a NetworkMode literal. - The cell carries ``network_mode`` as ``default-off`` | ``allowlist`` (Q3: - derived from the task config, NOT passed to bench). Translate to the - benchflow ``NetworkMode`` literals the static checker understands, or use an - explicit per-cell ``network_mode`` override if the planner emitted one. + The cell carries ``network_mode`` as ``default-off`` | ``allowlist`` | + ``denylist`` (Q3: derived from the task config, NOT passed to bench). + Translate to the benchflow ``NetworkMode`` literals the static checker + understands, or use an explicit per-cell ``network_mode`` override if the + planner emitted one. """ explicit = cell.raw.get("network_mode") mode = str(explicit) if explicit is not None else "default-off" @@ -806,6 +809,8 @@ def _cell_network_config(cell: Cell) -> str | None: return "no-network" if norm == "allowlist": return "allowlist" + if norm == "denylist": + return "denylist" if norm == "public": return "public" return None diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d66c6f8b..ccb5d94db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ ## [Unreleased] +### Added +- **`network_mode: denylist` blocks a list of URLs and hosts for the agent on + Docker and Daytona.** The task keeps internet access; `blocked_urls` and + `blocked_hosts` are enforced by a root-owned loopback proxy behind the + sandbox-user firewall, hosted search tools are switched off per harness, + and every refused request lands in `trajectory/egress_denylist.jsonl`. + Other backends refuse the mode at preflight. (#1113) + ## 0.7.6 — 2026-09-04 ### Added diff --git a/docs/running-benchmarks.md b/docs/running-benchmarks.md index 8da6013b9..d501e57af 100644 --- a/docs/running-benchmarks.md +++ b/docs/running-benchmarks.md @@ -349,13 +349,21 @@ The **Harvey LAB harness** agent is special — it runs Harvey LAB's own agent l | Modal | `--sandbox modal` | Serverless, high concurrency (needs Modal auth) | | AgentCore | `--sandbox agentcore` | AWS-native isolated microVMs (needs AWS credentials) | +`network_mode = "denylist"` tasks run on Docker, where BenchFlow adds +`NET_ADMIN` to the agent container through its own compose overlay, and on +direct Daytona sandboxes, where the mode was verified with `iptables`. The +other backends refuse the mode at preflight. See +[sandbox hardening](./sandbox-hardening.md#network-policy-denylist-egress) +for what the mode does and does not guarantee. + Apple Container requires Apple Container 1.1+ on Apple Silicon and runs the model proxy inside each VM. It supports public-network, single-container arm64 tasks and has no snapshot support. BenchFlow serializes Apple rollouts within each process and blocks new VMs when the live `data.kalloc.1024` headroom is unsafe. Avoid running concurrent BenchFlow processes, because the macOS allocation leak is system-wide. Use Docker, Daytona, or Modal for `network_mode = "no-network"`, -multi-service, snapshot, or high-concurrency runs. +multi-service, snapshot, or high-concurrency runs. Apple Container also +refuses `network_mode = "denylist"`; use Docker or Daytona for those tasks. ### Amazon Bedrock AgentCore @@ -411,7 +419,8 @@ Constraints: `linux/arm64` only, single container (no compose/multi-service tasks), no snapshot support, and `network_mode = "no-network"` is **not** enforceable — AgentCore's network mode is either `PUBLIC` or `VPC`, so BenchFlow refuses no-network tasks on this backend rather than running them -unisolated. The model proxy runs inside the sandbox, as on Daytona and Modal. +unisolated. `network_mode = "denylist"` is refused on this backend as well. +The model proxy runs inside the sandbox, as on Daytona and Modal. Sessions default to a 15-minute idle timeout and an 8-hour lifetime; override with `BENCHFLOW_AGENTCORE_IDLE_TIMEOUT_SEC` / `BENCHFLOW_AGENTCORE_MAX_LIFETIME_SEC` if agent turns are long enough to risk reclamation mid-run. diff --git a/docs/sandbox-hardening.md b/docs/sandbox-hardening.md index cdc7ae870..e76787d78 100644 --- a/docs/sandbox-hardening.md +++ b/docs/sandbox-hardening.md @@ -41,6 +41,60 @@ Unknown keys in `[verifier.hardening]` are warned and ignored. String values for See [`progressive-disclosure.md`](./progressive-disclosure.md#per-task-hardening-opt-outs) for the qutebrowser case study (legitimate `conftest.py` for circular-import fix). +## Network policy: denylist egress + +`network_mode: denylist` keeps the internet reachable and makes a list of URLs and hosts unreachable for the agent. The use case is a task built from a published paper: the agent may search and read freely, but the paper, its mirrors, and its code repository are off limits ([benchflow-ai/FrontierPhysics#365](https://github.com/benchflow-ai/FrontierPhysics/issues/365)). + +```yaml +sandbox: + network_mode: denylist + blocked_urls: + - https://example.org/papers/lattice-qcd-2026 + - github.com/example-org/lattice-qcd-code + blocked_hosts: + - mirror.example.net +``` + +See [task authoring](./task-authoring-task-md.md#network-policy) for the field rules. The proxy filters the agent only; oracle runs and the verifier are not filtered. + +### Mechanism + +1. **Loopback proxy.** Before the agent starts, benchflow uploads a stdlib Python proxy (`src/benchflow/sandbox/_egress_denylist_proxy.py`) and starts it as root on `127.0.0.1:18628`. A request that matches the denylist gets `403 Forbidden` with an `X-BenchFlow-Blocked: 1` header; everything else is tunneled to its destination. +2. **Uid firewall.** The same `iptables` owner rule that backs the no-web mode lets the sandbox user reach loopback only. Every other outbound packet from that uid is rejected, so the proxy is the only way out. `iptables` is installed on first use (apt, dnf, or apk) when the image lacks it. +3. **Proxy and CA environment.** The agent env gets `HTTP_PROXY`, `HTTPS_PROXY`, `NO_PROXY`, `SSL_CERT_FILE`, `REQUESTS_CA_BUNDLE`, `CURL_CA_BUNDLE`, `GIT_SSL_CAINFO`, `NODE_EXTRA_CA_CERTS`, and `NODE_USE_ENV_PROXY`, plus the `BENCHFLOW_EGRESS_DENYLIST=1` marker that arms the firewall. These are added after the sandbox-local LiteLLM gateway starts, so model traffic does not pass through the egress proxy. +4. **Selective TLS interception.** Hosts named in `blocked_urls` need their paths inspected, so the proxy terminates TLS for those hosts with a leaf certificate signed by a per-rollout CA (`BenchFlow egress policy CA`). Certificates are minted on the host; the CA private key never enters the sandbox. Hosts in `blocked_hosts` are refused at `CONNECT` time, and every other host passes through as an opaque tunnel. +5. **Hosted search off.** Provider-side search tools fetch pages from the model provider's servers, outside the sandbox, so the proxy cannot see them. Benchflow disables them per harness: + + | Harness | Switched off | Still on | + |---|---|---| + | `claude-agent-acp` | `WebSearch` | `WebFetch` (fetches from inside the sandbox, through the proxy) | + | `codex-acp` | `tools.web_search` | | + | `gemini` | `google_web_search`, `web_fetch` (tries a hosted fetch first) | | + | `opencode`, `mimo` | `websearch` | `webfetch` | + | other harnesses | nothing | whatever hosted tools they ship | + +6. **Block log.** Each refused attempt is appended to a root-owned log that benchflow downloads to `trajectory/egress_denylist.jsonl` in the rollout directory at cleanup: one JSON object per line with `ts`, `action`, `method`, `url`, and `rule` (`host:`, `url:`, or `ip-literal`). For a refused `CONNECT`, `url` holds the `host:port` the client asked for. + +Matching ignores scheme, port, query string, and case, strips a leading `www.`, and compares a normalized path: percent-encoding is decoded (repeatedly), `.` and `..` segments are resolved, duplicate slashes and backslashes collapse, and `;` path parameters are dropped, so `/abs/../abs/2401.12345` and `/abs/%2e%2e/abs/2401.12345` match the same entry as `/abs/2401.12345`. A `blocked_urls` entry blocks every path under it; a `blocked_hosts` entry blocks the host and its subdomains. Requests to addresses are refused in every notation a resolver accepts (dotted, decimal, hex, octal) and through wildcard DNS names that embed an address (`1-2-3-4.sslip.io`), so a blocked host cannot be reached by its address. A name the agent controls that resolves to the blocked address is not detected; that is the inherent limit of a hostname denylist. Before connecting anywhere, the proxy resolves the destination and refuses names that resolve to loopback, private, link-local, or other non-global addresses (cloud metadata included), so a hostname the agent controls cannot turn the root proxy into a bridge to sandbox-internal or host services. The uid firewall stays for the rest of the sandbox life, as in the no-web mode: a later oracle role in the same sandbox, and a verifier configured with `verifier.user` equal to the sandbox user, run without egress. + +### Requirements + +- A non-root `sandbox_user`. Setup fails closed before any sandbox is created when it is missing. +- `python3` (or `python`) on `PATH` in the task image. The proxy is a stdlib script and installs nothing. +- An ACP agent. Session-factory agents raise at connect time because the uid firewall only runs in the ACP path. +- `docker`: the agent container needs `NET_ADMIN` for `iptables`. Benchflow adds it through its own compose overlay (`src/benchflow/sandbox/_compose_files/docker-compose-net-admin.yaml`), so the task's `Dockerfile` and `docker-compose.yaml` need nothing extra. +- `daytona`: verified on direct sandboxes with `iptables`. +- `modal`, `apple-container`, and `agentcore` refuse the mode at preflight. `bench tasks check --sandbox ` reports it, and the runtime raises before the sandbox is created. + +### Limits + +- **A blocklist hides pages, not knowledge.** The model may have read the paper in training. The denylist only stops it from re-reading the page during the run. +- **Mirrors and citations stay reachable.** Only listed prefixes and hosts are blocked. Preprint mirrors, aggregator copies, cached versions, citing papers that restate the result, and the same repository under another host all stay reachable unless listed. `blocked_hosts` covers subdomains, not other domains. List every form of a page you know about (abstract, PDF, HTML, versioned). +- **The block is visible.** A refused request gets a `403` that names the policy, and the agent can tell intercepted hosts from the certificate issuer (`BenchFlow egress policy CA`). Do not expect the agent to be unaware that a page is off limits. +- **Tools that ignore proxy variables fail closed.** A client that does not honor `HTTP_PROXY` and `HTTPS_PROXY` cannot reach the network at all, because the uid firewall rejects non-loopback traffic. It gets a connection error, not the page, and the attempt does not appear in the block log. +- **Hosted search coverage is per harness.** Only the harnesses in the table have a switch. A harness with a hosted fetch that is not listed there can reach blocked pages through the provider. +- **Only HTTP requests and `CONNECT` targets are inspected.** A tunnel to an unblocked host carries any protocol the client chooses, uninspected. A client that does not speak the HTTP proxy protocol at all is rejected by the firewall rather than filtered. + ## Threat model and known gaps Benchflow's hardening assumes: diff --git a/docs/task-authoring-task-md.md b/docs/task-authoring-task-md.md index 7a04ebd69..e20ecc6e4 100644 --- a/docs/task-authoring-task-md.md +++ b/docs/task-authoring-task-md.md @@ -76,7 +76,7 @@ so typos fail at parse time instead of becoming silently-ignored config: | `metadata` | Freeform mapping — difficulty, category, tags, anything descriptive | | `agent` | Agent run policy: `timeout_sec`, `user`, `network_mode`, `allowed_hosts` | | `verifier` | Verifier run policy: `timeout_sec` (default 600), `env`, `user`, `service`, … | -| `sandbox` | Sandbox: `docker_image`, `cpus`, `memory_mb`, `storage_mb`, `network_mode`, `env`, `workdir`, … (legacy `task.toml` imports convert the Harbor `environment` table to this key; `environment:` in `task.md` is rejected with a rename hint) | +| `sandbox` | Sandbox: `docker_image`, `cpus`, `memory_mb`, `storage_mb`, `network_mode`, `allowed_hosts`, `blocked_urls`, `blocked_hosts`, `env`, `workdir`, … (legacy `task.toml` imports convert the Harbor `environment` table to this key; `environment:` in `task.md` is rejected with a rename hint) | | `oracle` | Oracle run policy: `env`, `timeout_sec` (import alias: `solution`) | | `source`, `artifacts`, `steps`, `multi_step_reward_strategy`, `reward` | Provenance, artifact, and reward metadata | @@ -111,6 +111,41 @@ profile name is a parse error. `bench tasks normalize ` prints the fully expanded canonical document (`--write` replaces `task.md` in place), so a minimal authored file and its canonical form never drift apart. +### Network policy + +`sandbox.network_mode` selects `no-network`, `allowlist`, `public`, or +`denylist`. `denylist` keeps the internet reachable and makes the listed pages +unreachable for the agent. Declare it on `sandbox`; the runtime reads the +sandbox policy when it starts the filter: + +```yaml +sandbox: + network_mode: denylist + blocked_urls: + - https://example.org/papers/lattice-qcd-2026 + - github.com/example-org/lattice-qcd-code + blocked_hosts: + - mirror.example.net +``` + +`blocked_urls` entries are prefixes: the scheme is optional (`https://` is +assumed), the host is lowercased, the query string is dropped, and every path +under the prefix is blocked, including the path itself (`/abs/2401.12345` also +covers `/abs/2401.12345v2`). A trailing slash is dropped. An entry without a +path blocks every path on that host (with or without a leading `www.`) but not +its other subdomains; use `blocked_hosts` for that. `blocked_hosts` entries +block the host and all of its subdomains. IP literals, wildcards, ports, +and userinfo are rejected. `denylist` requires at least one entry in either +list, both lists are rejected under any other mode, and `denylist` is a +sandbox-level policy: `agent.network_mode` and `verifier.network_mode` reject it. + +The mode needs a non-root `sandbox_user` and `python3` in the task image. It +runs on `docker` and `daytona`; other backends refuse it at preflight. Blocked +attempts are written to `trajectory/egress_denylist.jsonl` in the rollout +directory. Read the +[sandbox hardening notes](./sandbox-hardening.md#network-policy-denylist-egress) +before relying on it: a blocklist hides pages, not knowledge. + --- ## Prompt body and prompts/ sidecars diff --git a/docs/task-standard.md b/docs/task-standard.md index 3145f3adc..06455b63b 100644 --- a/docs/task-standard.md +++ b/docs/task-standard.md @@ -840,6 +840,7 @@ Current implementation status: | imported `steps` | yes | no/partial | fail closed per sandbox until implemented | | root/step artifacts | yes | no/partial | implement collection or fail closed | | network allowlist | yes | no/partial | per-sandbox capability check | +| network denylist | yes | partial | `docker`: yes; `daytona`: yes; `modal`, `apple-container`, and `agentcore` refuse the mode at the capability gate | | separate verifier env | yes | no/partial | materializer plus verifier runner support | | Windows / TPU | yes | no | fail closed | | healthcheck | yes | no/partial | fail closed until sandbox healthcheck support lands | diff --git a/src/benchflow/agents/install.py b/src/benchflow/agents/install.py index a65d8d6df..b9c66c08b 100644 --- a/src/benchflow/agents/install.py +++ b/src/benchflow/agents/install.py @@ -99,7 +99,7 @@ def _owner_from_home(home: str) -> str | None: def _policy_home_dirs(agent: str, agent_cfg: AgentConfig) -> list[str]: - """Agent home dirs a no-web setup command may create.""" + """Agent home dirs a web-policy setup command may create.""" dirs = set(agent_cfg.home_dirs) for owned_path in agent_cfg.disallow_web_tools_owned_paths: if not owned_path.startswith("$HOME/"): @@ -265,15 +265,21 @@ async def apply_web_tool_policy( home: str, *, disallow: bool, + disallow_hosted_search: bool = False, ) -> None: - """Apply an agent-specific hard web-tool disable in the agent home.""" - if not disallow or not agent_cfg or not agent_cfg.disallow_web_tools_setup_cmd: + """Apply the agent's no-web or hosted-search-off setup command in the agent home.""" + if not agent_cfg: + return + if disallow: + policy, setup_cmd = "no-web", agent_cfg.disallow_web_tools_setup_cmd + elif disallow_hosted_search: + policy, setup_cmd = "hosted-search", agent_cfg.disallow_hosted_search_setup_cmd + else: + return + if not setup_cmd: return - cmd = ( - f"export BENCHFLOW_AGENT_HOME={shlex.quote(home)}; " - f"{agent_cfg.disallow_web_tools_setup_cmd}" - ) + cmd = f"export BENCHFLOW_AGENT_HOME={shlex.quote(home)}; {setup_cmd}" owner = _owner_from_home(home) if owner: q_owner = shlex.quote(owner) @@ -296,7 +302,7 @@ async def apply_web_tool_policy( if stderr: details.append(f"stderr: {stderr}") raise RuntimeError( - f"Failed to apply no-web policy for {agent}: {'; '.join(details)}" + f"Failed to apply {policy} policy for {agent}: {'; '.join(details)}" ) diff --git a/src/benchflow/agents/manifest.py b/src/benchflow/agents/manifest.py index daebc8cc0..e4f676ae2 100644 --- a/src/benchflow/agents/manifest.py +++ b/src/benchflow/agents/manifest.py @@ -89,6 +89,8 @@ "disallow_web_tools_setup_cmd", "disallow_web_tools_owned_paths", "disallow_web_tools_launch_suffix", + "disallow_hosted_search_setup_cmd", + "disallow_hosted_search_launch_suffix", "task_mcp_transport", "task_mcp_config_path", } diff --git a/src/benchflow/agents/registry.py b/src/benchflow/agents/registry.py index 127ef423f..b3d17539a 100644 --- a/src/benchflow/agents/registry.py +++ b/src/benchflow/agents/registry.py @@ -502,6 +502,12 @@ class AgentConfig: # String appended to launch_cmd when BenchFlow's no-web policy is active. # Use for agents whose supported toggle is a launch/config override. disallow_web_tools_launch_suffix: str = "" + # Shell snippet that switches off hosted (provider-side) search tools when + # the denylist egress policy is active; local fetch tools stay on because + # they go through the egress proxy. Reuses disallow_web_tools_owned_paths. + disallow_hosted_search_setup_cmd: str = "" + # String appended to launch_cmd when the denylist egress policy is active. + disallow_hosted_search_launch_suffix: str = "" # How task-declared MCP servers are delivered to the agent: # "acp" sends them in session/new; "native-config" writes an agent-specific # config file before launch (for agents whose ACP server drops/reformats @@ -511,6 +517,15 @@ class AgentConfig: task_mcp_config_path: str = "" +_GEMINI_EXCLUDE_WEB_TOOLS_CMD = _json_settings_merge( + "$BENCHFLOW_AGENT_HOME/.gemini/settings.json", + 'd.setdefault("tools",{}).setdefault("exclude",[]);' + '[d["tools"]["exclude"].append(t) for t in ' + '["google_web_search","web_fetch"] ' + 'if t not in d["tools"]["exclude"]]', +) + + # Agent registry — all supported agents AGENTS: dict[str, AgentConfig] = { "claude-agent-acp": AgentConfig( @@ -552,6 +567,12 @@ class AgentConfig: 'if t not in d["permissions"]["deny"]]', ), disallow_web_tools_owned_paths=["$HOME/.claude"], + disallow_hosted_search_setup_cmd=_json_settings_merge( + "$BENCHFLOW_AGENT_HOME/.claude/settings.json", + 'd.setdefault("permissions",{}).setdefault("deny",[]);' + '[d["permissions"]["deny"].append(t) for t in ["WebSearch"] ' + 'if t not in d["permissions"]["deny"]]', + ), supports_acp_set_model=False, acp_model_config_id="model", acp_effort_config_id="effort", @@ -650,6 +671,7 @@ class AgentConfig: ], ), disallow_web_tools_launch_suffix=" -c tools.web_search=false", + disallow_hosted_search_launch_suffix=" -c tools.web_search=false", ), "gemini": AgentConfig( name="gemini", @@ -694,14 +716,11 @@ class AgentConfig: ), ], ), - disallow_web_tools_setup_cmd=_json_settings_merge( - "$BENCHFLOW_AGENT_HOME/.gemini/settings.json", - 'd.setdefault("tools",{}).setdefault("exclude",[]);' - '[d["tools"]["exclude"].append(t) for t in ' - '["google_web_search","web_fetch"] ' - 'if t not in d["tools"]["exclude"]]', - ), + disallow_web_tools_setup_cmd=_GEMINI_EXCLUDE_WEB_TOOLS_CMD, disallow_web_tools_owned_paths=["$HOME/.gemini"], + # web_fetch tries the hosted urlContext path before any local fetch, so + # the egress proxy cannot filter it; the denylist excludes both tools. + disallow_hosted_search_setup_cmd=_GEMINI_EXCLUDE_WEB_TOOLS_CMD, ), "opencode": AgentConfig( name="opencode", @@ -730,6 +749,10 @@ class AgentConfig: 'd.setdefault("tools",{})["webfetch"]=False', ), disallow_web_tools_owned_paths=["$HOME/.config/opencode"], + disallow_hosted_search_setup_cmd=_json_settings_merge( + "$BENCHFLOW_AGENT_HOME/.config/opencode/opencode.json", + 'd.setdefault("tools",{})["websearch"]=False', + ), ), "mimo": AgentConfig( name="mimo", @@ -779,6 +802,10 @@ class AgentConfig: 'd.setdefault("tools",{})["webfetch"]=False', ), disallow_web_tools_owned_paths=["$HOME/.config/mimocode"], + disallow_hosted_search_setup_cmd=_json_settings_merge( + "$BENCHFLOW_AGENT_HOME/.config/mimocode/mimocode.json", + 'd.setdefault("tools",{})["websearch"]=False', + ), ), "harvey-lab-harness": AgentConfig( name="harvey-lab-harness", @@ -1183,6 +1210,8 @@ def _acpx_wrap(config: AgentConfig) -> AgentConfig: disallow_web_tools_setup_cmd=config.disallow_web_tools_setup_cmd, disallow_web_tools_owned_paths=config.disallow_web_tools_owned_paths, disallow_web_tools_launch_suffix=config.disallow_web_tools_launch_suffix, + disallow_hosted_search_setup_cmd=config.disallow_hosted_search_setup_cmd, + disallow_hosted_search_launch_suffix=config.disallow_hosted_search_launch_suffix, task_mcp_transport=config.task_mcp_transport, task_mcp_config_path=config.task_mcp_config_path, ) @@ -1377,6 +1406,8 @@ def register_agent( disallow_web_tools_setup_cmd: str = "", disallow_web_tools_owned_paths: list[str] | None = None, disallow_web_tools_launch_suffix: str = "", + disallow_hosted_search_setup_cmd: str = "", + disallow_hosted_search_launch_suffix: str = "", ) -> AgentConfig: """Register a custom agent at runtime. @@ -1416,6 +1447,8 @@ def register_agent( disallow_web_tools_setup_cmd=disallow_web_tools_setup_cmd, disallow_web_tools_owned_paths=disallow_web_tools_owned_paths or [], disallow_web_tools_launch_suffix=disallow_web_tools_launch_suffix, + disallow_hosted_search_setup_cmd=disallow_hosted_search_setup_cmd, + disallow_hosted_search_launch_suffix=disallow_hosted_search_launch_suffix, ) AGENTS[name] = config AGENT_INSTALLERS[name] = install_cmd diff --git a/src/benchflow/contracts/planes.py b/src/benchflow/contracts/planes.py index 839be0937..90492b078 100644 --- a/src/benchflow/contracts/planes.py +++ b/src/benchflow/contracts/planes.py @@ -8,10 +8,13 @@ from __future__ import annotations from pathlib import Path -from typing import Any, Protocol, runtime_checkable +from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable from benchflow.environment.manifest import EnvironmentManifest +if TYPE_CHECKING: + from benchflow.sandbox.egress_denylist import EgressDenylist + class LiveUsageGateway(Protocol): """A provider gateway process that reports live cumulative token usage. @@ -33,7 +36,13 @@ def live_usage_tokens(self) -> int | None: ... class RolloutPlanes(Protocol): """Concrete-plane operations the rollout kernel needs.""" - def agent_launch(self, agent: str, *, disallow_web_tools: bool) -> str: ... + def agent_launch( + self, + agent: str, + *, + disallow_web_tools: bool, + disallow_hosted_search: bool = False, + ) -> str: ... def agent_config(self, agent: str) -> Any: ... def resolve_agent_env( @@ -102,6 +111,10 @@ async def apply_web_tool_policy(self, *args: Any, **kwargs: Any) -> None: ... async def link_skill_paths(self, *args: Any, **kwargs: Any) -> None: ... async def ensure_litellm_runtime(self, *args: Any, **kwargs: Any) -> Any: ... async def stop_provider_runtime(self, runtime: Any) -> None: ... + async def start_egress_denylist( + self, env: Any, sandbox_user: str | None, denylist: EgressDenylist + ) -> None: ... + async def stop_egress_denylist(self, env: Any, rollout_dir: Path) -> None: ... def extract_usage(self, runtime: Any) -> dict[str, Any]: ... async def connect_acp(self, *args: Any, **kwargs: Any) -> Any: ... async def execute_prompts(self, *args: Any, **kwargs: Any) -> Any: ... diff --git a/src/benchflow/rollout/__init__.py b/src/benchflow/rollout/__init__.py index f7f6695ca..daadd2655 100644 --- a/src/benchflow/rollout/__init__.py +++ b/src/benchflow/rollout/__init__.py @@ -149,6 +149,7 @@ from benchflow.rollout._setup import ( _task_disallows_internet as _task_disallows_internet, ) +from benchflow.rollout._setup import _task_egress_denylist as _task_egress_denylist from benchflow.rollout._setup import _verify_rollout as _verify_rollout from benchflow.rollout._skills import ( _resolve_skill_creator_root as _resolve_skill_creator_root, @@ -197,6 +198,7 @@ from benchflow.rollout.task_runtime import TaskRuntimeResult as TaskRuntimeResult from benchflow.rollout_branch import ChildRunner from benchflow.rollout_branch import branch as _branch_engine +from benchflow.sandbox.egress_denylist import EgressDenylist, denylist_agent_env from benchflow.sandbox.metadata import persist_sandbox_info from benchflow.scenes import compile_scenes_to_steps from benchflow.scenes import scene_step_prompt as scene_step_prompt @@ -658,6 +660,8 @@ def __init__(self, config: RolloutConfig) -> None: self._timing: dict[str, float] = {} self._effective_locked: list[str] = [] self._disallow_web_tools: bool = False + self._egress_denylist: EgressDenylist | None = None + self._disallow_hosted_search: bool = False self._effective_skills_dir: Path | None = None self._effective_skills_sandbox_dir: str | None = None # Task dir actually deployed: a temp copy (self._task_tmp) when @@ -952,6 +956,14 @@ async def setup(self) -> None: self._disallow_web_tools = ( _task_disallows_internet(self._task) or cfg.self_gen_no_internet ) and cfg.primary_agent != "oracle" + self._egress_denylist = ( + None + if self._disallow_web_tools or cfg.primary_agent == "oracle" + else _task_egress_denylist(self._task) + ) + if self._egress_denylist is not None and not cfg.sandbox_user: + raise ValueError("network_mode='denylist' requires a sandbox_user") + self._disallow_hosted_search = self._egress_denylist is not None self._agent_env = _apply_web_policy( self._planes.resolve_agent_env( cfg.primary_agent, cfg.primary_model, cfg.agent_env @@ -973,6 +985,7 @@ async def setup(self) -> None: self._agent_launch = self._planes.agent_launch( cfg.primary_agent, disallow_web_tools=self._disallow_web_tools, + disallow_hosted_search=self._disallow_hosted_search, ) # Copy task dir to temp when Dockerfile mutations are needed @@ -1231,6 +1244,7 @@ async def install_agent(self) -> None: self._agent_cfg, cred_home, disallow=self._disallow_web_tools, + disallow_hosted_search=self._disallow_hosted_search, ) await self._planes.snapshot_build_config(self._env, workspace=self._agent_cwd) await self._planes.seed_verifier_workspace( @@ -1274,11 +1288,18 @@ def _session_factory_entrypoint(self, agent_name: str) -> str | None: return cfg.session_factory return None + async def _start_egress_denylist(self, denylist: EgressDenylist) -> None: + """(Re)start the egress proxy before an ACP connection; a restored sandbox has none running.""" + await self._planes.start_egress_denylist( + self._env, self._config.sandbox_user, denylist + ) + async def connect(self) -> None: """Open an ACP connection to the agent. Can be called multiple times.""" cfg = self._config rollout_dir = self._require_rollout_dir() t0 = datetime.now() + egress_denylist = getattr(self, "_egress_denylist", None) ( self._agent_env, @@ -1295,11 +1316,16 @@ async def connect(self) -> None: sandbox_setup_timeout=cfg.sandbox_setup_timeout, required_skill_names=getattr(self, "_required_skill_names", ()), live_trajectory_path=rollout_dir / "trajectory" / "llm_trajectory.jsonl", - force_sandbox_local=getattr(self, "_disallow_web_tools", False), + force_sandbox_local=getattr(self, "_disallow_web_tools", False) + or egress_denylist is not None, ) + if egress_denylist is not None: + self._agent_env = denylist_agent_env(self._agent_env) sf_entrypoint = self._session_factory_entrypoint(cfg.primary_agent) self._is_session_factory = sf_entrypoint is not None if sf_entrypoint is not None: + if egress_denylist is not None: + raise RuntimeError("network_mode='denylist' requires an ACP agent") ( self._acp_client, self._session, @@ -1317,6 +1343,8 @@ async def connect(self) -> None: agent_cwd=self._agent_cwd, ) else: + if egress_denylist is not None: + await self._start_egress_denylist(egress_denylist) ( self._acp_client, self._session, @@ -2012,6 +2040,17 @@ async def cleanup(self) -> None: finally: self._usage_runtime = None + rollout_dir = getattr(self, "_rollout_dir", None) + if ( + getattr(self, "_egress_denylist", None) is not None + and self._env is not None + and rollout_dir is not None + ): + try: + await self._planes.stop_egress_denylist(self._env, rollout_dir) + except Exception as e: + logger.warning(f"Egress denylist proxy stop failed: {e}") + self._finalize_usage_metrics() self._enforce_required_usage_tracking() @@ -2272,9 +2311,16 @@ async def connect_as(self, role: Role) -> None: if disallow_web_tools is None: disallow_web_tools = _task_disallows_internet(getattr(self, "_task", None)) disallow_web_tools = bool(disallow_web_tools and role.agent != "oracle") + egress_denylist = ( + None + if disallow_web_tools or role.agent == "oracle" + else _task_egress_denylist(getattr(self, "_task", None)) + ) + disallow_hosted_search = egress_denylist is not None agent_launch = self._planes.agent_launch( role.agent, disallow_web_tools=disallow_web_tools, + disallow_hosted_search=disallow_hosted_search, ) agent_env = _apply_web_policy( self._planes.resolve_agent_env( @@ -2296,8 +2342,10 @@ async def connect_as(self, role: Role) -> None: sandbox_setup_timeout=cfg.sandbox_setup_timeout, required_skill_names=getattr(self, "_required_skill_names", ()), live_trajectory_path=rollout_dir / "trajectory" / "llm_trajectory.jsonl", - force_sandbox_local=disallow_web_tools, + force_sandbox_local=disallow_web_tools or disallow_hosted_search, ) + if egress_denylist is not None: + agent_env = denylist_agent_env(agent_env) role_agent_differs = role.agent != cfg.primary_agent needs_role_credentials = ( @@ -2342,6 +2390,7 @@ async def connect_as(self, role: Role) -> None: agent_cfg, cred_home, disallow=disallow_web_tools, + disallow_hosted_search=disallow_hosted_search, ) self._agent_launch = agent_launch @@ -2349,6 +2398,8 @@ async def connect_as(self, role: Role) -> None: sf_entrypoint = self._session_factory_entrypoint(role.agent) self._is_session_factory = sf_entrypoint is not None if sf_entrypoint is not None: + if egress_denylist is not None: + raise RuntimeError("network_mode='denylist' requires an ACP agent") ( self._acp_client, self._session, @@ -2368,6 +2419,8 @@ async def connect_as(self, role: Role) -> None: agent_cwd=self._agent_cwd, ) else: + if egress_denylist is not None: + await self._start_egress_denylist(egress_denylist) ( self._acp_client, self._session, diff --git a/src/benchflow/rollout/_setup.py b/src/benchflow/rollout/_setup.py index 33c95c7e8..5a7bc6732 100644 --- a/src/benchflow/rollout/_setup.py +++ b/src/benchflow/rollout/_setup.py @@ -42,6 +42,7 @@ validate_reward_map, ) from benchflow.rollout._results import _DIAG_TRUNCATE +from benchflow.sandbox.egress_denylist import EgressDenylist, egress_denylist_for from benchflow.trajectories.types import redact_acp_trajectory_jsonl logger = logging.getLogger(__name__) @@ -55,6 +56,11 @@ def _task_disallows_internet(task: Any) -> bool: return getattr(env_config, "allow_internet", True) is False +def _task_egress_denylist(task: Any) -> EgressDenylist | None: + """Return the egress denylist the task's sandbox config declares, if any.""" + return egress_denylist_for(getattr(getattr(task, "config", None), "sandbox", None)) + + def _read_task_instruction(task_path: Path) -> str: """Read the agent-facing instruction from legacy files or ``task.md``.""" document_path = task_path / "task.md" diff --git a/src/benchflow/rollout/task_runtime.py b/src/benchflow/rollout/task_runtime.py index 89c733ef7..ad6723c3b 100644 --- a/src/benchflow/rollout/task_runtime.py +++ b/src/benchflow/rollout/task_runtime.py @@ -159,6 +159,11 @@ async def start(self) -> None: rollout = await Rollout.create(self.config.to_rollout_config()) try: await rollout.setup() + if getattr(rollout, "_egress_denylist", None) is not None: + raise RuntimeError( + "network_mode='denylist' requires an ACP agent rollout; " + "the bash primitive runs without the egress proxy" + ) await rollout.start() await rollout.install_agent() except BaseException: diff --git a/src/benchflow/rollout_planes.py b/src/benchflow/rollout_planes.py index 6d2272d9a..035591ec4 100644 --- a/src/benchflow/rollout_planes.py +++ b/src/benchflow/rollout_planes.py @@ -31,6 +31,11 @@ extract_usage, stop_provider_runtime, ) +from benchflow.sandbox.egress_denylist import ( + EgressDenylist, + start_egress_denylist, + stop_egress_denylist, +) from benchflow.sandbox.lockdown import ( _resolve_locked_paths, _seed_verifier_workspace, @@ -53,13 +58,21 @@ class DefaultRolloutPlanes: """Default bindings for the four concrete planes.""" - def agent_launch(self, agent: str, *, disallow_web_tools: bool) -> str: + def agent_launch( + self, + agent: str, + *, + disallow_web_tools: bool, + disallow_hosted_search: bool = False, + ) -> str: launch = AGENT_LAUNCH.get(agent, agent) - if not disallow_web_tools: - return launch agent_cfg = AGENTS.get(agent) - if agent_cfg and agent_cfg.disallow_web_tools_launch_suffix: + if agent_cfg is None: + return launch + if disallow_web_tools: return launch + agent_cfg.disallow_web_tools_launch_suffix + if disallow_hosted_search: + return launch + agent_cfg.disallow_hosted_search_launch_suffix return launch def agent_config(self, agent: str) -> Any: @@ -179,6 +192,14 @@ async def ensure_litellm_runtime(self, *args: Any, **kwargs: Any) -> Any: async def stop_provider_runtime(self, runtime: Any) -> None: await stop_provider_runtime(runtime) + async def start_egress_denylist( + self, env: Any, sandbox_user: str | None, denylist: EgressDenylist + ) -> None: + await start_egress_denylist(env, sandbox_user, denylist) + + async def stop_egress_denylist(self, env: Any, rollout_dir: Path) -> None: + await stop_egress_denylist(env, rollout_dir) + def extract_usage(self, runtime: Any) -> dict[str, Any]: return extract_usage(runtime) diff --git a/src/benchflow/sandbox/_compose.py b/src/benchflow/sandbox/_compose.py index 2217aacff..02d1def06 100644 --- a/src/benchflow/sandbox/_compose.py +++ b/src/benchflow/sandbox/_compose.py @@ -9,6 +9,7 @@ COMPOSE_BUILD_PATH = COMPOSE_DIR / "docker-compose-build.yaml" COMPOSE_PREBUILT_PATH = COMPOSE_DIR / "docker-compose-prebuilt.yaml" COMPOSE_NO_NETWORK_PATH = COMPOSE_DIR / "docker-compose-no-network.yaml" +COMPOSE_NET_ADMIN_PATH = COMPOSE_DIR / "docker-compose-net-admin.yaml" # Back-off delays for retrying a `compose up` that hit a daemon-side network # create/attach race. Shared by the host docker.py path and the Daytona DinD diff --git a/src/benchflow/sandbox/_compose_files/docker-compose-net-admin.yaml b/src/benchflow/sandbox/_compose_files/docker-compose-net-admin.yaml new file mode 100644 index 000000000..116c06f9a --- /dev/null +++ b/src/benchflow/sandbox/_compose_files/docker-compose-net-admin.yaml @@ -0,0 +1,4 @@ +services: + main: + cap_add: + - NET_ADMIN diff --git a/src/benchflow/sandbox/_egress_denylist_proxy.py b/src/benchflow/sandbox/_egress_denylist_proxy.py new file mode 100644 index 000000000..b00252dd4 --- /dev/null +++ b/src/benchflow/sandbox/_egress_denylist_proxy.py @@ -0,0 +1,494 @@ +"""Loopback egress proxy for network_mode='denylist'. Stdlib only; runs inside the sandbox as root. + +Hosts named in ``blocked_urls`` are TLS-intercepted with pre-generated +certificates so the full path is visible; every other host passes through as +an opaque CONNECT tunnel. +""" + +from __future__ import annotations + +import argparse +import contextlib +import ipaddress +import json +import re +import socket +import socketserver +import ssl +import sys +import threading +import urllib.parse +from datetime import datetime, timezone +from pathlib import Path +from typing import cast + +HEAD_LIMIT = 64 * 1024 +HEAD_TIMEOUT = 30 +IDLE_TIMEOUT = 900 +BLOCK_BODY = "Blocked by the task network policy: {url}\n" + + +def host_key(host: str) -> str: + """Comparison form of a hostname: lowercase, no trailing dot, no leading www.""" + host = host.strip().rstrip(".").lower() + return host[4:] if host.startswith("www.") else host + + +def _path_key(path: str) -> str: + """Comparison form of a path: decoded, dot segments resolved, one slash between segments.""" + path = path.split("?", 1)[0].split("#", 1)[0] + for _ in range(3): + decoded = urllib.parse.unquote(path) + if decoded == path: + break + path = decoded + path = path.replace("\\", "/") + segments: list[str] = [] + for segment in path.split("/"): + segment = segment.split(";", 1)[0] + if segment in ("", "."): + continue + if segment == "..": + if segments: + segments.pop() + continue + segments.append(segment) + key = "/" + "/".join(segments) + if path.endswith(("/", "/.", "/..")) and key != "/": + key += "/" + return key.lower() + + +_EMBEDDED_IPV4 = re.compile(r"(?:^|[.-])(?:\d{1,3}[.-]){3}\d{1,3}(?:[.-]|$)") +_HEX_IPV4_LABEL = re.compile(r"^[0-9a-f]{8}$") +_WILDCARD_DNS = ( + "nip.io", + "sslip.io", + "xip.io", + "traefik.me", + "localtest.me", + "lvh.me", + "vcap.me", +) + + +def _looks_like_address(host: str) -> bool: + """True for anything that names an address rather than a site. + + glibc accepts decimal, octal, hex and short dotted forms (``3232235777``, + ``0xc0a80101``, ``0300.0250.1.1``, ``127.1``), and wildcard DNS services + resolve an address embedded in the name (``1-2-3-4.sslip.io``). + """ + try: + ipaddress.ip_address(host) + return True + except ValueError: + pass + labels = host.split(".") + if not any(c.isalpha() for c in host): + return True + if labels[-1].isdigit(): + return True + if any(label.startswith("0x") or _HEX_IPV4_LABEL.match(label) for label in labels): + return True + if _EMBEDDED_IPV4.search(host): + return True + return any(host == d or host.endswith("." + d) for d in _WILDCARD_DNS) + + +class Policy: + """Match hosts and URLs against the denylist; scheme, port and query are ignored.""" + + def __init__(self, blocked_urls: list[str], blocked_hosts: list[str]): + self.prefixes: list[tuple[str, str]] = [] + for raw in blocked_urls: + url = raw if "://" in raw else "https://" + raw + parts = urllib.parse.urlsplit(url) + if not parts.hostname: + raise ValueError(f"blocked_urls entry has no host: {raw!r}") + prefix = _path_key(parts.path).rstrip("/") or "/" + self.prefixes.append((host_key(parts.hostname), prefix)) + self.hosts = {h.strip().rstrip(".").lower() for h in blocked_hosts if h.strip()} + self.inspect_hosts = {host for host, _ in self.prefixes} + + @classmethod + def load(cls, path: str) -> Policy: + with open(path, encoding="utf-8") as fh: + data = json.load(fh) + return cls( + list(data.get("blocked_urls") or []), + list(data.get("blocked_hosts") or []), + ) + + def host_rule(self, host: str) -> str | None: + name = host.strip().rstrip(".").lower() + if _looks_like_address(name): + return "ip-literal" + for blocked in self.hosts: + if name == blocked or name.endswith("." + blocked): + return f"host:{blocked}" + return None + + def url_rule(self, host: str, path: str) -> str | None: + rule = self.host_rule(host) + if rule: + return rule + key, pkey = host_key(host), _path_key(path) + for bhost, bpath in self.prefixes: + if key == bhost and pkey.startswith(bpath): + return f"url:{bhost}{bpath}" + return None + + def inspect(self, host: str) -> bool: + return host_key(host) in self.inspect_hosts + + +class CertStore: + """Server TLS contexts for intercepted hosts, from ``/.pem`` (cert + key).""" + + def __init__(self, cert_dir: str): + self.cert_dir = Path(cert_dir) + self._lock = threading.Lock() + self._contexts: dict[str, ssl.SSLContext] = {} + + def context_for(self, host: str) -> ssl.SSLContext: + key = host_key(host) + with self._lock: + ctx = self._contexts.get(key) + if ctx is None: + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ctx.load_cert_chain(str(self.cert_dir / f"{key}.pem")) + ctx.set_alpn_protocols(["http/1.1"]) + self._contexts[key] = ctx + return ctx + + +class Log: + def __init__(self, path: str | None): + self.path = path + self._lock = threading.Lock() + + def write(self, **fields: object) -> None: + stamp = datetime.now(timezone.utc).isoformat(timespec="seconds") # noqa: UP017 + line = json.dumps({"ts": stamp, **fields}, sort_keys=True) + with self._lock: + if self.path: + with open(self.path, "a", encoding="utf-8") as fh: + fh.write(line + "\n") + else: + print(line, file=sys.stderr, flush=True) + + +def _read_head(sock: socket.socket) -> bytes: + buf = b"" + sock.settimeout(HEAD_TIMEOUT) + while b"\r\n\r\n" not in buf: + chunk = sock.recv(4096) + if not chunk: + raise ConnectionError("client closed before request head") + buf += chunk + if len(buf) > HEAD_LIMIT: + raise ConnectionError("request head too large") + return buf + + +def _parse_head(head: bytes) -> tuple[str, str, str, list[tuple[str, str]], bytes]: + raw, _, rest = head.partition(b"\r\n\r\n") + lines = raw.decode("latin-1").split("\r\n") + parts = lines[0].split(" ") + if len(parts) != 3: + raise ConnectionError(f"bad request line {lines[0]!r}") + headers = [] + for line in lines[1:]: + name, sep, value = line.partition(":") + if sep: + headers.append((name.strip(), value.strip())) + return parts[0], parts[1], parts[2], headers, rest + + +_HOP_HEADERS = {"proxy-connection", "proxy-authorization", "connection", "keep-alive"} +_ABSOLUTE_FORM = re.compile(r"^https?://", re.IGNORECASE) + + +class _PrivateDestination(Exception): + """The destination resolves to a loopback, private or otherwise non-global address.""" + + +def _resolve(host: str, port: int) -> list[str]: + addresses: dict[str, None] = {} + for _family, _type, _proto, _name, sockaddr in socket.getaddrinfo( + host, port, type=socket.SOCK_STREAM + ): + addresses.setdefault(str(sockaddr[0])) + return list(addresses) + + +def _upstream_allowed(address: str) -> bool: + try: + return ipaddress.ip_address(address).is_global + except ValueError: + return False + + +def _connect_upstream(host: str, port: int) -> socket.socket: + """Connect to a vetted address of ``host``; the root proxy must not reach sandbox-internal services.""" + addresses = _resolve(host, port) + if not addresses or not all(_upstream_allowed(a) for a in addresses): + raise _PrivateDestination(host) + error: OSError | None = None + for address in addresses: + try: + return socket.create_connection((address, port), timeout=HEAD_TIMEOUT) + except OSError as exc: + error = exc + raise error or OSError(f"cannot connect to {host}:{port}") + + +def _body_prefix(headers: list[tuple[str, str]], rest: bytes) -> bytes: + """Bytes after the head that belong to this request's body; a pipelined request is dropped.""" + names = {n.lower(): v for n, v in headers} + if "transfer-encoding" in names: + return rest + try: + return rest[: int(names.get("content-length", "0"))] + except ValueError: + return b"" + + +def _build_head( + method: str, target: str, version: str, headers: list[tuple[str, str]] +) -> bytes: + kept = [(n, v) for n, v in headers if n.lower() not in _HOP_HEADERS] + kept.append(("Connection", "close")) + lines = [f"{method} {target} {version}"] + [f"{n}: {v}" for n, v in kept] + return ("\r\n".join(lines) + "\r\n\r\n").encode("latin-1") + + +def _response(status: str, body: str, extra: str = "") -> bytes: + data = body.encode("utf-8") + head = ( + f"HTTP/1.1 {status}\r\nContent-Type: text/plain; charset=utf-8\r\n" + f"Content-Length: {len(data)}\r\nConnection: close\r\n{extra}\r\n" + ) + return head.encode("latin-1") + data + + +def _pump(src: socket.socket, dst: socket.socket) -> None: + """Copy until EOF, then half-close the destination so the other direction can finish.""" + try: + while True: + data = src.recv(65536) + if not data: + break + dst.sendall(data) + except OSError: + with contextlib.suppress(OSError): + dst.shutdown(socket.SHUT_RDWR) + return + with contextlib.suppress(OSError): + dst.shutdown(socket.SHUT_WR) + + +def _relay(client: socket.socket, upstream: socket.socket) -> None: + client.settimeout(IDLE_TIMEOUT) + upstream.settimeout(IDLE_TIMEOUT) + t = threading.Thread(target=_pump, args=(upstream, client), daemon=True) + t.start() + _pump(client, upstream) + t.join() + with contextlib.suppress(OSError): + upstream.close() + + +class Proxy(socketserver.ThreadingTCPServer): + daemon_threads = True + allow_reuse_address = True + + def __init__( + self, + addr: tuple[str, int], + policy: Policy, + certs: CertStore, + log: Log, + upstream_ca: str | None = None, + ): + super().__init__(addr, Handler) + self.policy = policy + self.certs = certs + self.log = log + self.upstream_ctx = ssl.create_default_context(cafile=upstream_ca) + + +class Handler(socketserver.BaseRequestHandler): + @property + def proxy(self) -> Proxy: + return cast(Proxy, self.server) + + def handle(self) -> None: + try: + self._handle() + except (OSError, ConnectionError, ssl.SSLError, ValueError): + pass + finally: + with contextlib.suppress(OSError): + self.request.close() + + def _handle(self) -> None: + method, target, version, headers, rest = _parse_head(_read_head(self.request)) + if method == "CONNECT": + self._connect(target, rest) + elif target.startswith("/"): + if target == "/healthz": + self.request.sendall(_response("200 OK", "ok\n")) + else: + self.request.sendall( + _response("400 Bad Request", "absolute URL required\n") + ) + else: + self._forward( + self.request, method, target, version, headers, rest, secure=False + ) + + def _deny(self, sock: socket.socket, method: str, url: str, rule: str) -> None: + self.proxy.log.write(action="blocked", method=method, url=url, rule=rule) + sock.sendall( + _response( + "403 Forbidden", + BLOCK_BODY.format(url=url), + "X-BenchFlow-Blocked: 1\r\n", + ) + ) + + def _connect(self, target: str, early: bytes) -> None: + host, _, port_s = target.rpartition(":") + host = host.strip("[]").rstrip(".").lower() + port = int(port_s) if port_s.isdigit() else 443 + rule = self.proxy.policy.host_rule(host) + if rule: + self._deny(self.request, "CONNECT", f"{host}:{port}", rule) + return + if self.proxy.policy.inspect(host): + self.request.sendall(b"HTTP/1.1 200 Connection Established\r\n\r\n") + tls = self.proxy.certs.context_for(host).wrap_socket( + self.request, server_side=True + ) + method, path, ver, headers, rest = _parse_head(_read_head(tls)) + self._forward( + tls, method, path, ver, headers, rest, secure=True, host=host, port=port + ) + return + try: + upstream = _connect_upstream(host, port) + except _PrivateDestination: + self._deny(self.request, "CONNECT", f"{host}:{port}", "private-address") + return + except OSError as exc: + self.request.sendall( + _response( + "502 Bad Gateway", f"cannot connect to {host}:{port}: {exc}\n" + ) + ) + return + self.request.sendall(b"HTTP/1.1 200 Connection Established\r\n\r\n") + if early: + upstream.sendall(early) + _relay(self.request, upstream) + + def _forward( + self, + sock: socket.socket, + method: str, + target: str, + version: str, + headers: list[tuple[str, str]], + rest: bytes, + *, + secure: bool, + host: str = "", + port: int = 0, + ) -> None: + if secure: + path = target + if _ABSOLUTE_FORM.match(target): + path = "/" + target.split("://", 1)[1].partition("/")[2] + url = f"https://{host}{path}" + authority = host if port == 443 else f"{host}:{port}" + else: + parts = urllib.parse.urlsplit(target) + host, port = (parts.hostname or "").rstrip("."), parts.port or 80 + path = (parts.path or "/") + (("?" + parts.query) if parts.query else "") + url = f"http://{host}{path}" + authority = host if port == 80 else f"{host}:{port}" + if not host: + sock.sendall(_response("400 Bad Request", "absolute URL required\n")) + return + rule = self.proxy.policy.url_rule(host, path) + if rule: + self._deny(sock, method, url, rule) + return + headers = [(n, v) for n, v in headers if n.lower() != "host"] + headers.insert(0, ("Host", authority)) + rest = _body_prefix(headers, rest) + try: + upstream = _connect_upstream(host, port) + if secure: + upstream = self.proxy.upstream_ctx.wrap_socket( + upstream, server_hostname=host + ) + except _PrivateDestination: + self._deny(sock, method, url, "private-address") + return + except (OSError, ssl.SSLError) as exc: + sock.sendall( + _response( + "502 Bad Gateway", f"cannot connect to {host}:{port}: {exc}\n" + ) + ) + return + upstream.sendall(_build_head(method, path, version, headers) + rest) + _relay(sock, upstream) + + +def serve( + port: int, + policy: Policy, + certs: CertStore, + log: Log, + *, + upstream_ca: str | None = None, +) -> Proxy: + """Bind the proxy on loopback and return it (callers run ``serve_forever``).""" + return Proxy(("127.0.0.1", port), policy, certs, log, upstream_ca) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--port", type=int, required=True) + parser.add_argument( + "--policy", required=True, help="JSON with blocked_urls, blocked_hosts" + ) + parser.add_argument( + "--cert-dir", required=True, help="directory of .pem leaf certificates" + ) + parser.add_argument( + "--log", help="JSONL file for blocked attempts (default stderr)" + ) + parser.add_argument("--upstream-ca", help="CA bundle for upstream TLS (tests)") + args = parser.parse_args(argv) + server = serve( + args.port, + Policy.load(args.policy), + CertStore(args.cert_dir), + Log(args.log), + upstream_ca=args.upstream_ca, + ) + print( + f"egress proxy listening on 127.0.0.1:{args.port}", file=sys.stderr, flush=True + ) + with contextlib.suppress(KeyboardInterrupt): + server.serve_forever(poll_interval=0.5) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/benchflow/sandbox/daytona_dind.py b/src/benchflow/sandbox/daytona_dind.py index 8a77b694c..3a965c783 100644 --- a/src/benchflow/sandbox/daytona_dind.py +++ b/src/benchflow/sandbox/daytona_dind.py @@ -233,6 +233,8 @@ def _compose_file_flags(self) -> list[str]: ] if not self._env.task_env_config.allow_internet: files.append(f"{self._COMPOSE_DIR}/docker-compose-no-network.yaml") + if self._env.task_env_config.network_mode == "denylist": + files.append(f"{self._COMPOSE_DIR}/docker-compose-net-admin.yaml") flags: list[str] = [] for f in files: diff --git a/src/benchflow/sandbox/docker.py b/src/benchflow/sandbox/docker.py index 4ae9f362d..d49d32942 100644 --- a/src/benchflow/sandbox/docker.py +++ b/src/benchflow/sandbox/docker.py @@ -31,13 +31,14 @@ from benchflow.sandbox._compose import ( COMPOSE_BASE_PATH, COMPOSE_BUILD_PATH, + COMPOSE_NET_ADMIN_PATH, COMPOSE_NO_NETWORK_PATH, COMPOSE_PREBUILT_PATH, COMPOSE_UP_RETRY_DELAYS_SEC, is_compose_up_network_race_error, ) from benchflow.sandbox.protocol import SandboxImage -from benchflow.task.config import SandboxConfig +from benchflow.task.config import NetworkMode, SandboxConfig from benchflow.task.env import resolve_env_vars from benchflow.task.paths import RolloutPaths, SandboxPaths @@ -114,6 +115,7 @@ class DockerSandbox(BaseSandbox): _DOCKER_COMPOSE_BUILD_PATH = COMPOSE_BUILD_PATH _DOCKER_COMPOSE_PREBUILT_PATH = COMPOSE_PREBUILT_PATH _DOCKER_COMPOSE_NO_NETWORK_PATH = COMPOSE_NO_NETWORK_PATH + _DOCKER_COMPOSE_NET_ADMIN_PATH = COMPOSE_NET_ADMIN_PATH _image_build_locks: ClassVar[dict[str, asyncio.Lock]] = {} _build_semaphore: ClassVar[asyncio.Semaphore | None] = None @@ -300,6 +302,9 @@ def _docker_compose_paths(self) -> list[Path]: if not self.task_env_config.allow_internet: paths.append(self._DOCKER_COMPOSE_NO_NETWORK_PATH) + if self.task_env_config.network_mode == NetworkMode.DENYLIST: + paths.append(self._DOCKER_COMPOSE_NET_ADMIN_PATH) + return paths def _docker_compose_env(self) -> dict[str, str]: @@ -788,6 +793,8 @@ async def restore(self, image: SandboxImage) -> None: "sleep", "infinity", ] + if self.task_env_config.network_mode == NetworkMode.DENYLIST: + run_cmd.insert(1, "--cap-add=NET_ADMIN") result = await self._docker_cli(run_cmd, check=False) if result.return_code != 0: raise RuntimeError( diff --git a/src/benchflow/sandbox/egress_denylist.py b/src/benchflow/sandbox/egress_denylist.py new file mode 100644 index 000000000..ada31656e --- /dev/null +++ b/src/benchflow/sandbox/egress_denylist.py @@ -0,0 +1,332 @@ +"""network_mode='denylist': a root-owned loopback proxy keeps listed URLs out of the agent's reach. + +The sandbox user can only reach loopback (the uid firewall in ``lockdown``), +so every HTTP(S) request goes through the proxy, which refuses the denylist +and tunnels everything else. Only hosts named in ``blocked_urls`` are +TLS-intercepted; their leaf certificates are minted on the host and the CA +private key never enters the sandbox. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import json +import shlex +import tempfile +import urllib.parse +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any + +from benchflow.sandbox._egress_denylist_proxy import host_key +from benchflow.sandbox.lockdown import ( + EGRESS_DENYLIST_ENV, + _exec_failure_detail, + _exec_return_code, +) + +__all__ = [ + "CA_BUNDLE_PATH", + "CA_CERT_PATH", + "EGRESS_DENYLIST_ENV", + "EGRESS_PORT", + "TRAJECTORY_LOG_NAME", + "EgressDenylist", + "certificate_material", + "denylist_agent_env", + "egress_denylist_for", + "start_egress_denylist", + "stop_egress_denylist", +] + +EGRESS_PORT = 18628 +TRAJECTORY_LOG_NAME = "egress_denylist.jsonl" +RUNTIME_DIR = "/opt/benchflow-egress" +CA_DIR = "/etc/benchflow-egress" +CA_CERT_PATH = f"{CA_DIR}/ca.crt" +CA_BUNDLE_PATH = f"{CA_DIR}/ca-bundle.crt" +PROXY_URL = f"http://127.0.0.1:{EGRESS_PORT}" +NO_PROXY = "127.0.0.1,localhost,::1" + +_PROXY_SCRIPT = Path(__file__).with_name("_egress_denylist_proxy.py") +_LOG_PATH = f"{RUNTIME_DIR}/blocked.jsonl" +_PID_PATH = f"{RUNTIME_DIR}/proxy.pid" +_STDERR_PATH = f"{RUNTIME_DIR}/stderr.log" +_CERT_DAYS = 30 +_HEALTH_POLL_SEC = 0.5 + + +@dataclass(frozen=True) +class EgressDenylist: + """URL prefixes and hosts a denylist task keeps out of reach.""" + + blocked_urls: tuple[str, ...] + blocked_hosts: tuple[str, ...] + + @property + def inspect_hosts(self) -> tuple[str, ...]: + """Hosts that must be TLS-intercepted so their paths are visible.""" + hosts: dict[str, None] = {} + for url in self.blocked_urls: + parts = urllib.parse.urlsplit(url if "://" in url else f"https://{url}") + if parts.hostname: + hosts.setdefault(host_key(parts.hostname)) + return tuple(hosts) + + +def egress_denylist_for(sandbox_config: Any) -> EgressDenylist | None: + """The denylist a sandbox config declares, or None for every other network mode.""" + if getattr(sandbox_config, "network_mode", None) != "denylist": + return None + return EgressDenylist( + tuple(getattr(sandbox_config, "blocked_urls", None) or ()), + tuple(getattr(sandbox_config, "blocked_hosts", None) or ()), + ) + + +def denylist_agent_env(agent_env: dict[str, str]) -> dict[str, str]: + """A copy of ``agent_env`` routed through the proxy and trusting its CA.""" + return { + **agent_env, + EGRESS_DENYLIST_ENV: "1", + "HTTP_PROXY": PROXY_URL, + "HTTPS_PROXY": PROXY_URL, + "http_proxy": PROXY_URL, + "https_proxy": PROXY_URL, + "NO_PROXY": NO_PROXY, + "no_proxy": NO_PROXY, + "SSL_CERT_FILE": CA_BUNDLE_PATH, + "REQUESTS_CA_BUNDLE": CA_BUNDLE_PATH, + "CURL_CA_BUNDLE": CA_BUNDLE_PATH, + "GIT_SSL_CAINFO": CA_BUNDLE_PATH, + "NODE_EXTRA_CA_CERTS": CA_CERT_PATH, + "NODE_USE_ENV_PROXY": "1", + } + + +def certificate_material( + hosts: tuple[str, ...], *, now: datetime | None = None +) -> dict[str, bytes]: + """PEM files for the proxy: ``ca.crt`` plus one ``.pem`` (leaf cert and key) per host.""" + try: + from cryptography import x509 + from cryptography.hazmat.primitives import hashes, serialization + from cryptography.hazmat.primitives.asymmetric import ec + from cryptography.x509.oid import ExtendedKeyUsageOID, NameOID + except ImportError as exc: + raise RuntimeError( + "network_mode='denylist' needs the 'cryptography' package on the host: " + "pip install cryptography" + ) from exc + now = now or datetime.now(UTC) + not_before, not_after = now - timedelta(minutes=5), now + timedelta(days=_CERT_DAYS) + ca_key = ec.generate_private_key(ec.SECP256R1()) + ca_name = x509.Name( + [x509.NameAttribute(NameOID.COMMON_NAME, "BenchFlow egress policy CA")] + ) + ca_cert = ( + x509.CertificateBuilder() + .subject_name(ca_name) + .issuer_name(ca_name) + .public_key(ca_key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(not_before) + .not_valid_after(not_after) + .add_extension(x509.BasicConstraints(ca=True, path_length=0), critical=True) + .add_extension( + x509.KeyUsage( + digital_signature=True, + key_cert_sign=True, + crl_sign=True, + content_commitment=False, + key_encipherment=False, + data_encipherment=False, + key_agreement=False, + encipher_only=False, + decipher_only=False, + ), + critical=True, + ) + .add_extension( + x509.SubjectKeyIdentifier.from_public_key(ca_key.public_key()), + critical=False, + ) + .sign(ca_key, hashes.SHA256()) + ) + files = {"ca.crt": ca_cert.public_bytes(serialization.Encoding.PEM)} + for host in hosts: + key = ec.generate_private_key(ec.SECP256R1()) + cert = ( + x509.CertificateBuilder() + .subject_name(x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, host)])) + .issuer_name(ca_name) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(not_before) + .not_valid_after(not_after) + .add_extension( + x509.SubjectAlternativeName( + [x509.DNSName(host), x509.DNSName(f"www.{host}")] + ), + critical=False, + ) + .add_extension( + x509.BasicConstraints(ca=False, path_length=None), critical=True + ) + .add_extension( + x509.ExtendedKeyUsage([ExtendedKeyUsageOID.SERVER_AUTH]), critical=False + ) + .add_extension( + x509.AuthorityKeyIdentifier.from_issuer_public_key(ca_key.public_key()), + critical=False, + ) + .sign(ca_key, hashes.SHA256()) + ) + files[f"{host}.pem"] = cert.public_bytes( + serialization.Encoding.PEM + ) + key.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.PKCS8, + serialization.NoEncryption(), + ) + return files + + +def _setup_cmd(*, runtime_dir: str = RUNTIME_DIR, ca_dir: str = CA_DIR) -> str: + """Root shell: find python, install the CA and bundle, replace any running proxy, start it detached.""" + q = shlex.quote + ca_cert, bundle = f"{ca_dir}/ca.crt", f"{ca_dir}/ca-bundle.crt" + log, pid = f"{runtime_dir}/blocked.jsonl", f"{runtime_dir}/proxy.pid" + proxy = ( + f'"$PY" {q(runtime_dir + "/proxy.py")} --port {EGRESS_PORT} ' + f"--policy {q(runtime_dir + '/policy.json')} --cert-dir {q(runtime_dir + '/certs')} --log {q(log)}" + ) + return ( + "set -e; " + 'PY="$(command -v python3 || command -v python || true)"; ' + '[ -n "$PY" ] || { echo "network_mode=denylist needs python3 in the task image" >&2; exit 87; }; ' + f"mkdir -p {q(ca_dir)} && chmod 755 {q(ca_dir)}; " + f"cp {q(runtime_dir + '/ca.crt')} {q(ca_cert)} && chmod 644 {q(ca_cert)}; " + 'SYS=""; for f in /etc/ssl/certs/ca-certificates.crt /etc/pki/tls/certs/ca-bundle.crt /etc/ssl/cert.pem; do ' + 'if [ -s "$f" ]; then SYS="$f"; break; fi; done; ' + '[ -n "$SYS" ] || SYS="$("$PY" -c \'import ssl; print(ssl.get_default_verify_paths().cafile or "")\')"; ' + f'if [ -n "$SYS" ] && [ -s "$SYS" ]; then cat "$SYS" {q(ca_cert)} > {q(bundle)}; ' + f"else cp {q(ca_cert)} {q(bundle)}; fi; chmod 644 {q(bundle)}; " + "if command -v update-ca-certificates >/dev/null 2>&1; then " + f"mkdir -p /usr/local/share/ca-certificates && cp {q(ca_cert)} /usr/local/share/ca-certificates/benchflow-egress.crt " + "&& update-ca-certificates >/dev/null 2>&1 || true; fi; " + f"touch {q(log)}; chmod 600 {q(log)}; " + f'if [ -s {q(pid)} ]; then old="$(cat {q(pid)})"; ' + 'kill -TERM "$old" 2>/dev/null || true; ' + 'for i in 1 2 3 4 5 6 7 8 9 10; do kill -0 "$old" 2>/dev/null || break; sleep 0.5; done; fi; ' + f"nohup {proxy} {q(runtime_dir + '/stdout.log')} 2>{q(runtime_dir + '/stderr.log')} & " + f"echo $! > {q(pid)}" + ) + + +def _health_cmd() -> str: + probe = ( + "import urllib.request, sys; " + "opener = urllib.request.build_opener(urllib.request.ProxyHandler({})); " + f'sys.exit(0 if opener.open("{PROXY_URL}/healthz", timeout=2).status == 200 else 1)' + ) + return f'PY="$(command -v python3 || command -v python)"; "$PY" -c {shlex.quote(probe)}' + + +async def _run(env: Any, command: str, label: str, *, timeout_sec: int) -> None: + result = await env.exec(command, user="root", timeout_sec=timeout_sec) + rc = _exec_return_code(result) + if rc != 0: + raise RuntimeError( + f"{label} failed with rc={rc}.{_exec_failure_detail(result)}" + ) + + +async def _upload(env: Any, files: dict[str, bytes]) -> None: + with tempfile.TemporaryDirectory() as tmp: + for name, data in files.items(): + local = Path(tmp) / name + local.parent.mkdir(parents=True, exist_ok=True) + local.write_bytes(data) + await env.upload_file(local, f"{RUNTIME_DIR}/{name}", mode="600") + + +async def _wait_healthy(env: Any, timeout_sec: int) -> None: + deadline = asyncio.get_running_loop().time() + timeout_sec + probe = getattr(env, "exec_transient", None) or env.exec + while True: + result = await probe(_health_cmd(), user="root", timeout_sec=10) + if _exec_return_code(result) == 0: + return + if asyncio.get_running_loop().time() >= deadline: + break + await asyncio.sleep(_HEALTH_POLL_SEC) + tail = await env.exec( + f"tail -c 2000 {shlex.quote(_STDERR_PATH)} 2>/dev/null", + user="root", + timeout_sec=10, + ) + raise RuntimeError( + f"egress denylist proxy did not become healthy within {timeout_sec}s: " + f"{(getattr(tail, 'stdout', '') or '').strip()[-2000:]}" + ) + + +async def start_egress_denylist( + env: Any, + sandbox_user: str | None, + denylist: EgressDenylist, + *, + timeout_sec: int = 120, +) -> None: + """Upload policy, certificates and the proxy script, then start the proxy as root.""" + if not sandbox_user: + raise RuntimeError("network_mode='denylist' requires a sandbox_user") + material = certificate_material(denylist.inspect_hosts) + policy = { + "blocked_urls": list(denylist.blocked_urls), + "blocked_hosts": list(denylist.blocked_hosts), + } + files = { + "policy.json": json.dumps(policy, indent=2).encode("utf-8"), + "proxy.py": _PROXY_SCRIPT.read_bytes(), + "ca.crt": material["ca.crt"], + **{f"certs/{name}": pem for name, pem in material.items() if name != "ca.crt"}, + } + await _run( + env, + f"mkdir -p {shlex.quote(RUNTIME_DIR + '/certs')} && chmod 700 {shlex.quote(RUNTIME_DIR)}", + "egress runtime dir", + timeout_sec=30, + ) + await _upload(env, files) + await _run( + env, _setup_cmd(), "egress denylist proxy setup", timeout_sec=timeout_sec + ) + await _wait_healthy(env, timeout_sec) + + +async def stop_egress_denylist(env: Any, rollout_dir: Path) -> None: + """Pull the block log into the rollout trajectory dir, then stop the proxy; never raises.""" + target = Path(rollout_dir) / "trajectory" / TRAJECTORY_LOG_NAME + try: + target.parent.mkdir(parents=True, exist_ok=True) + await env.download_file(_LOG_PATH, target) + except Exception: + try: + result = await env.exec( + f"cat {shlex.quote(_LOG_PATH)}", user="root", timeout_sec=30 + ) + if _exec_return_code(result) == 0: + target.write_text(getattr(result, "stdout", "") or "", encoding="utf-8") + except Exception: + pass + with contextlib.suppress(Exception): + await env.exec( + f"kill -TERM $(cat {shlex.quote(_PID_PATH)}) 2>/dev/null; " + f"rm -rf {shlex.quote(RUNTIME_DIR)} {shlex.quote(CA_DIR)}", + user="root", + timeout_sec=30, + ) diff --git a/src/benchflow/sandbox/lockdown.py b/src/benchflow/sandbox/lockdown.py index 74eff863e..378973461 100644 --- a/src/benchflow/sandbox/lockdown.py +++ b/src/benchflow/sandbox/lockdown.py @@ -143,24 +143,51 @@ def build_priv_drop_cmd(agent_launch: str, sandbox_user: str) -> str: ) +EGRESS_DENYLIST_ENV = "BENCHFLOW_EGRESS_DENYLIST" + + +def _is_loopback_http(url: str) -> bool: + parsed = urlsplit(url) + return ( + parsed.scheme == "http" + and parsed.hostname in {"127.0.0.1", "localhost"} + and parsed.port is not None + ) + + async def enforce_agent_egress_firewall( env: Any, sandbox_user: str | None, agent_env: dict[str, str], ) -> None: - """Block sandbox-user external egress after ACP bootstrap, before prompting.""" - if not sandbox_user or agent_env.get("BENCHFLOW_DISALLOW_WEB_TOOLS") != "1": + """Block sandbox-user external egress after ACP bootstrap, before prompting. + + Armed by the no-web policy (model traffic must already use the sandbox-local + proxy) or by the denylist mode (all traffic must already use the loopback + egress proxy). + """ + no_web = agent_env.get("BENCHFLOW_DISALLOW_WEB_TOOLS") == "1" + denylist = agent_env.get(EGRESS_DENYLIST_ENV) == "1" + if not (no_web or denylist): + return + if not sandbox_user: + if denylist: + raise RuntimeError("network_mode='denylist' requires a sandbox_user") return base_url = agent_env.get("BENCHFLOW_PROVIDER_BASE_URL") or agent_env.get( "LLM_BASE_URL", "" ) - parsed = urlsplit(base_url) - if ( - parsed.scheme != "http" - or parsed.hostname not in {"127.0.0.1", "localhost"} - or parsed.port is None - ): + if denylist: + if not _is_loopback_http(agent_env.get("HTTPS_PROXY", "")): + raise RuntimeError( + "Denylist agent requires HTTPS_PROXY on an HTTP loopback port" + ) + if base_url and not _is_loopback_http(base_url): + raise RuntimeError( + "Denylist agent requires an HTTP loopback provider base URL with a port" + ) + elif not _is_loopback_http(base_url): raise RuntimeError( "No-web agent requires an HTTP loopback provider base URL with a port" ) diff --git a/src/benchflow/sandbox/providers.py b/src/benchflow/sandbox/providers.py index f26fbcaa3..708e991dc 100644 --- a/src/benchflow/sandbox/providers.py +++ b/src/benchflow/sandbox/providers.py @@ -44,6 +44,9 @@ class SandboxProvider: #: Whether the backend can run a task's docker-compose side services. #: ``False`` means a multi-service task must be refused, not run partially. supports_compose: bool = False + #: Whether the backend can enforce ``network_mode = "denylist"`` (a + #: root-owned loopback proxy behind the uid firewall). + enforces_denylist: bool = False @property def off_box_model(self) -> bool: @@ -59,6 +62,7 @@ def off_box_model(self) -> bool: extra=None, model_proxy=ModelProxyLocation.HOST, supports_compose=True, + enforces_denylist=True, ), SandboxProvider( "daytona", @@ -66,6 +70,7 @@ def off_box_model(self) -> bool: model_proxy=ModelProxyLocation.SANDBOX, # The DinD strategy runs compose inside the sandbox VM. supports_compose=True, + enforces_denylist=True, ), SandboxProvider( "modal", @@ -115,6 +120,10 @@ def off_box_model(self) -> bool: NO_NETWORK_UNSUPPORTED_PROVIDERS: frozenset[str] = frozenset( p.name for p in _PROVIDERS if not p.enforces_no_network ) +#: Providers that cannot enforce ``network_mode = "denylist"``. +DENYLIST_UNSUPPORTED_PROVIDERS: frozenset[str] = frozenset( + p.name for p in _PROVIDERS if not p.enforces_denylist +) def is_known_provider(name: str) -> bool: diff --git a/src/benchflow/task/config.py b/src/benchflow/task/config.py index ddc0b75b5..492b3cff3 100644 --- a/src/benchflow/task/config.py +++ b/src/benchflow/task/config.py @@ -9,11 +9,13 @@ from __future__ import annotations +import ipaddress import re import tomllib import warnings from enum import StrEnum from typing import Any, ClassVar, Literal +from urllib.parse import urlsplit from pydantic import ( AliasChoices, @@ -157,6 +159,7 @@ class NetworkMode(StrEnum): NO_NETWORK = "no-network" PUBLIC = "public" ALLOWLIST = "allowlist" + DENYLIST = "denylist" class TaskOS(StrEnum): @@ -180,36 +183,100 @@ class MultiStepRewardStrategy(StrEnum): FINAL = "final" -def _validate_allowed_hosts(hosts: list[str] | None) -> list[str] | None: +def _validate_hostnames(hosts: list[str] | None, field_name: str) -> list[str] | None: + """Normalize a hostname list, rejecting URLs, ports, paths, and bad labels.""" if hosts is None: return None normalized: list[str] = [] for raw_host in hosts: host = raw_host.strip().lower().rstrip(".") if not host: - raise ValueError("allowed_hosts entries must be non-empty hostnames") + raise ValueError(f"{field_name} entries must be non-empty hostnames") if "://" in host or "/" in host or ":" in host: raise ValueError( - "allowed_hosts entries must be hostnames, not URLs, ports, or paths" + f"{field_name} entries must be hostnames, not URLs, ports, or paths" ) labels = host.split(".") if not all(_NETWORK_HOST_LABEL_PATTERN.match(label) for label in labels): raise ValueError( - "allowed_hosts entries must be valid hostnames containing only " + f"{field_name} entries must be valid hostnames containing only " "letters, digits, hyphens, and dots" ) normalized.append(host) return normalized +def _validate_allowed_hosts(hosts: list[str] | None) -> list[str] | None: + return _validate_hostnames(hosts, "allowed_hosts") + + +def _validate_blocked_hosts(hosts: list[str] | None) -> list[str] | None: + normalized = _validate_hostnames(hosts, "blocked_hosts") + for host in normalized or (): + try: + ipaddress.ip_address(host) + except ValueError: + continue + raise ValueError("blocked_hosts entries must name a hostname, not an IP") + return normalized + + +def _validate_blocked_urls(urls: list[str] | None) -> list[str] | None: + """Normalize URL prefixes to ``scheme://host/path`` with query and fragment dropped.""" + if urls is None: + return None + normalized: dict[str, None] = {} + for raw_url in urls: + url = raw_url.strip() + if not url: + raise ValueError("blocked_urls entries must be non-empty URLs") + if "://" not in url: + url = f"https://{url}" + parts = urlsplit(url) + if parts.scheme not in ("http", "https"): + raise ValueError("blocked_urls entries must use the http or https scheme") + if "@" in parts.netloc: + raise ValueError("blocked_urls entries must not contain userinfo") + if ":" in parts.netloc: + raise ValueError("blocked_urls entries must not contain a port") + hosts = _validate_hostnames([parts.netloc], "blocked_urls") + host = hosts[0] if hosts else "" + try: + ipaddress.ip_address(host) + except ValueError: + pass + else: + raise ValueError("blocked_urls entries must name a hostname, not an IP") + normalized.setdefault(f"{parts.scheme}://{host}{parts.path.rstrip('/')}") + return list(normalized) + + +def _reject_role_denylist(network_mode: NetworkMode | None, role: str) -> None: + if network_mode == NetworkMode.DENYLIST: + raise ValueError( + f"{role}.network_mode='denylist' is not supported; " + "declare the denylist on sandbox.network_mode" + ) + + def _validate_network_policy_fields( network_mode: NetworkMode | None, allowed_hosts: list[str] | None, + blocked_urls: list[str] | None = None, + blocked_hosts: list[str] | None = None, ) -> None: if network_mode == NetworkMode.ALLOWLIST and not allowed_hosts: raise ValueError("allowed_hosts must be non-empty for network_mode='allowlist'") if network_mode != NetworkMode.ALLOWLIST and allowed_hosts: raise ValueError("allowed_hosts is only valid for network_mode='allowlist'") + if network_mode == NetworkMode.DENYLIST and not (blocked_urls or blocked_hosts): + raise ValueError( + "network_mode='denylist' requires blocked_urls or blocked_hosts" + ) + if network_mode != NetworkMode.DENYLIST and (blocked_urls or blocked_hosts): + raise ValueError( + "blocked_urls and blocked_hosts are only valid for network_mode='denylist'" + ) class Author(TaskConfigModel): @@ -488,6 +555,7 @@ def reject_renamed_environment_keys(cls, data: Any) -> Any: @model_validator(mode="after") def validate_verifier_sandbox(self) -> VerifierConfig: + _reject_role_denylist(self.network_mode, "verifier") _validate_network_policy_fields(self.network_mode, self.allowed_hosts) if self.sandbox_mode == VerifierSandboxMode.SHARED and self.sandbox is not None: raise ValueError( @@ -547,6 +615,7 @@ def validate_allowed_hosts(cls, hosts: list[str] | None) -> list[str] | None: @model_validator(mode="after") def validate_network_policy(self) -> AgentConfig: + _reject_role_denylist(self.network_mode, "agent") _validate_network_policy_fields(self.network_mode, self.allowed_hosts) return self @@ -725,6 +794,20 @@ class SandboxConfig(TaskConfigModel): default=None, description="Hostnames reachable when network_mode='allowlist'.", ) + blocked_urls: list[str] | None = Field( + default=None, + description=( + "URL prefixes the agent cannot reach when network_mode='denylist'; " + "scheme optional, query ignored" + ), + ) + blocked_hosts: list[str] | None = Field( + default=None, + description=( + "Hostnames (and their subdomains) the agent cannot reach when " + "network_mode='denylist'" + ), + ) build_timeout_sec: float = 600.0 docker_image: str | None = Field( default=None, @@ -812,6 +895,16 @@ def _parse_size_to_mb(size_str: str) -> int: def validate_allowed_hosts(cls, hosts: list[str] | None) -> list[str] | None: return _validate_allowed_hosts(hosts) + @field_validator("blocked_urls") + @classmethod + def validate_blocked_urls(cls, urls: list[str] | None) -> list[str] | None: + return _validate_blocked_urls(urls) + + @field_validator("blocked_hosts") + @classmethod + def validate_blocked_hosts(cls, hosts: list[str] | None) -> list[str] | None: + return _validate_blocked_hosts(hosts) + @field_validator("os", mode="before") @classmethod def normalize_os(cls, value: Any) -> Any: @@ -821,7 +914,12 @@ def normalize_os(cls, value: Any) -> Any: @model_validator(mode="after") def handle_deprecated_fields_and_network_policy(self) -> SandboxConfig: - _validate_network_policy_fields(self.network_mode, self.allowed_hosts) + _validate_network_policy_fields( + self.network_mode, + self.allowed_hosts, + self.blocked_urls, + self.blocked_hosts, + ) memory = self.__dict__.get("memory") storage = self.__dict__.get("storage") if memory is not None: @@ -862,7 +960,12 @@ def handle_deprecated_fields_and_network_policy(self) -> SandboxConfig: self.allow_internet = False # Reconciliation must never leave the object in a state that # _validate_network_policy_fields itself rejects. - _validate_network_policy_fields(self.network_mode, self.allowed_hosts) + _validate_network_policy_fields( + self.network_mode, + self.allowed_hosts, + self.blocked_urls, + self.blocked_hosts, + ) return self diff --git a/src/benchflow/task/runtime_capabilities.py b/src/benchflow/task/runtime_capabilities.py index e624d5441..3762a49ec 100644 --- a/src/benchflow/task/runtime_capabilities.py +++ b/src/benchflow/task/runtime_capabilities.py @@ -17,6 +17,7 @@ from benchflow.rewards.rubric_config import criteria_aggregate_policy_from_rubric from benchflow.sandbox._compose import compose_definition_path from benchflow.sandbox.providers import ( + DENYLIST_UNSUPPORTED_PROVIDERS, NO_NETWORK_UNSUPPORTED_PROVIDERS, SANDBOX_PROVIDER_SET, SINGLE_CONTAINER_PROVIDERS, @@ -281,6 +282,13 @@ def _append_network_issue( reason="network allowlists are parsed but not enforced per sandbox", sandbox=sandbox, ) + if mode == NetworkMode.DENYLIST and sandbox in DENYLIST_UNSUPPORTED_PROVIDERS: + _issue( + unsupported, + path=path, + reason=f"network_mode='denylist' is not enforced by {sandbox}", + sandbox=sandbox, + ) def _append_document_issues( diff --git a/tests/integration/rubric_checks.py b/tests/integration/rubric_checks.py index d2bff0efd..f4ceecef9 100644 --- a/tests/integration/rubric_checks.py +++ b/tests/integration/rubric_checks.py @@ -1071,7 +1071,17 @@ def _telemetry_shape(evidence: Evidence) -> set[str]: _NET_NO_NETWORK = "no-network" _NET_ALLOWLIST = "allowlist" _NET_PUBLIC = "public" -_VALID_NETWORK_MODES = frozenset({_NET_NO_NETWORK, _NET_ALLOWLIST, _NET_PUBLIC}) +_NET_DENYLIST = "denylist" +_VALID_NETWORK_MODES = frozenset( + {_NET_NO_NETWORK, _NET_ALLOWLIST, _NET_PUBLIC, _NET_DENYLIST} +) + + +def _str_list(value: Any) -> list[str]: + """Non-empty stripped strings from a list-valued config field.""" + if not isinstance(value, list): + return [] + return [str(item).strip() for item in value if str(item).strip()] def _norm_network_mode(value: Any) -> str | None: @@ -1092,22 +1102,22 @@ def network_hardening( Policy (CONTRACT Q3): the default safe posture is ``no-network``. Network access is only acceptable as ``allowlist`` with a NON-EMPTY ``allowed_hosts`` - set. A bare ``public`` mode is always flagged; on a PR that touches the - verifier or the sandbox/lockdown surface a ``public`` mode is a hard - ``fail`` (blocker) because that surface controls the isolation boundary. + set, or as ``denylist`` with a NON-EMPTY ``blocked_urls`` or + ``blocked_hosts`` set. A bare ``public`` mode is always flagged; on a PR + that touches the verifier or the sandbox/lockdown surface a ``public`` mode + is a hard ``fail`` (blocker) because that surface controls the isolation + boundary. Returns the ``V-NETWORK`` gate outcome. ``pass`` for a hardened config, - ``fail`` for an unsafe one (missing allowlist hosts, or ``public`` on a - verifier/sandbox PR), ``quarantine`` for a ``public`` config on an unrelated - PR (documented, needs human sign-off), ``na`` when no policy is declared. + ``fail`` for an unsafe one (missing allowlist hosts, missing denylist + entries, or ``public`` on a verifier/sandbox PR), ``quarantine`` for a + ``public`` config on an unrelated PR (documented, needs human sign-off), + ``na`` when no policy is declared. """ mode = _norm_network_mode(task_config.get("network_mode")) - raw_hosts = task_config.get("allowed_hosts") - allowed_hosts = [ - str(h).strip() - for h in (raw_hosts if isinstance(raw_hosts, list) else []) - if str(h).strip() - ] + allowed_hosts = _str_list(task_config.get("allowed_hosts")) + blocked_urls = _str_list(task_config.get("blocked_urls")) + blocked_hosts = _str_list(task_config.get("blocked_hosts")) if mode is None: # No declared policy => the runtime default (no-network) applies; an @@ -1145,6 +1155,26 @@ def network_hardening( f"allowlist hardened: hosts={sorted(allowed_hosts)}", ) + if mode == _NET_DENYLIST: + if allowed_hosts: + return ( + "V-NETWORK", + "fail", + "allowed_hosts is only valid for network_mode='allowlist'", + ) + if not (blocked_urls or blocked_hosts): + return ( + "V-NETWORK", + "fail", + "denylist without blocked_urls/blocked_hosts", + ) + return ( + "V-NETWORK", + "pass", + f"denylist hardened: urls={sorted(blocked_urls)} " + f"hosts={sorted(blocked_hosts)}", + ) + # mode == public if verifier_or_sandbox_pr: return ( diff --git a/tests/test_agent_registry.py b/tests/test_agent_registry.py index 3ae2d3266..7d6142fc6 100644 --- a/tests/test_agent_registry.py +++ b/tests/test_agent_registry.py @@ -413,6 +413,8 @@ def test_defaults(self, cleanup_agent): assert cfg.session_factory == "" assert cfg.disallow_web_tools_setup_cmd == "" assert cfg.disallow_web_tools_launch_suffix == "" + assert cfg.disallow_hosted_search_setup_cmd == "" + assert cfg.disallow_hosted_search_launch_suffix == "" def test_passes_through_new_fields(self, cleanup_agent): cleanup_agent.append("rt-full-agent") @@ -426,6 +428,8 @@ def test_passes_through_new_fields(self, cleanup_agent): api_protocol="openai-completions", disallow_web_tools_setup_cmd="printf 'no web' > /tmp/policy", disallow_web_tools_launch_suffix=" --no-web", + disallow_hosted_search_setup_cmd="printf 'no search' > /tmp/policy", + disallow_hosted_search_launch_suffix=" --no-search", ) assert cfg.protocol == "session-factory" assert cfg.session_factory == "my_agent.factory:create_agent" @@ -433,6 +437,10 @@ def test_passes_through_new_fields(self, cleanup_agent): assert cfg.api_protocol == "openai-completions" assert cfg.disallow_web_tools_setup_cmd == "printf 'no web' > /tmp/policy" assert cfg.disallow_web_tools_launch_suffix == " --no-web" + assert ( + cfg.disallow_hosted_search_setup_cmd == "printf 'no search' > /tmp/policy" + ) + assert cfg.disallow_hosted_search_launch_suffix == " --no-search" # And the registered entry reflects them. registered = AGENTS["rt-full-agent"] @@ -441,3 +449,4 @@ def test_passes_through_new_fields(self, cleanup_agent): assert registered.default_model == "rt-model-1" assert registered.api_protocol == "openai-completions" assert registered.disallow_web_tools_launch_suffix == " --no-web" + assert registered.disallow_hosted_search_launch_suffix == " --no-search" diff --git a/tests/test_agent_spec.py b/tests/test_agent_spec.py index c58a75410..c2bd7bda5 100644 --- a/tests/test_agent_spec.py +++ b/tests/test_agent_spec.py @@ -108,6 +108,14 @@ def test_acpx_wrap_carries_routing_fields(self): wrapped.disallow_web_tools_launch_suffix == underlying.disallow_web_tools_launch_suffix ) + assert ( + wrapped.disallow_hosted_search_setup_cmd + == underlying.disallow_hosted_search_setup_cmd + ) + assert ( + wrapped.disallow_hosted_search_launch_suffix + == underlying.disallow_hosted_search_launch_suffix + ) def test_acpx_cached_config_keeps_api_protocol(self): """Regression for PR #322: the cached acpx runtime key in AGENTS must diff --git a/tests/test_egress_denylist.py b/tests/test_egress_denylist.py new file mode 100644 index 000000000..090d1a6c3 --- /dev/null +++ b/tests/test_egress_denylist.py @@ -0,0 +1,570 @@ +"""Denylist egress mode: proxy policy, in-process proxy, agent env, start/stop, firewall gate. + +Guards the denylist egress mode added for benchflow-ai/FrontierPhysics#365. +""" + +from __future__ import annotations + +import http.server +import json +import socket +import ssl +import subprocess +import threading +import urllib.error +import urllib.request +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from benchflow.sandbox import _egress_denylist_proxy as proxy_mod +from benchflow.sandbox.egress_denylist import ( + CA_BUNDLE_PATH, + CA_CERT_PATH, + EGRESS_DENYLIST_ENV, + EGRESS_PORT, + TRAJECTORY_LOG_NAME, + EgressDenylist, + _health_cmd, + _setup_cmd, + certificate_material, + denylist_agent_env, + egress_denylist_for, + start_egress_denylist, + stop_egress_denylist, +) +from benchflow.sandbox.lockdown import enforce_agent_egress_firewall + + +class TestPolicy: + def test_url_prefix_matches_across_scheme_case_and_www(self): + policy = proxy_mod.Policy(["https://arxiv.org/abs/2401.12345"], []) + assert ( + policy.url_rule("arxiv.org", "/abs/2401.12345") + == "url:arxiv.org/abs/2401.12345" + ) + assert policy.url_rule("www.arxiv.org", "/abs/2401.12345v2?x=1") is not None + assert policy.url_rule("ARXIV.ORG", "/ABS/2401.12345") is not None + assert policy.url_rule("arxiv.org", "/abs/2401.12346") is None + assert policy.url_rule("arxiv.org", "/") is None + + def test_url_prefix_without_scheme_and_percent_encoding(self): + policy = proxy_mod.Policy(["example.org/Blocked Dir/"], []) + assert policy.url_rule("example.org", "/blocked%20dir/paper.pdf") is not None + assert policy.url_rule("example.org", "/blocked") is None + + @pytest.mark.parametrize( + "path", + [ + "/abs/../abs/2401.12345", + "//abs/2401.12345", + "/abs/./2401.12345", + "/./abs//2401.12345", + "/x/../abs/2401.12345", + "/abs/%2e%2e/abs/2401.12345", + "/abs/%252e%252e/abs/2401.12345", + "/%61bs/2401.12345", + "\\abs\\2401.12345", + "/abs;v=1/2401.12345;jsessionid=1", + "/ABS/2401.12345/", + ], + ) + def test_path_normalization_defeats_traversal_variants(self, path): + """Guards the traversal bypass found in review of the denylist mode (FrontierPhysics#365).""" + policy = proxy_mod.Policy(["https://arxiv.org/abs/2401.12345"], []) + assert policy.url_rule("arxiv.org", path) == "url:arxiv.org/abs/2401.12345" + + def test_path_normalization_keeps_siblings_open(self): + policy = proxy_mod.Policy(["https://arxiv.org/abs/2401.12345"], []) + for path in ( + "/abs/2401.1234", + "/abs/../pdf/2401.12345", + "/abs/2401.12345/../2401.99999", + ): + assert policy.url_rule("arxiv.org", path) is None + + def test_root_prefix_blocks_whole_host_and_trailing_slash_is_dropped(self): + assert ( + proxy_mod.Policy(["https://example.org"], []).url_rule( + "example.org", "/any" + ) + is not None + ) + policy = proxy_mod.Policy(["https://example.org/blocked/"], []) + assert policy.url_rule("example.org", "/blocked/x") is not None + assert policy.url_rule("example.org", "/blocked") is not None + assert policy.url_rule("example.org", "/block") is None + + @pytest.mark.parametrize( + "host", + [ + "1-2-3-4.sslip.io", + "1.2.3.4.nip.io", + "c0a80101.sslip.io", + "app.10.0.0.1.xip.io", + "paper.localtest.me", + ], + ) + def test_wildcard_dns_names_count_as_addresses(self, host): + """Guards the wildcard-DNS route around the address rule (FrontierPhysics#365).""" + assert proxy_mod.Policy([], []).host_rule(host) == "ip-literal" + + @pytest.mark.parametrize( + "address", + [ + "127.0.0.1", + "10.0.0.5", + "172.17.0.1", + "192.168.1.2", + "169.254.169.254", + "::1", + "fd00::1", + "100.64.0.1", + "0.0.0.0", + ], + ) + def test_non_global_upstreams_are_refused(self, address): + assert not proxy_mod._upstream_allowed(address) + + def test_global_upstreams_are_allowed(self): + assert proxy_mod._upstream_allowed("93.184.216.34") + assert proxy_mod._upstream_allowed("2606:2800:220:1:248:1893:25c8:1946") + + def test_connect_upstream_refuses_when_any_answer_is_private(self, monkeypatch): + monkeypatch.setattr( + proxy_mod, "_resolve", lambda host, port: ["93.184.216.34", "10.0.0.5"] + ) + with pytest.raises(proxy_mod._PrivateDestination): + proxy_mod._connect_upstream("rebind.test", 443) + + def test_body_prefix_drops_pipelined_requests(self): + rest = b"abc" + b"GET /blocked HTTP/1.1\r\nHost: x\r\n\r\n" + assert proxy_mod._body_prefix([("Content-Length", "3")], rest) == b"abc" + assert proxy_mod._body_prefix([], rest) == b"" + assert proxy_mod._body_prefix([("Transfer-Encoding", "chunked")], rest) == rest + + def test_host_rule_covers_subdomains_but_not_suffix_lookalikes(self): + policy = proxy_mod.Policy([], ["example.org"]) + assert policy.host_rule("example.org") == "host:example.org" + assert policy.host_rule("a.b.example.org") == "host:example.org" + assert policy.host_rule("evil-example.org") is None + assert policy.host_rule("example.org.evil.com") is None + + @pytest.mark.parametrize( + "host", + [ + "93.184.216.34", + "2606:2800:220:1:248:1893:25c8:1946", + "3232235777", + "0xc0a80101", + "0300.0250.1.1", + "127.1", + "0x7f.1", + "192.168.1.1.", + ], + ) + def test_addresses_in_every_resolver_notation_are_refused(self, host): + """Guards the non-canonical IPv4 bypass found in review (FrontierPhysics#365).""" + assert proxy_mod.Policy([], []).host_rule(host) == "ip-literal" + + def test_names_with_numeric_labels_are_still_names(self): + policy = proxy_mod.Policy([], []) + assert policy.host_rule("1e100.net") is None + assert policy.host_rule("3.example.org") is None + + def test_blocked_hosts_keep_www_exact(self): + policy = proxy_mod.Policy([], ["www.example.org"]) + assert policy.host_rule("www.example.org") == "host:www.example.org" + assert policy.host_rule("a.www.example.org") == "host:www.example.org" + assert policy.host_rule("example.org") is None + assert policy.host_rule("docs.example.org") is None + + def test_only_url_hosts_are_inspected(self): + policy = proxy_mod.Policy(["https://arxiv.org/abs/1"], ["alphaxiv.org"]) + assert policy.inspect("www.arxiv.org") + assert not policy.inspect("alphaxiv.org") + assert not policy.inspect("example.com") + + def test_entry_without_host_is_rejected(self): + with pytest.raises(ValueError, match="no host"): + proxy_mod.Policy(["https:///abs"], []) + + +def _free_port() -> int: + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] + + +class _Upstream(http.server.BaseHTTPRequestHandler): + def do_GET(self): + body = f"hello {self.path}".encode() + self.send_response(200) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + pass + + +@pytest.fixture +def stack(tmp_path: Path, monkeypatch): + """Proxy plus TLS and plain upstreams; test hostnames resolve to the local servers.""" + upstream_ca = certificate_material(("paper.test", "other.test")) + proxy_ca = certificate_material(("paper.test",)) + (tmp_path / "upstream-ca.crt").write_bytes(upstream_ca["ca.crt"]) + (tmp_path / "proxy-ca.crt").write_bytes(proxy_ca["ca.crt"]) + (tmp_path / "client-ca.crt").write_bytes(upstream_ca["ca.crt"] + proxy_ca["ca.crt"]) + certs = tmp_path / "certs" + certs.mkdir() + (certs / "paper.test.pem").write_bytes(proxy_ca["paper.test.pem"]) + + def tls_server(host: str) -> http.server.ThreadingHTTPServer: + pem = tmp_path / f"upstream-{host}.pem" + pem.write_bytes(upstream_ca[f"{host}.pem"]) + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), _Upstream) + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ctx.load_cert_chain(str(pem)) + server.socket = ctx.wrap_socket(server.socket, server_side=True) + return server + + tls = tls_server("paper.test") + tls_other = tls_server("other.test") + plain = http.server.ThreadingHTTPServer(("127.0.0.1", 0), _Upstream) + ports = { + "paper.test": tls.server_address[1], + "other.test": tls_other.server_address[1], + "plain.test": plain.server_address[1], + } + + def fake_connect_upstream(host, port): + if host == "internal.test": + raise proxy_mod._PrivateDestination(host) + return socket.create_connection(("127.0.0.1", ports[host]), timeout=10) + + monkeypatch.setattr(proxy_mod, "_connect_upstream", fake_connect_upstream) + log = tmp_path / "blocked.jsonl" + server = proxy_mod.serve( + _free_port(), + proxy_mod.Policy(["https://paper.test/abs/2401.12345"], ["mirror.test"]), + proxy_mod.CertStore(str(certs)), + proxy_mod.Log(str(log)), + upstream_ca=str(tmp_path / "upstream-ca.crt"), + ) + threads = [ + threading.Thread(target=s.serve_forever, daemon=True) + for s in (tls, tls_other, plain, server) + ] + for t in threads: + t.start() + proxy_url = f"http://127.0.0.1:{server.server_address[1]}" + client_ctx = ssl.create_default_context(cafile=str(tmp_path / "client-ca.crt")) + opener = urllib.request.build_opener( + urllib.request.ProxyHandler({"http": proxy_url, "https": proxy_url}), + urllib.request.HTTPSHandler(context=client_ctx), + ) + yield SimpleNamespace(opener=opener, proxy_url=proxy_url, log=log) + for s in (server, tls, tls_other, plain): + s.shutdown() + s.server_close() + + +def _status(opener, url: str) -> tuple[int, str]: + try: + with opener.open(url, timeout=10) as resp: + return resp.status, resp.read().decode() + except urllib.error.HTTPError as exc: + return exc.code, exc.read().decode() + + +class TestProxy: + def test_blocked_url_is_refused_and_logged(self, stack): + code, body = _status(stack.opener, "https://paper.test/abs/2401.12345v3") + assert code == 403 + assert "Blocked by the task network policy" in body + entry = json.loads(stack.log.read_text().splitlines()[-1]) + assert entry["action"] == "blocked" + assert entry["url"] == "https://paper.test/abs/2401.12345v3" + assert entry["rule"] == "url:paper.test/abs/2401.12345" + + def test_sibling_path_on_inspected_host_is_served(self, stack): + code, body = _status(stack.opener, "https://paper.test/abs/1706.03762") + assert (code, body) == (200, "hello /abs/1706.03762") + assert not stack.log.exists() + + def test_other_host_is_tunnelled_end_to_end(self, stack): + code, body = _status(stack.opener, "https://other.test/anything") + assert (code, body) == (200, "hello /anything") + + def test_plain_http_is_forwarded(self, stack): + code, body = _status(stack.opener, "http://plain.test/x?y=1") + assert (code, body) == (200, "hello /x?y=1") + + def test_query_with_scheme_is_not_mistaken_for_absolute_form(self, stack): + code, body = _status(stack.opener, "https://paper.test/search?q=http://x/y") + assert (code, body) == (200, "hello /search?q=http://x/y") + + def test_private_destination_is_refused_and_logged(self, stack): + """Guards the SSRF flag raised on PR #1113: the root proxy must not bridge to internal services.""" + with pytest.raises(OSError, match="403"): + stack.opener.open("https://internal.test/", timeout=10) + assert json.loads(stack.log.read_text())["rule"] == "private-address" + code, _body = _status(stack.opener, "http://internal.test/") + assert code == 403 + + def test_blocked_host_connect_is_refused(self, stack): + with pytest.raises(OSError, match="403"): + stack.opener.open("https://mirror.test/", timeout=10) + assert json.loads(stack.log.read_text())["rule"] == "host:mirror.test" + + def test_healthz(self, stack): + with urllib.request.urlopen(f"{stack.proxy_url}/healthz", timeout=5) as resp: + assert resp.status == 200 + + +class TestCertificateMaterial: + def test_leaf_is_signed_by_ca_and_covers_www(self, tmp_path: Path): + from cryptography import x509 + + material = certificate_material(("paper.test",)) + assert set(material) == {"ca.crt", "paper.test.pem"} + ca = x509.load_pem_x509_certificate(material["ca.crt"]) + leaf = x509.load_pem_x509_certificate(material["paper.test.pem"]) + assert leaf.issuer == ca.subject + san = leaf.extensions.get_extension_for_class(x509.SubjectAlternativeName).value + assert set(san.get_values_for_type(x509.DNSName)) == { + "paper.test", + "www.paper.test", + } + assert b"PRIVATE KEY" in material["paper.test.pem"] + assert b"PRIVATE KEY" not in material["ca.crt"] + + +class TestAgentEnv: + def test_env_routes_through_loopback_proxy_and_trusts_ca(self): + env = denylist_agent_env({"KEEP": "1"}) + assert env["KEEP"] == "1" + assert env[EGRESS_DENYLIST_ENV] == "1" + assert ( + env["HTTPS_PROXY"] + == env["https_proxy"] + == f"http://127.0.0.1:{EGRESS_PORT}" + ) + assert "localhost" in env["NO_PROXY"] and "127.0.0.1" in env["no_proxy"] + assert env["SSL_CERT_FILE"] == env["REQUESTS_CA_BUNDLE"] == CA_BUNDLE_PATH + assert env["NODE_EXTRA_CA_CERTS"] == CA_CERT_PATH + + def test_input_is_not_mutated(self): + original = {"A": "1"} + denylist_agent_env(original) + assert original == {"A": "1"} + + def test_denylist_for_config(self): + assert egress_denylist_for(SimpleNamespace(network_mode="public")) is None + found = egress_denylist_for( + SimpleNamespace( + network_mode="denylist", + blocked_urls=["https://a.test/x"], + blocked_hosts=None, + ) + ) + assert found == EgressDenylist(("https://a.test/x",), ()) + assert found.inspect_hosts == ("a.test",) + + +class TestShellCommands: + @pytest.mark.parametrize("shell", ["bash", "sh"]) + def test_commands_parse(self, shell): + for cmd in (_setup_cmd(), _health_cmd()): + subprocess.run([shell, "-n", "-c", cmd], check=True) + + def test_setup_replaces_a_running_proxy_and_keeps_the_log(self, tmp_path: Path): + """Guards the restart path found in review: a stale or live pid file must not abort setup, and the block log must survive.""" + runtime, ca, fake_bin = tmp_path / "rt", tmp_path / "ca", tmp_path / "bin" + (runtime / "certs").mkdir(parents=True) + fake_bin.mkdir() + (runtime / "ca.crt").write_text("cert\n") + python = fake_bin / "python3" + python.write_text("#!/bin/sh\nexec sleep 60\n") + python.chmod(0o755) + (runtime / "proxy.pid").write_text("999999\n") + (runtime / "blocked.jsonl").write_text('{"action": "blocked"}\n') + env = {"PATH": f"{fake_bin}:/usr/bin:/bin"} + cmd = _setup_cmd(runtime_dir=str(runtime), ca_dir=str(ca)) + for _ in range(2): + result = subprocess.run( + ["/bin/sh", "-c", cmd], + env=env, + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == 0, result.stderr + pid = int((runtime / "proxy.pid").read_text()) + subprocess.run(["kill", str(pid)], check=False) + assert (runtime / "blocked.jsonl").read_text() == '{"action": "blocked"}\n' + assert (ca / "ca-bundle.crt").read_text().endswith("cert\n") + + def test_setup_fails_closed_without_python(self, tmp_path: Path): + empty_bin = tmp_path / "bin" + empty_bin.mkdir() + result = subprocess.run( + ["/bin/sh", "-c", _setup_cmd()], + env={"PATH": str(empty_bin)}, + capture_output=True, + text=True, + timeout=15, + ) + assert result.returncode == 87 + assert "needs python3" in result.stderr + assert not (tmp_path / "proxy.pid").exists() + + +def _fake_env(health_rc: int = 0): + env = MagicMock() + + async def exec_(command, **kwargs): + rc = health_rc if "healthz" in command else 0 + return MagicMock(return_code=rc, stdout="", stderr="") + + env.exec = AsyncMock(side_effect=exec_) + env.exec_transient = env.exec + env.upload_file = AsyncMock() + env.download_file = AsyncMock() + return env + + +class TestStartStop: + async def test_start_requires_sandbox_user(self): + with pytest.raises(RuntimeError, match="sandbox_user"): + await start_egress_denylist( + _fake_env(), None, EgressDenylist(("https://a.test/x",), ()) + ) + + async def test_start_uploads_policy_certs_and_script_then_probes_health(self): + env = _fake_env() + await start_egress_denylist( + env, "agent", EgressDenylist(("https://a.test/x",), ("b.test",)) + ) + uploaded = {call.args[1]: call for call in env.upload_file.await_args_list} + assert set(uploaded) == { + "/opt/benchflow-egress/policy.json", + "/opt/benchflow-egress/proxy.py", + "/opt/benchflow-egress/ca.crt", + "/opt/benchflow-egress/certs/a.test.pem", + } + assert all(call.kwargs == {"mode": "600"} for call in uploaded.values()) + commands = [call.args[0] for call in env.exec.await_args_list] + assert all(call.kwargs["user"] == "root" for call in env.exec.await_args_list) + assert commands[0].startswith("mkdir -p /opt/benchflow-egress/certs") + assert commands[1] == _setup_cmd() + assert "healthz" in commands[2] + + async def test_start_raises_with_stderr_when_unhealthy(self): + env = _fake_env(health_rc=1) + with pytest.raises(RuntimeError, match="did not become healthy"): + await start_egress_denylist( + env, "agent", EgressDenylist((), ("b.test",)), timeout_sec=1 + ) + + async def test_stop_downloads_log_and_kills_proxy(self, tmp_path: Path): + env = _fake_env() + await stop_egress_denylist(env, tmp_path) + target = tmp_path / "trajectory" / TRAJECTORY_LOG_NAME + env.download_file.assert_awaited_once_with( + "/opt/benchflow-egress/blocked.jsonl", target + ) + (kill_cmd,) = [c.args[0] for c in env.exec.await_args_list] + assert ( + "kill -TERM" in kill_cmd + and "rm -rf /opt/benchflow-egress /etc/benchflow-egress" in kill_cmd + ) + + async def test_stop_falls_back_to_cat_and_never_raises(self, tmp_path: Path): + env = _fake_env() + env.download_file = AsyncMock(side_effect=RuntimeError("no cp")) + env.exec = AsyncMock( + side_effect=[ + MagicMock(return_code=0, stdout='{"action": "blocked"}\n'), + RuntimeError("gone"), + ] + ) + await stop_egress_denylist(env, tmp_path) + assert ( + tmp_path / "trajectory" / TRAJECTORY_LOG_NAME + ).read_text() == '{"action": "blocked"}\n' + + +class TestFirewallGate: + async def test_denylist_marker_arms_firewall_without_provider_url(self): + env = MagicMock() + env.exec = AsyncMock(return_value=MagicMock(return_code=0)) + await enforce_agent_egress_firewall( + env, + "agent", + { + EGRESS_DENYLIST_ENV: "1", + "HTTPS_PROXY": f"http://127.0.0.1:{EGRESS_PORT}", + }, + ) + env.exec.assert_awaited_once() + assert "iptables" in env.exec.await_args.args[0] + assert env.exec.await_args.kwargs == {"user": "root", "timeout_sec": 120} + + async def test_denylist_marker_accepts_loopback_provider_url(self): + env = MagicMock() + env.exec = AsyncMock(return_value=MagicMock(return_code=0)) + await enforce_agent_egress_firewall( + env, + "agent", + { + EGRESS_DENYLIST_ENV: "1", + "HTTPS_PROXY": "http://127.0.0.1:18628", + "LLM_BASE_URL": "http://127.0.0.1:4000", + }, + ) + env.exec.assert_awaited_once() + + async def test_denylist_marker_rejects_missing_proxy(self): + env = MagicMock() + env.exec = AsyncMock() + with pytest.raises(RuntimeError, match="HTTPS_PROXY"): + await enforce_agent_egress_firewall( + env, "agent", {EGRESS_DENYLIST_ENV: "1"} + ) + env.exec.assert_not_called() + + async def test_denylist_marker_rejects_remote_provider_url(self): + env = MagicMock() + env.exec = AsyncMock() + with pytest.raises(RuntimeError, match="loopback provider base URL"): + await enforce_agent_egress_firewall( + env, + "agent", + { + EGRESS_DENYLIST_ENV: "1", + "HTTPS_PROXY": "http://127.0.0.1:18628", + "LLM_BASE_URL": "http://172.17.0.1:4000", + }, + ) + + async def test_denylist_marker_requires_sandbox_user(self): + env = MagicMock() + env.exec = AsyncMock() + with pytest.raises(RuntimeError, match="sandbox_user"): + await enforce_agent_egress_firewall( + env, + None, + {EGRESS_DENYLIST_ENV: "1", "HTTPS_PROXY": "http://127.0.0.1:1"}, + ) + + async def test_no_web_without_sandbox_user_still_skips(self): + env = MagicMock() + env.exec = AsyncMock() + await enforce_agent_egress_firewall( + env, None, {"BENCHFLOW_DISALLOW_WEB_TOOLS": "1"} + ) + env.exec.assert_not_called() diff --git a/tests/test_eval_source_provenance.py b/tests/test_eval_source_provenance.py index eec2a957e..8c3bc122c 100644 --- a/tests/test_eval_source_provenance.py +++ b/tests/test_eval_source_provenance.py @@ -644,7 +644,9 @@ def resolve_locked_paths(self, _sandbox_user, _locked_paths): def resolve_agent_env(self, _agent, _model, agent_env): return agent_env or {} - def agent_launch(self, agent, *, disallow_web_tools): + def agent_launch( + self, agent, *, disallow_web_tools, disallow_hosted_search=False + ): return agent def stage_dockerfile_deps(self, *_args, **_kwargs): diff --git a/tests/test_hosted_search_policy.py b/tests/test_hosted_search_policy.py new file mode 100644 index 000000000..aea15f24c --- /dev/null +++ b/tests/test_hosted_search_policy.py @@ -0,0 +1,769 @@ +"""Hosted-search switches and rollout wiring for network_mode='denylist'. + +Guards the denylist egress mode, benchflow-ai/FrontierPhysics#365: a task +keeps internet access, listed URLs and hosts are unreachable through the +loopback egress proxy, and each harness's hosted (provider-side) search tool +is switched off because the proxy cannot see those requests. +""" + +from __future__ import annotations + +import dataclasses +import json +import subprocess +from datetime import datetime +from pathlib import Path +from types import SimpleNamespace +from typing import Any +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from benchflow.agents.install import apply_web_tool_policy +from benchflow.agents.manifest import _SHIM_ONLY +from benchflow.agents.registry import ( + AGENT_INSTALLERS, + AGENT_LAUNCH, + AGENTS, + AgentConfig, + register_agent, +) +from benchflow.rollout import Role, Rollout, RolloutConfig, Scene +from benchflow.rollout_planes import DefaultRolloutPlanes +from benchflow.sandbox.egress_denylist import EGRESS_DENYLIST_ENV, EgressDenylist +from benchflow.task import RolloutPaths + +_USAGE_UNAVAILABLE = { + "n_input_tokens": None, + "n_output_tokens": None, + "n_cache_read_tokens": None, + "n_cache_creation_tokens": None, + "total_tokens": None, + "cost_usd": None, + "usage_source": "unavailable", + "price_source": None, +} +_BLOCKED_URL = "https://arxiv.org/abs/2401.00001" +_BLOCKED_HOST = "github.com" +_DENYLIST = EgressDenylist((_BLOCKED_URL,), (_BLOCKED_HOST,)) + + +# Registry: per-harness hosted-search switches + + +def _run_hosted_search_cmd(agent_name: str, home: Path) -> None: + """Execute an agent's disallow_hosted_search_setup_cmd against a temp home.""" + cmd = AGENTS[agent_name].disallow_hosted_search_setup_cmd + assert cmd, f"{agent_name} has no disallow_hosted_search_setup_cmd" + result = subprocess.run( + ["bash", "-c", cmd], + env={"BENCHFLOW_AGENT_HOME": str(home), "PATH": "/usr/bin:/bin"}, + capture_output=True, + text=True, + timeout=15, + ) + assert result.returncode == 0, f"{agent_name} setup_cmd failed: {result.stderr}" + + +def _run_no_web_cmd(agent_name: str, home: Path) -> None: + result = subprocess.run( + ["bash", "-c", AGENTS[agent_name].disallow_web_tools_setup_cmd], + env={"BENCHFLOW_AGENT_HOME": str(home), "PATH": "/usr/bin:/bin"}, + capture_output=True, + text=True, + timeout=15, + ) + assert result.returncode == 0, result.stderr + + +def test_claude_hosted_search_cmd_denies_websearch_but_keeps_webfetch(tmp_path): + """WebFetch is a local fetch that goes through the proxy, so it stays on.""" + _run_hosted_search_cmd("claude-agent-acp", tmp_path) + settings = json.loads((tmp_path / ".claude" / "settings.json").read_text()) + + deny = settings["permissions"]["deny"] + assert "WebSearch" in deny + assert "WebFetch" not in deny + + +def test_claude_hosted_search_cmd_is_idempotent(tmp_path): + _run_hosted_search_cmd("claude-agent-acp", tmp_path) + _run_hosted_search_cmd("claude-agent-acp", tmp_path) + settings = json.loads((tmp_path / ".claude" / "settings.json").read_text()) + + assert settings["permissions"]["deny"].count("WebSearch") == 1 + + +def test_gemini_hosted_search_cmd_excludes_search_and_fetch(tmp_path): + """Gemini's web_fetch uses the hosted urlContext path first, so both go.""" + _run_hosted_search_cmd("gemini", tmp_path) + settings = json.loads((tmp_path / ".gemini" / "settings.json").read_text()) + + excluded = settings["tools"]["exclude"] + assert "google_web_search" in excluded + assert "web_fetch" in excluded + + +@pytest.mark.parametrize( + ("agent_name", "config_path"), + [ + ("opencode", ".config/opencode/opencode.json"), + ("mimo", ".config/mimocode/mimocode.json"), + ], +) +def test_opencode_family_hosted_search_cmd_leaves_webfetch_on( + agent_name, config_path, tmp_path +): + _run_hosted_search_cmd(agent_name, tmp_path) + tools = json.loads((tmp_path / config_path).read_text())["tools"] + + assert tools["websearch"] is False + assert "webfetch" not in tools + + +def test_opencode_hosted_search_cmd_merges_with_no_web_settings(tmp_path): + """Both policies write the same config file without clobbering each other.""" + _run_no_web_cmd("opencode", tmp_path) + _run_hosted_search_cmd("opencode", tmp_path) + tools = json.loads( + (tmp_path / ".config" / "opencode" / "opencode.json").read_text() + )["tools"] + + assert tools == {"webfetch": False, "websearch": False} + + +def test_agents_without_hosted_search_switch_keep_defaults(): + for name in ("pi-acp", "openclaw", "harvey-lab-harness", "deepagents", "openhands"): + assert AGENTS[name].disallow_hosted_search_setup_cmd == "", name + assert AGENTS[name].disallow_hosted_search_launch_suffix == "", name + + +def test_hosted_search_fields_are_shim_only(): + """A data-only manifest cannot carry the switches; core owns them.""" + assert { + "disallow_hosted_search_setup_cmd", + "disallow_hosted_search_launch_suffix", + } <= _SHIM_ONLY + + +# Planes: launch suffix selection + + +def test_codex_launch_suffix_disables_web_search_for_hosted_search(): + planes = DefaultRolloutPlanes() + base = AGENT_LAUNCH["codex-acp"] + + assert ( + planes.agent_launch( + "codex-acp", disallow_web_tools=False, disallow_hosted_search=True + ) + == f"{base} -c tools.web_search=false" + ) + assert planes.agent_launch("codex-acp", disallow_web_tools=False) == base + + +def test_launch_without_suffix_is_unchanged_for_hosted_search(): + planes = DefaultRolloutPlanes() + + assert ( + planes.agent_launch( + "claude-agent-acp", disallow_web_tools=False, disallow_hosted_search=True + ) + == AGENT_LAUNCH["claude-agent-acp"] + ) + assert ( + planes.agent_launch( + "not-a-real-agent", disallow_web_tools=False, disallow_hosted_search=True + ) + == "not-a-real-agent" + ) + + +def test_no_web_launch_suffix_wins_over_hosted_search_suffix(): + register_agent( + "hosted-search-probe", + "true", + "probe --acp", + disallow_web_tools_launch_suffix=" --no-web", + disallow_hosted_search_launch_suffix=" --no-search", + ) + try: + planes = DefaultRolloutPlanes() + assert ( + planes.agent_launch( + "hosted-search-probe", + disallow_web_tools=True, + disallow_hosted_search=True, + ) + == "probe --acp --no-web" + ) + assert ( + planes.agent_launch( + "hosted-search-probe", + disallow_web_tools=False, + disallow_hosted_search=True, + ) + == "probe --acp --no-search" + ) + assert ( + planes.agent_launch("hosted-search-probe", disallow_web_tools=False) + == "probe --acp" + ) + finally: + AGENTS.pop("hosted-search-probe", None) + AGENT_INSTALLERS.pop("hosted-search-probe", None) + AGENT_LAUNCH.pop("hosted-search-probe", None) + + +# Install: apply_web_tool_policy chooses the hosted-search command + + +def _shell_env() -> SimpleNamespace: + calls: list[str] = [] + + async def exec_cmd(cmd, *, timeout_sec=None, **kwargs): + calls.append(cmd) + result = subprocess.run( + cmd, shell=True, text=True, capture_output=True, timeout=timeout_sec + ) + return SimpleNamespace( + return_code=result.returncode, stdout=result.stdout, stderr=result.stderr + ) + + return SimpleNamespace(exec=exec_cmd, calls=calls) + + +def _probe_agent_cfg(**overrides: Any) -> AgentConfig: + fields = { + "name": "probe", + "install_cmd": "true", + "launch_cmd": "true", + "disallow_web_tools_setup_cmd": ( + 'mkdir -p "$BENCHFLOW_AGENT_HOME" && ' + 'printf no-web > "$BENCHFLOW_AGENT_HOME/policy"' + ), + "disallow_hosted_search_setup_cmd": ( + 'mkdir -p "$BENCHFLOW_AGENT_HOME" && ' + 'printf hosted-search > "$BENCHFLOW_AGENT_HOME/policy"' + ), + } + fields.update(overrides) + return AgentConfig(**fields) + + +@pytest.mark.asyncio +async def test_apply_web_tool_policy_runs_hosted_search_cmd(tmp_path): + env = _shell_env() + home = tmp_path / "home" + + await apply_web_tool_policy( + env, + "probe", + _probe_agent_cfg(), + str(home), + disallow=False, + disallow_hosted_search=True, + ) + + assert (home / "policy").read_text() == "hosted-search" + assert len(env.calls) == 1 + assert env.calls[0].startswith("export BENCHFLOW_AGENT_HOME=") + + +@pytest.mark.asyncio +async def test_apply_web_tool_policy_prefers_no_web_cmd_when_both_requested(tmp_path): + env = _shell_env() + home = tmp_path / "home" + + await apply_web_tool_policy( + env, + "probe", + _probe_agent_cfg(), + str(home), + disallow=True, + disallow_hosted_search=True, + ) + + assert (home / "policy").read_text() == "no-web" + assert len(env.calls) == 1 + + +@pytest.mark.asyncio +async def test_apply_web_tool_policy_is_noop_without_either_flag(): + env = MagicMock() + env.exec = AsyncMock() + + await apply_web_tool_policy( + env, "probe", _probe_agent_cfg(), "/home/agent", disallow=False + ) + + env.exec.assert_not_called() + + +@pytest.mark.asyncio +async def test_apply_web_tool_policy_skips_agents_without_hosted_search_cmd(): + env = MagicMock() + env.exec = AsyncMock() + + await apply_web_tool_policy( + env, + "probe", + _probe_agent_cfg(disallow_hosted_search_setup_cmd=""), + "/home/agent", + disallow=False, + disallow_hosted_search=True, + ) + + env.exec.assert_not_called() + + +@pytest.mark.asyncio +async def test_apply_web_tool_policy_reports_hosted_search_failures(): + env = _shell_env() + + with pytest.raises(RuntimeError, match="Failed to apply hosted-search policy"): + await apply_web_tool_policy( + env, + "probe", + _probe_agent_cfg(disallow_hosted_search_setup_cmd="false"), + "/home/agent", + disallow=False, + disallow_hosted_search=True, + ) + + +@pytest.mark.asyncio +async def test_apply_web_tool_policy_repairs_ownership_for_hosted_search(): + env = MagicMock() + env.exec = AsyncMock(return_value=MagicMock(return_code=0, stdout="", stderr="")) + + await apply_web_tool_policy( + env, + "claude-agent-acp", + AGENTS["claude-agent-acp"], + "/home/agent", + disallow=False, + disallow_hosted_search=True, + ) + + cmd = env.exec.await_args.args[0] + assert "chown -R agent:agent /home/agent/.claude" in cmd + assert "WebSearch" in cmd + assert "WebFetch" not in cmd + + +# Rollout wiring + + +def _denylist_task(network_mode: str = "denylist") -> SimpleNamespace: + sandbox = SimpleNamespace( + network_mode=network_mode, + blocked_urls=[_BLOCKED_URL], + blocked_hosts=[_BLOCKED_HOST], + allow_internet=True, + skills_dir=None, + docker_image=None, + workdir=None, + ) + return SimpleNamespace( + name="denylist-task", + config=SimpleNamespace( + sandbox=sandbox, + agent=SimpleNamespace(prompt_prefix=None, timeout_sec=60), + ), + ) + + +def _fake_sandbox() -> MagicMock: + env = MagicMock() + env.exec = AsyncMock(return_value=MagicMock(return_code=0, stdout="", stderr="")) + env.stop = AsyncMock() + env.download_file = AsyncMock() + return env + + +def _fake_acp_connection() -> tuple[Any, Any, Any, str]: + """A connect_acp result whose client and session survive disconnect().""" + client = MagicMock() + client.close = AsyncMock() + client.session = None + session = MagicMock() + session.latest_usage_totals.return_value = None + return client, session, MagicMock(), "agent" + + +def _fake_planes(env: Any) -> MagicMock: + planes = MagicMock() + planes.extract_usage.return_value = dict(_USAGE_UNAVAILABLE) + planes.resolve_locked_paths.return_value = [] + planes.resolve_agent_env.side_effect = lambda _agent, _model, agent_env: dict( + agent_env or {} + ) + planes.agent_launch.side_effect = ( + lambda agent, *, disallow_web_tools, disallow_hosted_search=False: agent + ) + planes.create_environment.return_value = env + planes.ensure_litellm_runtime = AsyncMock( + side_effect=lambda **kwargs: (kwargs["agent_env"], None) + ) + planes.start_egress_denylist = AsyncMock() + planes.stop_egress_denylist = AsyncMock() + planes.stop_provider_runtime = AsyncMock() + planes.install_agent = AsyncMock(return_value=MagicMock()) + planes.write_credential_files = AsyncMock() + planes.upload_subscription_auth = AsyncMock() + planes.apply_web_tool_policy = AsyncMock() + planes.connect_acp = AsyncMock(side_effect=lambda **k: _fake_acp_connection()) + planes.connect_session_factory = AsyncMock( + side_effect=lambda **k: (None, _fake_acp_connection()[1], None, "agent") + ) + return planes + + +def _rollout( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + *, + task: Any, + planes: Any, + agent: str = "claude-agent-acp", + sandbox_user: str | None = "agent", +) -> Rollout: + task_dir = tmp_path / "task" + task_dir.mkdir(exist_ok=True) + (task_dir / "instruction.md").write_text("Solve it.\n") + rollout_dir = tmp_path / "rollout" + + def fake_init_rollout(task_path, job_name, rollout_name, jobs_dir): + for subdir in ("agent", "verifier", "artifacts", "trajectory"): + (rollout_dir / subdir).mkdir(parents=True, exist_ok=True) + return ( + task, + rollout_dir, + RolloutPaths(rollout_dir=rollout_dir), + datetime.now(), + "job", + "rollout", + ) + + monkeypatch.setattr("benchflow.rollout._init_rollout", fake_init_rollout) + cfg = RolloutConfig( + task_path=task_dir, + agent=agent, + model="test-model", + sandbox_user=sandbox_user, + planes=planes, + ) + return Rollout(cfg) + + +@pytest.mark.asyncio +async def test_setup_defers_proxy_env_until_connect(tmp_path, monkeypatch): + """Proxy and CA vars must not reach the LiteLLM launcher: the CA bundle does not exist yet.""" + env = _fake_sandbox() + planes = _fake_planes(env) + rollout = _rollout(tmp_path, monkeypatch, task=_denylist_task(), planes=planes) + + await rollout.setup() + + assert rollout._egress_denylist == _DENYLIST + assert rollout._disallow_hosted_search is True + assert rollout._disallow_web_tools is False + assert EGRESS_DENYLIST_ENV not in rollout._agent_env + assert "HTTPS_PROXY" not in rollout._agent_env + assert "BENCHFLOW_DISALLOW_WEB_TOOLS" not in rollout._agent_env + planes.agent_launch.assert_called_once_with( + "claude-agent-acp", disallow_web_tools=False, disallow_hosted_search=True + ) + + +@pytest.mark.parametrize( + ("network_mode", "agent"), + [("public", "claude-agent-acp"), ("denylist", "oracle")], + ids=["other-network-mode", "oracle"], +) +@pytest.mark.asyncio +async def test_setup_leaves_env_alone_outside_denylist_agent_runs( + tmp_path, monkeypatch, network_mode, agent +): + env = _fake_sandbox() + planes = _fake_planes(env) + rollout = _rollout( + tmp_path, + monkeypatch, + task=_denylist_task(network_mode), + planes=planes, + agent=agent, + ) + + await rollout.setup() + + assert rollout._egress_denylist is None + assert rollout._disallow_hosted_search is False + assert EGRESS_DENYLIST_ENV not in rollout._agent_env + assert "HTTPS_PROXY" not in rollout._agent_env + planes.agent_launch.assert_called_once_with( + agent, disallow_web_tools=False, disallow_hosted_search=False + ) + + +@pytest.mark.asyncio +async def test_setup_fails_closed_without_sandbox_user(tmp_path, monkeypatch): + env = _fake_sandbox() + planes = _fake_planes(env) + rollout = _rollout( + tmp_path, monkeypatch, task=_denylist_task(), planes=planes, sandbox_user=None + ) + + with pytest.raises( + ValueError, match="network_mode='denylist' requires a sandbox_user" + ): + await rollout.setup() + + planes.create_environment.assert_not_called() + + +@pytest.mark.asyncio +async def test_install_agent_requests_hosted_search_policy(tmp_path, monkeypatch): + env = _fake_sandbox() + planes = _fake_planes(env) + planes.setup_sandbox_user = AsyncMock(return_value="/workspace") + planes.snapshot_build_config = AsyncMock() + planes.seed_verifier_workspace = AsyncMock() + planes.deploy_skills = AsyncMock() + planes.lockdown_paths = AsyncMock() + rollout = _rollout(tmp_path, monkeypatch, task=_denylist_task(), planes=planes) + await rollout.setup() + env.exec.return_value = MagicMock(return_code=0, stdout="/workspace\n", stderr="") + + await rollout.install_agent() + + kwargs = planes.apply_web_tool_policy.await_args.kwargs + assert kwargs == {"disallow": False, "disallow_hosted_search": True} + + +@pytest.mark.asyncio +async def test_connect_starts_proxy_before_acp_and_cleanup_stops_it( + tmp_path, monkeypatch +): + env = _fake_sandbox() + planes = _fake_planes(env) + order: list[str] = [] + planes.start_egress_denylist.side_effect = lambda *a, **k: order.append("start") + planes.connect_acp.side_effect = lambda **k: ( + order.append("connect_acp"), + _fake_acp_connection(), + )[1] + planes.stop_egress_denylist.side_effect = lambda *a, **k: order.append("stop") + env.stop.side_effect = lambda **k: order.append("env-stop") + rollout = _rollout(tmp_path, monkeypatch, task=_denylist_task(), planes=planes) + await rollout.setup() + + await rollout.connect() + + litellm_kwargs = planes.ensure_litellm_runtime.await_args.kwargs + assert litellm_kwargs["force_sandbox_local"] is True + assert "SSL_CERT_FILE" not in litellm_kwargs["agent_env"] + assert "HTTPS_PROXY" not in litellm_kwargs["agent_env"] + planes.start_egress_denylist.assert_awaited_once_with(env, "agent", _DENYLIST) + acp_env = planes.connect_acp.await_args.kwargs["agent_env"] + assert acp_env[EGRESS_DENYLIST_ENV] == "1" + assert acp_env["HTTPS_PROXY"] == "http://127.0.0.1:18628" + assert acp_env["NO_PROXY"] == "127.0.0.1,localhost,::1" + + await rollout.cleanup() + + planes.stop_egress_denylist.assert_awaited_once_with(env, tmp_path / "rollout") + assert order == ["start", "connect_acp", "stop", "env-stop"] + + +@pytest.mark.asyncio +async def test_connect_restarts_proxy_on_every_reconnect(tmp_path, monkeypatch): + """A restored sandbox has no proxy running, so each connect starts it again.""" + env = _fake_sandbox() + planes = _fake_planes(env) + rollout = _rollout(tmp_path, monkeypatch, task=_denylist_task(), planes=planes) + await rollout.setup() + + await rollout.connect() + await rollout.disconnect() + await rollout.connect() + + assert planes.start_egress_denylist.await_count == 2 + assert planes.connect_acp.await_count == 2 + + +@pytest.mark.asyncio +async def test_connect_rejects_session_factory_agents(tmp_path, monkeypatch): + """The uid firewall only runs on the ACP path, so a session-factory agent fails closed.""" + register_agent( + "denylist-sf-probe", + "true", + "true", + protocol="session-factory", + session_factory="fake_mod:build_agent", + ) + try: + env = _fake_sandbox() + planes = _fake_planes(env) + rollout = _rollout( + tmp_path, + monkeypatch, + task=_denylist_task(), + planes=planes, + agent="denylist-sf-probe", + ) + await rollout.setup() + + with pytest.raises( + RuntimeError, match="network_mode='denylist' requires an ACP agent" + ): + await rollout.connect() + + planes.start_egress_denylist.assert_not_awaited() + planes.connect_session_factory.assert_not_awaited() + finally: + AGENTS.pop("denylist-sf-probe", None) + AGENT_INSTALLERS.pop("denylist-sf-probe", None) + AGENT_LAUNCH.pop("denylist-sf-probe", None) + + +@pytest.mark.asyncio +async def test_connect_as_applies_denylist_to_role_env(tmp_path): + """A role connected without setup() derives the denylist from the task.""" + role = Role(name="coder", agent="gemini", model="gemini/test") + cfg = RolloutConfig( + task_path=tmp_path / "task", + scenes=[ + Scene( + roles=[ + Role(name="primary", agent="claude-agent-acp", model="test-model"), + role, + ] + ) + ], + ) + env = _fake_sandbox() + planes = _fake_planes(env) + planes.install_agent.return_value = AGENTS["gemini"] + trial = Rollout.__new__(Rollout) + trial._config = cfg + trial._env = env + trial._rollout_dir = tmp_path + trial._timing = {} + trial._agent_cwd = "/app" + trial._phase = "idle" + trial._task = _denylist_task() + trial._planes = planes + + await trial.connect_as(role) + + planes.agent_launch.assert_called_once_with( + "gemini", disallow_web_tools=False, disallow_hosted_search=True + ) + assert ( + planes.ensure_litellm_runtime.await_args.kwargs["force_sandbox_local"] is True + ) + assert planes.apply_web_tool_policy.await_args.kwargs == { + "disallow": False, + "disallow_hosted_search": True, + } + planes.start_egress_denylist.assert_awaited_once_with(env, "agent", _DENYLIST) + acp_env = planes.connect_acp.await_args.kwargs["agent_env"] + assert acp_env[EGRESS_DENYLIST_ENV] == "1" + assert acp_env["HTTPS_PROXY"] == "http://127.0.0.1:18628" + assert "BENCHFLOW_DISALLOW_WEB_TOOLS" not in acp_env + + +@pytest.mark.asyncio +async def test_connect_as_applies_denylist_when_primary_is_oracle(tmp_path): + """Guards the oracle-primary gap found in review: a later real role still gets the denylist.""" + role = Role(name="coder", agent="gemini", model="gemini/test") + cfg = RolloutConfig( + task_path=tmp_path / "task", + agent="oracle", + scenes=[Scene(roles=[Role(name="primary", agent="oracle", model=None), role])], + ) + env = _fake_sandbox() + planes = _fake_planes(env) + planes.install_agent.return_value = AGENTS["gemini"] + trial = Rollout.__new__(Rollout) + trial._config = cfg + trial._env = env + trial._rollout_dir = tmp_path + trial._timing = {} + trial._agent_cwd = "/app" + trial._phase = "idle" + trial._task = _denylist_task() + trial._planes = planes + trial._egress_denylist = None + + await trial.connect_as(role) + + planes.start_egress_denylist.assert_awaited_once_with(env, "agent", _DENYLIST) + assert planes.connect_acp.await_args.kwargs["agent_env"][EGRESS_DENYLIST_ENV] == "1" + + +@pytest.mark.asyncio +async def test_no_web_policy_wins_over_denylist(tmp_path, monkeypatch): + """--self-gen-no-internet must not hand the agent internet through the egress proxy.""" + env = _fake_sandbox() + planes = _fake_planes(env) + rollout = _rollout(tmp_path, monkeypatch, task=_denylist_task(), planes=planes) + rollout._config = dataclasses.replace(rollout._config, self_gen_no_internet=True) + + await rollout.setup() + await rollout.connect() + + assert rollout._disallow_web_tools is True + assert rollout._egress_denylist is None + planes.start_egress_denylist.assert_not_awaited() + acp_env = planes.connect_acp.await_args.kwargs["agent_env"] + assert acp_env["BENCHFLOW_DISALLOW_WEB_TOOLS"] == "1" + assert "HTTPS_PROXY" not in acp_env + + +@pytest.mark.asyncio +async def test_task_runtime_rejects_denylist_tasks(monkeypatch): + """The bash primitive never arms the proxy or firewall, so it fails closed.""" + from benchflow.rollout.task_runtime import TaskRuntime + + fake = MagicMock() + fake.setup = AsyncMock() + fake.cleanup = AsyncMock() + fake._egress_denylist = _DENYLIST + monkeypatch.setattr( + "benchflow.rollout.Rollout.create", AsyncMock(return_value=fake) + ) + runtime = TaskRuntime.__new__(TaskRuntime) + runtime._started = False + runtime.config = MagicMock() + + with pytest.raises(RuntimeError, match="requires an ACP agent rollout"): + await runtime.start() + fake.cleanup.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_connect_as_skips_denylist_for_oracle_role(tmp_path): + role = Role(name="checker", agent="oracle", model=None) + cfg = RolloutConfig(task_path=tmp_path / "task", scenes=[Scene(roles=[role])]) + env = _fake_sandbox() + planes = _fake_planes(env) + trial = Rollout.__new__(Rollout) + trial._config = cfg + trial._env = env + trial._rollout_dir = tmp_path + trial._timing = {} + trial._agent_cwd = "/app" + trial._phase = "idle" + trial._task = _denylist_task() + trial._planes = planes + + await trial.connect_as(role) + + planes.agent_launch.assert_called_once_with( + "oracle", disallow_web_tools=False, disallow_hosted_search=False + ) + planes.start_egress_denylist.assert_not_awaited() + assert EGRESS_DENYLIST_ENV not in planes.connect_acp.await_args.kwargs["agent_env"] diff --git a/tests/test_internet_policy.py b/tests/test_internet_policy.py index f34e2daaf..1958e3c32 100644 --- a/tests/test_internet_policy.py +++ b/tests/test_internet_policy.py @@ -18,8 +18,10 @@ def _wire_fake_planes(trial: Rollout) -> MagicMock: planes = MagicMock() - planes.agent_launch.side_effect = lambda agent, *, disallow_web_tools: ( - f"{agent} --no-web" if disallow_web_tools else agent + planes.agent_launch.side_effect = ( + lambda agent, *, disallow_web_tools, disallow_hosted_search=False: ( + f"{agent} --no-web" if disallow_web_tools else agent + ) ) planes.resolve_agent_env.side_effect = lambda _agent, _model, env: env or {} planes.ensure_litellm_runtime = AsyncMock( diff --git a/tests/test_network_denylist_config.py b/tests/test_network_denylist_config.py new file mode 100644 index 000000000..cc3e04106 --- /dev/null +++ b/tests/test_network_denylist_config.py @@ -0,0 +1,305 @@ +"""Denylist egress mode: task config, capability gate, and stdlib mirrors. + +Guards the denylist egress mode, benchflow-ai/FrontierPhysics#365. +""" + +from __future__ import annotations + +import sys +from pathlib import Path +from textwrap import dedent + +import pytest + +from benchflow.sandbox.providers import ( + DENYLIST_UNSUPPORTED_PROVIDERS, + NO_NETWORK_UNSUPPORTED_PROVIDERS, + PROVIDERS_BY_NAME, +) +from benchflow.task import NetworkMode, TaskConfig, validate_task_runtime_support +from benchflow.task.config import ( + SandboxConfig, + VerifierConfig, + _validate_network_policy_fields, +) +from benchflow.task.document import TaskDocument + +_REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(_REPO_ROOT / "tests" / "integration")) +sys.path.insert(0, str(_REPO_ROOT / ".github" / "scripts")) + +import build_integration_review_pack as pack_mod # noqa: E402 +import rubric_checks # noqa: E402 + + +def test_denylist_is_a_network_mode() -> None: + """The enum accepts the new literal alongside the existing modes.""" + assert NetworkMode("denylist") is NetworkMode.DENYLIST + assert NetworkMode.DENYLIST.value == "denylist" + assert NetworkMode.DENYLIST != NetworkMode.ALLOWLIST + + +def test_denylist_fields_parse_from_task_md_frontmatter() -> None: + """task.md frontmatter carries blocked_urls/blocked_hosts on sandbox.""" + document = TaskDocument.from_text( + dedent( + """\ + --- + schema_version: "1.3" + sandbox: + network_mode: denylist + blocked_urls: [arxiv.org/abs/2401.00001] + blocked_hosts: [github.com] + --- + ## prompt + + Reproduce the paper. + """ + ) + ) + + cfg = document.config + assert cfg.agent.network_mode is None + assert cfg.sandbox.network_mode == NetworkMode.DENYLIST + assert cfg.sandbox.blocked_urls == ["https://arxiv.org/abs/2401.00001"] + assert cfg.sandbox.blocked_hosts == ["github.com"] + assert cfg.sandbox.allow_internet is True + + +def test_denylist_fields_parse_from_task_toml() -> None: + """Legacy task.toml spells the same fields under [environment].""" + cfg = TaskConfig.model_validate_toml( + dedent( + """\ + schema_version = "1.3" + + [environment] + network_mode = "denylist" + blocked_urls = ["https://github.com/org/repo"] + blocked_hosts = ["arxiv.org", "semanticscholar.org"] + """ + ) + ) + + assert cfg.sandbox.network_mode == NetworkMode.DENYLIST + assert cfg.sandbox.blocked_urls == ["https://github.com/org/repo"] + assert cfg.sandbox.blocked_hosts == ["arxiv.org", "semanticscholar.org"] + + +def test_blocked_urls_are_normalized() -> None: + """Scheme is added, host lowercased, query and fragment dropped, duplicates collapsed.""" + cfg = SandboxConfig( + network_mode="denylist", + blocked_urls=[ + "ArXiv.org/abs/2401.00001?context=physics", + "https://GitHub.com/Org/Repo#readme", + "http://example.com", + " example.com ", + "example.com", + "https://arxiv.org/abs/2401.00001", + ], + ) + + assert cfg.blocked_urls == [ + "https://arxiv.org/abs/2401.00001", + "https://github.com/Org/Repo", + "http://example.com", + "https://example.com", + ] + + +@pytest.mark.parametrize( + ("url", "message"), + [ + ("https://1.2.3.4/paper", "not an IP"), + ("https://*.example.com/x", "valid hostnames"), + ("https://user:secret@example.com/x", "userinfo"), + ("https://example.com:8443/x", "port"), + ("", "non-empty"), + ("ftp://example.com/x", "http or https"), + ("https:///x", "non-empty hostnames"), + ], +) +def test_blocked_urls_rejections(url: str, message: str) -> None: + """IP literals, wildcards, userinfo, ports, empties, and odd schemes are refused.""" + with pytest.raises(ValueError, match=message): + SandboxConfig(network_mode="denylist", blocked_urls=[url]) + + +def test_blocked_urls_drop_trailing_slash_and_blocked_hosts_refuse_addresses() -> None: + """Guards two review findings: a trailing slash must not exempt the page itself, and hosts cannot be IPs.""" + cfg = SandboxConfig(network_mode="denylist", blocked_urls=["github.com/org/repo/"]) + assert cfg.blocked_urls == ["https://github.com/org/repo"] + with pytest.raises(ValueError, match="not an IP"): + SandboxConfig(network_mode="denylist", blocked_hosts=["1.2.3.4"]) + + +def test_blocked_hosts_reuse_the_hostname_rules() -> None: + """blocked_hosts normalizes like allowed_hosts and refuses URL-shaped entries.""" + cfg = SandboxConfig( + network_mode="denylist", blocked_hosts=["ArXiv.org.", "github.com"] + ) + assert cfg.blocked_hosts == ["arxiv.org", "github.com"] + + with pytest.raises(ValueError, match="blocked_hosts entries must be hostnames"): + SandboxConfig(network_mode="denylist", blocked_hosts=["https://arxiv.org"]) + with pytest.raises(ValueError, match="blocked_hosts entries must be non-empty"): + SandboxConfig(network_mode="denylist", blocked_hosts=[" "]) + + +def test_allowed_hosts_error_strings_are_unchanged() -> None: + """Factoring the hostname rules must keep the allowed_hosts messages verbatim.""" + with pytest.raises(ValueError, match="allowed_hosts entries must be hostnames"): + SandboxConfig(network_mode="allowlist", allowed_hosts=["https://x.com"]) + with pytest.raises(ValueError, match="allowed_hosts entries must be non-empty"): + SandboxConfig(network_mode="allowlist", allowed_hosts=[""]) + with pytest.raises(ValueError, match="allowed_hosts entries must be valid"): + SandboxConfig(network_mode="allowlist", allowed_hosts=["bad_host"]) + + +@pytest.mark.parametrize("fields", [{}, {"blocked_urls": []}, {"blocked_hosts": []}]) +def test_denylist_without_lists_is_rejected(fields: dict[str, list[str]]) -> None: + """A denylist that blocks nothing is a misconfiguration, not a public sandbox.""" + with pytest.raises(ValueError, match="requires blocked_urls or blocked_hosts"): + SandboxConfig(network_mode="denylist", **fields) + + +@pytest.mark.parametrize("mode", ["public", "no-network", "allowlist"]) +def test_blocked_lists_outside_denylist_are_rejected(mode: str) -> None: + """blocked_urls/blocked_hosts carry no meaning under any other mode.""" + extra = {"allowed_hosts": ["x.com"]} if mode == "allowlist" else {} + with pytest.raises(ValueError, match="only valid for network_mode='denylist'"): + SandboxConfig(network_mode=mode, blocked_hosts=["arxiv.org"], **extra) + with pytest.raises(ValueError, match="only valid for network_mode='denylist'"): + SandboxConfig( + network_mode=mode, blocked_urls=["https://arxiv.org/abs/1"], **extra + ) + + +def test_allowlist_with_blocked_lists_is_rejected_before_allowlist_check() -> None: + """The allowlist messages stay verbatim and fire before the denylist ones.""" + with pytest.raises(ValueError, match="allowed_hosts must be non-empty"): + _validate_network_policy_fields(NetworkMode.ALLOWLIST, None, ["x"], None) + with pytest.raises(ValueError, match="allowed_hosts is only valid"): + _validate_network_policy_fields(NetworkMode.DENYLIST, ["x.com"], ["x"], None) + + +def test_verifier_rejects_denylist() -> None: + """The verifier has no blocked lists, so a denylist verifier cannot exist.""" + with pytest.raises(ValueError, match=r"verifier\.network_mode='denylist' is not"): + VerifierConfig(network_mode="denylist") + with pytest.raises(ValueError, match="blocked_hosts"): + VerifierConfig(network_mode="public", blocked_hosts=["arxiv.org"]) + + +def test_denylist_keeps_allow_internet_true() -> None: + """A denylist sandbox still has internet; only the listed targets are cut.""" + cfg = SandboxConfig(network_mode="denylist", blocked_hosts=["arxiv.org"]) + assert cfg.allow_internet is True + assert cfg.network_mode == NetworkMode.DENYLIST + + +def test_denylist_with_explicit_allow_internet_false_is_a_contradiction() -> None: + """Explicit allow_internet=False against denylist is the existing hard error.""" + with pytest.raises(ValueError, match="allow_internet=False contradicts"): + SandboxConfig( + network_mode="denylist", + blocked_hosts=["arxiv.org"], + allow_internet=False, + ) + + +def test_registry_declares_denylist_enforcement() -> None: + """Only docker and daytona run the loopback proxy behind the uid firewall.""" + assert PROVIDERS_BY_NAME["docker"].enforces_denylist is True + assert PROVIDERS_BY_NAME["daytona"].enforces_denylist is True + assert ( + frozenset({"modal", "apple-container", "agentcore"}) + == DENYLIST_UNSUPPORTED_PROVIDERS + ) + assert NO_NETWORK_UNSUPPORTED_PROVIDERS <= DENYLIST_UNSUPPORTED_PROVIDERS + + +@pytest.mark.parametrize("sandbox", ["docker", "daytona"]) +def test_capability_gate_accepts_denylist_on_enforcing_backends(sandbox: str) -> None: + """Docker and daytona launch a denylist task without a capability issue.""" + config = TaskConfig.model_validate( + { + "sandbox": { + "network_mode": "denylist", + "blocked_urls": ["https://arxiv.org/abs/2401.00001"], + }, + } + ) + + assert validate_task_runtime_support(config, sandbox=sandbox) == [] + + +@pytest.mark.parametrize("role", ["agent", "verifier"]) +def test_role_level_denylist_is_rejected(role: str) -> None: + """denylist is a sandbox policy; role overrides would be silently unenforced.""" + with pytest.raises( + ValueError, match=rf"{role}\.network_mode='denylist' is not supported" + ): + TaskConfig.model_validate({role: {"network_mode": "denylist"}}) + + +@pytest.mark.parametrize("sandbox", ["modal", "apple-container", "agentcore"]) +def test_capability_gate_refuses_denylist_elsewhere(sandbox: str) -> None: + """Backends without the proxy fail closed on the sandbox policy.""" + config = TaskConfig.model_validate( + { + "sandbox": {"network_mode": "denylist", "blocked_hosts": ["arxiv.org"]}, + } + ) + + issues = validate_task_runtime_support(config, sandbox=sandbox) + + assert [(issue.path, issue.reason) for issue in issues] == [ + ( + "sandbox.network_mode", + f"network_mode='denylist' is not enforced by {sandbox}", + ), + ] + + +def test_rubric_checks_v_network_denylist_outcomes() -> None: + """The stdlib mirror passes a populated denylist and fails an empty one.""" + assert "denylist" in rubric_checks._VALID_NETWORK_MODES + by_urls = rubric_checks.network_hardening( + {"network_mode": "denylist", "blocked_urls": ["https://arxiv.org/abs/1"]} + ) + by_hosts = rubric_checks.network_hardening( + {"network_mode": "denylist", "blocked_hosts": ["arxiv.org"]}, + verifier_or_sandbox_pr=True, + ) + empty = rubric_checks.network_hardening({"network_mode": "denylist"}) + stray = rubric_checks.network_hardening( + { + "network_mode": "denylist", + "blocked_hosts": ["arxiv.org"], + "allowed_hosts": ["x.com"], + } + ) + + assert by_urls[0] == "V-NETWORK" + assert by_urls[1] == "pass" + assert by_hosts[1] == "pass" + assert empty[1:] == ("fail", "denylist without blocked_urls/blocked_hosts") + assert stray[1] == "fail" + + +def test_review_pack_normalizes_denylist_cells() -> None: + """The review pack maps a denylist cell onto the V-NETWORK gate with its lists.""" + base = {"task": "citation-check", "agent": "openhands", "network_mode": "denylist"} + hardened = pack_mod.normalize_cell( + {**base, "id": "cit-deny", "blocked_hosts": ["arxiv.org"]} + ) + bare = pack_mod.normalize_cell({**base, "id": "cit-bare"}) + + assert pack_mod._cell_network_config(hardened) == "denylist" + md = pack_mod.hardening_summary_md( + slots=[], cells=[hardened, bare], verifier_or_sandbox_pr=True + ) + assert "cit-deny: V-NETWORK=pass" in md + assert "cit-bare: V-NETWORK=fail" in md diff --git a/tests/test_reexport.py b/tests/test_reexport.py index fe74da22d..1d127a3f3 100644 --- a/tests/test_reexport.py +++ b/tests/test_reexport.py @@ -80,6 +80,8 @@ def test_register_agent(): description="Test agent", disallow_web_tools_setup_cmd="true", disallow_web_tools_owned_paths=["$HOME/.test-agent"], + disallow_hosted_search_setup_cmd="false", + disallow_hosted_search_launch_suffix=" --no-search", ) assert "test-custom-agent" in AGENTS @@ -88,6 +90,8 @@ def test_register_agent(): assert cfg.requires_env == ["TEST_KEY"] assert cfg.disallow_web_tools_setup_cmd == "true" assert cfg.disallow_web_tools_owned_paths == ["$HOME/.test-agent"] + assert cfg.disallow_hosted_search_setup_cmd == "false" + assert cfg.disallow_hosted_search_launch_suffix == " --no-search" assert alias_model == "" finally: # register_agent writes to all three dicts; clean up all three to keep diff --git a/tests/test_registry_invariants.py b/tests/test_registry_invariants.py index efeb025be..962d764ee 100644 --- a/tests/test_registry_invariants.py +++ b/tests/test_registry_invariants.py @@ -107,6 +107,27 @@ def test_agent_collection_invariants(name, cfg): ) +@pytest.mark.parametrize("name,cfg", AGENTS.items(), ids=list(AGENTS.keys())) +def test_agent_hosted_search_switches_are_well_formed(name, cfg): + """Hosted-search switches target $BENCHFLOW_AGENT_HOME and append cleanly. + + Guards the denylist egress mode, benchflow-ai/FrontierPhysics#365: the + setup command runs with BENCHFLOW_AGENT_HOME exported, and the launch + suffix is concatenated onto launch_cmd without a separator. + """ + assert isinstance(cfg.disallow_hosted_search_setup_cmd, str) + assert isinstance(cfg.disallow_hosted_search_launch_suffix, str) + if cfg.disallow_hosted_search_setup_cmd: + assert "$BENCHFLOW_AGENT_HOME" in cfg.disallow_hosted_search_setup_cmd, ( + f"{name!r} disallow_hosted_search_setup_cmd must write under " + "$BENCHFLOW_AGENT_HOME" + ) + if cfg.disallow_hosted_search_launch_suffix: + assert cfg.disallow_hosted_search_launch_suffix[0].isspace(), ( + f"{name!r} disallow_hosted_search_launch_suffix must start with a space" + ) + + @pytest.mark.parametrize("name,cfg", AGENTS.items(), ids=list(AGENTS.keys())) def test_agent_install_cmd_targets_shared_paths(name, cfg): """Installed binaries must land in shared prefixes, not a root-only home. diff --git a/tests/test_rollout_planes_contract.py b/tests/test_rollout_planes_contract.py index 152c2400b..2e354caef 100644 --- a/tests/test_rollout_planes_contract.py +++ b/tests/test_rollout_planes_contract.py @@ -39,6 +39,8 @@ "link_skill_paths", "ensure_litellm_runtime", "stop_provider_runtime", + "start_egress_denylist", + "stop_egress_denylist", "extract_usage", "connect_acp", "execute_prompts", diff --git a/tests/test_task_runtime_primitive.py b/tests/test_task_runtime_primitive.py index 6b2658fc0..d6dbe3fcc 100644 --- a/tests/test_task_runtime_primitive.py +++ b/tests/test_task_runtime_primitive.py @@ -87,7 +87,13 @@ def install_docker_compat(self) -> None: def extract_usage(self, runtime: Any) -> dict[str, Any]: return {"usage_source": "unavailable"} - def agent_launch(self, agent: str, *, disallow_web_tools: bool) -> str: + def agent_launch( + self, + agent: str, + *, + disallow_web_tools: bool, + disallow_hosted_search: bool = False, + ) -> str: return agent def agent_config(self, agent: str) -> Any: diff --git a/tests/test_trial_litellm_runtime.py b/tests/test_trial_litellm_runtime.py index 183c0ba30..a9b60fa16 100644 --- a/tests/test_trial_litellm_runtime.py +++ b/tests/test_trial_litellm_runtime.py @@ -102,7 +102,9 @@ async def fake_connect_acp(**kwargs): return (AsyncMock(), AsyncMock(), AsyncMock(), "claude-agent-acp") rollout._planes = SimpleNamespace( - agent_launch=lambda agent, disallow_web_tools: agent, + agent_launch=lambda agent, disallow_web_tools, disallow_hosted_search=False: ( + agent + ), resolve_agent_env=lambda agent, model, env: dict(env or {}), ensure_litellm_runtime=fake_litellm, install_agent=AsyncMock(return_value=SimpleNamespace()), diff --git a/tests/test_verify.py b/tests/test_verify.py index d3821839b..8370631fa 100644 --- a/tests/test_verify.py +++ b/tests/test_verify.py @@ -977,7 +977,9 @@ def _patch_sdk_run(self, sdk, mock_env, extra_patches): } planes.resolve_locked_paths.return_value = [] planes.resolve_agent_env.side_effect = lambda _agent, _model, env: env or {} - planes.agent_launch.side_effect = lambda agent, *, disallow_web_tools: agent + planes.agent_launch.side_effect = ( + lambda agent, *, disallow_web_tools, disallow_hosted_search=False: agent + ) planes.create_environment.return_value = mock_env planes.stage_dockerfile_deps.return_value = None planes.inject_skills_into_dockerfile.return_value = None