feat(sandbox): enforce a URL denylist egress mode on docker and daytona

.github/scripts/build_integration_review_pack.py

@@ -760,6 +760,8 @@ def hardening_summary_md(
         cfg = {
             "network_mode": _cell_network_config(cell),
             "allowed_hosts": cell.raw.get("allowed_hosts"),
+            "blocked_urls": cell.raw.get("blocked_urls"),
+            "blocked_hosts": cell.raw.get("blocked_hosts"),
         }
         gate_id, status, detail = rubric_checks.network_hardening(
             cfg, verifier_or_sandbox_pr=verifier_or_sandbox_pr
@@ -794,10 +796,11 @@ def hardening_summary_md(
 def _cell_network_config(cell: Cell) -> str | None:
     """Map the cell's EXPECTED network_mode (Q3) to a NetworkMode literal.
 
-    The cell carries ``network_mode`` as ``default-off`` | ``allowlist`` (Q3:
-    derived from the task config, NOT passed to bench). Translate to the
-    benchflow ``NetworkMode`` literals the static checker understands, or use an
-    explicit per-cell ``network_mode`` override if the planner emitted one.
+    The cell carries ``network_mode`` as ``default-off`` | ``allowlist`` |
+    ``denylist`` (Q3: derived from the task config, NOT passed to bench).
+    Translate to the benchflow ``NetworkMode`` literals the static checker
+    understands, or use an explicit per-cell ``network_mode`` override if the
+    planner emitted one.
     """
     explicit = cell.raw.get("network_mode")
     mode = str(explicit) if explicit is not None else "default-off"
@@ -806,6 +809,8 @@ def _cell_network_config(cell: Cell) -> str | None:
         return "no-network"
     if norm == "allowlist":
         return "allowlist"
+    if norm == "denylist":
+        return "denylist"
     if norm == "public":
         return "public"
     return None

CHANGELOG.md

@@ -2,6 +2,14 @@
 
 ## [Unreleased]
 
+### Added
+- **`network_mode: denylist` blocks a list of URLs and hosts for the agent on
+  Docker and Daytona.** The task keeps internet access; `blocked_urls` and
+  `blocked_hosts` are enforced by a root-owned loopback proxy behind the
+  sandbox-user firewall, hosted search tools are switched off per harness,
+  and every refused request lands in `trajectory/egress_denylist.jsonl`.
+  Other backends refuse the mode at preflight. (#1113)
+
 ## 0.7.6 — 2026-09-04
 
 ### Added

docs/running-benchmarks.md

@@ -349,13 +349,21 @@ The **Harvey LAB harness** agent is special — it runs Harvey LAB's own agent l
 | Modal | `--sandbox modal` | Serverless, high concurrency (needs Modal auth) |
 | AgentCore | `--sandbox agentcore` | AWS-native isolated microVMs (needs AWS credentials) |
 
+`network_mode = "denylist"` tasks run on Docker, where BenchFlow adds
+`NET_ADMIN` to the agent container through its own compose overlay, and on
+direct Daytona sandboxes, where the mode was verified with `iptables`. The
+other backends refuse the mode at preflight. See
+[sandbox hardening](./sandbox-hardening.md#network-policy-denylist-egress)
+for what the mode does and does not guarantee.
+
 Apple Container requires Apple Container 1.1+ on Apple Silicon and runs the model
 proxy inside each VM. It supports public-network, single-container arm64 tasks and
 has no snapshot support. BenchFlow serializes Apple rollouts within each process
 and blocks new VMs when the live `data.kalloc.1024` headroom is unsafe. Avoid
 running concurrent BenchFlow processes, because the macOS allocation leak is
 system-wide. Use Docker, Daytona, or Modal for `network_mode = "no-network"`,
-multi-service, snapshot, or high-concurrency runs.
+multi-service, snapshot, or high-concurrency runs. Apple Container also
+refuses `network_mode = "denylist"`; use Docker or Daytona for those tasks.
 
 ### Amazon Bedrock AgentCore
 
@@ -411,7 +419,8 @@ Constraints: `linux/arm64` only, single container (no compose/multi-service
 tasks), no snapshot support, and `network_mode = "no-network"` is **not**
 enforceable — AgentCore's network mode is either `PUBLIC` or `VPC`, so
 BenchFlow refuses no-network tasks on this backend rather than running them
-unisolated. The model proxy runs inside the sandbox, as on Daytona and Modal.
+unisolated. `network_mode = "denylist"` is refused on this backend as well.
+The model proxy runs inside the sandbox, as on Daytona and Modal.
 Sessions default to a 15-minute idle timeout and an 8-hour lifetime; override
 with `BENCHFLOW_AGENTCORE_IDLE_TIMEOUT_SEC` / `BENCHFLOW_AGENTCORE_MAX_LIFETIME_SEC`
 if agent turns are long enough to risk reclamation mid-run.

docs/sandbox-hardening.md

@@ -41,6 +41,60 @@ Unknown keys in `[verifier.hardening]` are warned and ignored. String values for
 
 See [`progressive-disclosure.md`](./progressive-disclosure.md#per-task-hardening-opt-outs) for the qutebrowser case study (legitimate `conftest.py` for circular-import fix).
 
+## Network policy: denylist egress
+
+`network_mode: denylist` keeps the internet reachable and makes a list of URLs and hosts unreachable for the agent. The use case is a task built from a published paper: the agent may search and read freely, but the paper, its mirrors, and its code repository are off limits ([benchflow-ai/FrontierPhysics#365](https://github.com/benchflow-ai/FrontierPhysics/issues/365)).
+
+```yaml
+sandbox:
+  network_mode: denylist
+  blocked_urls:
+    - https://example.org/papers/lattice-qcd-2026
+    - github.com/example-org/lattice-qcd-code
+  blocked_hosts:
+    - mirror.example.net
+```
+
+See [task authoring](./task-authoring-task-md.md#network-policy) for the field rules. The proxy filters the agent only; oracle runs and the verifier are not filtered.
+
+### Mechanism
+
+1. **Loopback proxy.** Before the agent starts, benchflow uploads a stdlib Python proxy (`src/benchflow/sandbox/_egress_denylist_proxy.py`) and starts it as root on `127.0.0.1:18628`. A request that matches the denylist gets `403 Forbidden` with an `X-BenchFlow-Blocked: 1` header; everything else is tunneled to its destination.
+2. **Uid firewall.** The same `iptables` owner rule that backs the no-web mode lets the sandbox user reach loopback only. Every other outbound packet from that uid is rejected, so the proxy is the only way out. `iptables` is installed on first use (apt, dnf, or apk) when the image lacks it.
+3. **Proxy and CA environment.** The agent env gets `HTTP_PROXY`, `HTTPS_PROXY`, `NO_PROXY`, `SSL_CERT_FILE`, `REQUESTS_CA_BUNDLE`, `CURL_CA_BUNDLE`, `GIT_SSL_CAINFO`, `NODE_EXTRA_CA_CERTS`, and `NODE_USE_ENV_PROXY`, plus the `BENCHFLOW_EGRESS_DENYLIST=1` marker that arms the firewall. These are added after the sandbox-local LiteLLM gateway starts, so model traffic does not pass through the egress proxy.
+4. **Selective TLS interception.** Hosts named in `blocked_urls` need their paths inspected, so the proxy terminates TLS for those hosts with a leaf certificate signed by a per-rollout CA (`BenchFlow egress policy CA`). Certificates are minted on the host; the CA private key never enters the sandbox. Hosts in `blocked_hosts` are refused at `CONNECT` time, and every other host passes through as an opaque tunnel.
+5. **Hosted search off.** Provider-side search tools fetch pages from the model provider's servers, outside the sandbox, so the proxy cannot see them. Benchflow disables them per harness:
+
+   | Harness | Switched off | Still on |
+   |---|---|---|
+   | `claude-agent-acp` | `WebSearch` | `WebFetch` (fetches from inside the sandbox, through the proxy) |
+   | `codex-acp` | `tools.web_search` | |
+   | `gemini` | `google_web_search`, `web_fetch` (tries a hosted fetch first) | |
+   | `opencode`, `mimo` | `websearch` | `webfetch` |
+   | other harnesses | nothing | whatever hosted tools they ship |
+
+6. **Block log.** Each refused attempt is appended to a root-owned log that benchflow downloads to `trajectory/egress_denylist.jsonl` in the rollout directory at cleanup: one JSON object per line with `ts`, `action`, `method`, `url`, and `rule` (`host:<host>`, `url:<host><path>`, or `ip-literal`). For a refused `CONNECT`, `url` holds the `host:port` the client asked for.
+
+Matching ignores scheme, port, query string, and case, strips a leading `www.`, and compares a normalized path: percent-encoding is decoded (repeatedly), `.` and `..` segments are resolved, duplicate slashes and backslashes collapse, and `;` path parameters are dropped, so `/abs/../abs/2401.12345` and `/abs/%2e%2e/abs/2401.12345` match the same entry as `/abs/2401.12345`. A `blocked_urls` entry blocks every path under it; a `blocked_hosts` entry blocks the host and its subdomains. Requests to addresses are refused in every notation a resolver accepts (dotted, decimal, hex, octal) and through wildcard DNS names that embed an address (`1-2-3-4.sslip.io`), so a blocked host cannot be reached by its address. A name the agent controls that resolves to the blocked address is not detected; that is the inherent limit of a hostname denylist. Before connecting anywhere, the proxy resolves the destination and refuses names that resolve to loopback, private, link-local, or other non-global addresses (cloud metadata included), so a hostname the agent controls cannot turn the root proxy into a bridge to sandbox-internal or host services. The uid firewall stays for the rest of the sandbox life, as in the no-web mode: a later oracle role in the same sandbox, and a verifier configured with `verifier.user` equal to the sandbox user, run without egress.
+
+### Requirements
+
+- A non-root `sandbox_user`. Setup fails closed before any sandbox is created when it is missing.
+- `python3` (or `python`) on `PATH` in the task image. The proxy is a stdlib script and installs nothing.
+- An ACP agent. Session-factory agents raise at connect time because the uid firewall only runs in the ACP path.
+- `docker`: the agent container needs `NET_ADMIN` for `iptables`. Benchflow adds it through its own compose overlay (`src/benchflow/sandbox/_compose_files/docker-compose-net-admin.yaml`), so the task's `Dockerfile` and `docker-compose.yaml` need nothing extra.
+- `daytona`: verified on direct sandboxes with `iptables`.
+- `modal`, `apple-container`, and `agentcore` refuse the mode at preflight. `bench tasks check --sandbox <backend>` reports it, and the runtime raises before the sandbox is created.
+
+### Limits
+
+- **A blocklist hides pages, not knowledge.** The model may have read the paper in training. The denylist only stops it from re-reading the page during the run.
+- **Mirrors and citations stay reachable.** Only listed prefixes and hosts are blocked. Preprint mirrors, aggregator copies, cached versions, citing papers that restate the result, and the same repository under another host all stay reachable unless listed. `blocked_hosts` covers subdomains, not other domains. List every form of a page you know about (abstract, PDF, HTML, versioned).
+- **The block is visible.** A refused request gets a `403` that names the policy, and the agent can tell intercepted hosts from the certificate issuer (`BenchFlow egress policy CA`). Do not expect the agent to be unaware that a page is off limits.
+- **Tools that ignore proxy variables fail closed.** A client that does not honor `HTTP_PROXY` and `HTTPS_PROXY` cannot reach the network at all, because the uid firewall rejects non-loopback traffic. It gets a connection error, not the page, and the attempt does not appear in the block log.
+- **Hosted search coverage is per harness.** Only the harnesses in the table have a switch. A harness with a hosted fetch that is not listed there can reach blocked pages through the provider.
+- **Only HTTP requests and `CONNECT` targets are inspected.** A tunnel to an unblocked host carries any protocol the client chooses, uninspected. A client that does not speak the HTTP proxy protocol at all is rejected by the firewall rather than filtered.
+
 ## Threat model and known gaps
 
 Benchflow's hardening assumes:

docs/task-authoring-task-md.md

@@ -76,7 +76,7 @@ so typos fail at parse time instead of becoming silently-ignored config:
 | `metadata` | Freeform mapping — difficulty, category, tags, anything descriptive |
 | `agent` | Agent run policy: `timeout_sec`, `user`, `network_mode`, `allowed_hosts` |
 | `verifier` | Verifier run policy: `timeout_sec` (default 600), `env`, `user`, `service`, … |
-| `sandbox` | Sandbox: `docker_image`, `cpus`, `memory_mb`, `storage_mb`, `network_mode`, `env`, `workdir`, … (legacy `task.toml` imports convert the Harbor `environment` table to this key; `environment:` in `task.md` is rejected with a rename hint) |
+| `sandbox` | Sandbox: `docker_image`, `cpus`, `memory_mb`, `storage_mb`, `network_mode`, `allowed_hosts`, `blocked_urls`, `blocked_hosts`, `env`, `workdir`, … (legacy `task.toml` imports convert the Harbor `environment` table to this key; `environment:` in `task.md` is rejected with a rename hint) |
 | `oracle` | Oracle run policy: `env`, `timeout_sec` (import alias: `solution`) |
 | `source`, `artifacts`, `steps`, `multi_step_reward_strategy`, `reward` | Provenance, artifact, and reward metadata |
 
@@ -111,6 +111,41 @@ profile name is a parse error. `bench tasks normalize <task-dir>` prints the
 fully expanded canonical document (`--write` replaces `task.md` in place), so
 a minimal authored file and its canonical form never drift apart.
 
+### Network policy
+
+`sandbox.network_mode` selects `no-network`, `allowlist`, `public`, or
+`denylist`. `denylist` keeps the internet reachable and makes the listed pages
+unreachable for the agent. Declare it on `sandbox`; the runtime reads the
+sandbox policy when it starts the filter:
+
+```yaml
+sandbox:
+  network_mode: denylist
+  blocked_urls:
+    - https://example.org/papers/lattice-qcd-2026
+    - github.com/example-org/lattice-qcd-code
+  blocked_hosts:
+    - mirror.example.net
+```
+
+`blocked_urls` entries are prefixes: the scheme is optional (`https://` is
+assumed), the host is lowercased, the query string is dropped, and every path
+under the prefix is blocked, including the path itself (`/abs/2401.12345` also
+covers `/abs/2401.12345v2`). A trailing slash is dropped. An entry without a
+path blocks every path on that host (with or without a leading `www.`) but not
+its other subdomains; use `blocked_hosts` for that. `blocked_hosts` entries
+block the host and all of its subdomains. IP literals, wildcards, ports,
+and userinfo are rejected. `denylist` requires at least one entry in either
+list, both lists are rejected under any other mode, and `denylist` is a
+sandbox-level policy: `agent.network_mode` and `verifier.network_mode` reject it.
+
+The mode needs a non-root `sandbox_user` and `python3` in the task image. It
+runs on `docker` and `daytona`; other backends refuse it at preflight. Blocked
+attempts are written to `trajectory/egress_denylist.jsonl` in the rollout
+directory. Read the
+[sandbox hardening notes](./sandbox-hardening.md#network-policy-denylist-egress)
+before relying on it: a blocklist hides pages, not knowledge.
+
 ---
 
 ## Prompt body and prompts/ sidecars

docs/task-standard.md

@@ -840,6 +840,7 @@ Current implementation status:
 | imported `steps` | yes | no/partial | fail closed per sandbox until implemented |
 | root/step artifacts | yes | no/partial | implement collection or fail closed |
 | network allowlist | yes | no/partial | per-sandbox capability check |
+| network denylist | yes | partial | `docker`: yes; `daytona`: yes; `modal`, `apple-container`, and `agentcore` refuse the mode at the capability gate |
 | separate verifier env | yes | no/partial | materializer plus verifier runner support |
 | Windows / TPU | yes | no | fail closed |
 | healthcheck | yes | no/partial | fail closed until sandbox healthcheck support lands |

src/benchflow/agents/install.py

@@ -99,7 +99,7 @@ def _owner_from_home(home: str) -> str | None:
 
 
 def _policy_home_dirs(agent: str, agent_cfg: AgentConfig) -> list[str]:
-    """Agent home dirs a no-web setup command may create."""
+    """Agent home dirs a web-policy setup command may create."""
     dirs = set(agent_cfg.home_dirs)
     for owned_path in agent_cfg.disallow_web_tools_owned_paths:
         if not owned_path.startswith("$HOME/"):
@@ -265,15 +265,21 @@ async def apply_web_tool_policy(
     home: str,
     *,
     disallow: bool,
+    disallow_hosted_search: bool = False,
 ) -> None:
-    """Apply an agent-specific hard web-tool disable in the agent home."""
-    if not disallow or not agent_cfg or not agent_cfg.disallow_web_tools_setup_cmd:
+    """Apply the agent's no-web or hosted-search-off setup command in the agent home."""
+    if not agent_cfg:
+        return
+    if disallow:
+        policy, setup_cmd = "no-web", agent_cfg.disallow_web_tools_setup_cmd
+    elif disallow_hosted_search:
+        policy, setup_cmd = "hosted-search", agent_cfg.disallow_hosted_search_setup_cmd
+    else:
+        return
+    if not setup_cmd:
         return
 
-    cmd = (
-        f"export BENCHFLOW_AGENT_HOME={shlex.quote(home)}; "
-        f"{agent_cfg.disallow_web_tools_setup_cmd}"
-    )
+    cmd = f"export BENCHFLOW_AGENT_HOME={shlex.quote(home)}; {setup_cmd}"
     owner = _owner_from_home(home)
     if owner:
         q_owner = shlex.quote(owner)
@@ -296,7 +302,7 @@ async def apply_web_tool_policy(
         if stderr:
             details.append(f"stderr: {stderr}")
         raise RuntimeError(
-            f"Failed to apply no-web policy for {agent}: {'; '.join(details)}"
+            f"Failed to apply {policy} policy for {agent}: {'; '.join(details)}"
         )
 
 

src/benchflow/agents/manifest.py

@@ -89,6 +89,8 @@
         "disallow_web_tools_setup_cmd",
         "disallow_web_tools_owned_paths",
         "disallow_web_tools_launch_suffix",
+        "disallow_hosted_search_setup_cmd",
+        "disallow_hosted_search_launch_suffix",
         "task_mcp_transport",
         "task_mcp_config_path",
     }