From 2944239e3b400bbe5ff80a0e8793eabf3f532503 Mon Sep 17 00:00:00 2001 From: questiondlmarks <255436763+questiondlmarks@users.noreply.github.com> Date: Tue, 8 Sep 2026 17:15:09 -0700 Subject: [PATCH 1/2] feat(task): add network_mode=blocklist with blocked_urls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Task configs can declare a list of host or host/path-prefix entries that must stay unreachable while every other destination remains open — the inverse of allowlist, for experiments that hide specific papers from a web-enabled agent. The mode is parsed and validated (agent, sandbox, verifier sections), reported as an unsupported runtime feature until the egress layer enforces it, and graded like public by the integration rubric. --- CHANGELOG.md | 12 ++ docs/task-authoring-task-md.md | 4 +- src/benchflow/task/config.py | 118 +++++++++++++++- src/benchflow/task/runtime_capabilities.py | 7 + tests/integration/rubric_checks.py | 50 +++++-- tests/test_rubric_checks.py | 22 +++ tests/test_runtime_capabilities.py | 29 ++++ tests/test_task_config.py | 148 +++++++++++++++++++++ 8 files changed, 376 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d66c6f8b..b7bdfb104 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,18 @@ ## [Unreleased] +### Added +- **`network_mode = "blocklist"` with `blocked_urls`.** Task configs (`agent`, + `sandbox`, and `verifier` sections) can now declare a list of hosts or + `host/path-prefix` entries that must stay unreachable while every other + destination remains open — the inverse of `allowlist`, for experiments that + hide specific papers or pages from a web-enabled agent. Entries accept pasted + `http(s)://` URLs and are normalized to `host[/path]`; ports, query strings, + fragments, and wildcards are rejected. Like `allowlist`, the mode is parsed + and validated but not yet enforced by any sandbox backend, and + `validate_task_runtime_support` reports it as an unsupported feature until + the egress layer lands. + ## 0.7.6 — 2026-09-04 ### Added diff --git a/docs/task-authoring-task-md.md b/docs/task-authoring-task-md.md index 7a04ebd69..27f43c952 100644 --- a/docs/task-authoring-task-md.md +++ b/docs/task-authoring-task-md.md @@ -74,9 +74,9 @@ so typos fail at parse time instead of becoming silently-ignored config: | `schema_version` (alias `version`) | Config schema version, currently `"1.3"` | | `task` | Package identity: `name` (`org/name` format), `description`, `authors`, `keywords`, `version` (informational Harbor 1.3 field, stored verbatim) | | `metadata` | Freeform mapping — difficulty, category, tags, anything descriptive | -| `agent` | Agent run policy: `timeout_sec`, `user`, `network_mode`, `allowed_hosts` | +| `agent` | Agent run policy: `timeout_sec`, `user`, `network_mode`, `allowed_hosts`, `blocked_urls` | | `verifier` | Verifier run policy: `timeout_sec` (default 600), `env`, `user`, `service`, … | -| `sandbox` | Sandbox: `docker_image`, `cpus`, `memory_mb`, `storage_mb`, `network_mode`, `env`, `workdir`, … (legacy `task.toml` imports convert the Harbor `environment` table to this key; `environment:` in `task.md` is rejected with a rename hint) | +| `sandbox` | Sandbox: `docker_image`, `cpus`, `memory_mb`, `storage_mb`, `network_mode`, `allowed_hosts`, `blocked_urls`, `env`, `workdir`, … (legacy `task.toml` imports convert the Harbor `environment` table to this key; `environment:` in `task.md` is rejected with a rename hint) | | `oracle` | Oracle run policy: `env`, `timeout_sec` (import alias: `solution`) | | `source`, `artifacts`, `steps`, `multi_step_reward_strategy`, `reward` | Provenance, artifact, and reward metadata | diff --git a/src/benchflow/task/config.py b/src/benchflow/task/config.py index ddc0b75b5..aa77cc1d4 100644 --- a/src/benchflow/task/config.py +++ b/src/benchflow/task/config.py @@ -157,6 +157,7 @@ class NetworkMode(StrEnum): NO_NETWORK = "no-network" PUBLIC = "public" ALLOWLIST = "allowlist" + BLOCKLIST = "blocklist" class TaskOS(StrEnum): @@ -202,14 +203,76 @@ def _validate_allowed_hosts(hosts: list[str] | None) -> list[str] | None: return normalized +def _validate_blocked_urls(urls: list[str] | None) -> list[str] | None: + """Normalize ``blocked_urls`` entries to ``host`` or ``host/path-prefix``. + + Entries may be pasted as full URLs (``https://arxiv.org/abs/2401.12345``); + the scheme is dropped and the host lower-cased so the stored form is the + exact string the egress layer matches against. Ports, query strings, + fragments, userinfo, and wildcards are rejected: the enforcement layer + matches host suffixes and path prefixes only, so accepting them here would + silently widen or narrow the policy. + """ + if urls is None: + return None + normalized: list[str] = [] + for raw_url in urls: + entry = raw_url.strip() + if not entry: + raise ValueError("blocked_urls entries must be non-empty") + lowered = entry.lower() + for scheme in ("https://", "http://"): + if lowered.startswith(scheme): + entry = entry[len(scheme) :] + break + else: + if "://" in entry: + raise ValueError( + "blocked_urls entries must be http(s) URLs or bare " + "host[/path] values" + ) + if "*" in entry: + raise ValueError("blocked_urls entries must not contain wildcards") + if "?" in entry or "#" in entry: + raise ValueError( + "blocked_urls entries must not contain query strings or fragments" + ) + host, _, path = entry.partition("/") + if "@" in host: + raise ValueError("blocked_urls entries must not contain userinfo") + if ":" in host: + raise ValueError("blocked_urls entries must not contain ports") + host = host.lower().rstrip(".") + if not host: + raise ValueError("blocked_urls entries must start with a hostname") + labels = host.split(".") + if not all(_NETWORK_HOST_LABEL_PATTERN.match(label) for label in labels): + raise ValueError( + "blocked_urls hostnames must contain only letters, digits, " + "hyphens, and dots" + ) + path = path.strip("/") + if any(ch.isspace() for ch in path): + raise ValueError("blocked_urls paths must not contain whitespace") + canonical = f"{host}/{path}" if path else host + if canonical not in normalized: + normalized.append(canonical) + return normalized + + def _validate_network_policy_fields( network_mode: NetworkMode | None, allowed_hosts: list[str] | None, + blocked_urls: list[str] | None = None, ) -> None: if network_mode == NetworkMode.ALLOWLIST and not allowed_hosts: raise ValueError("allowed_hosts must be non-empty for network_mode='allowlist'") if network_mode != NetworkMode.ALLOWLIST and allowed_hosts: raise ValueError("allowed_hosts is only valid for network_mode='allowlist'") + if network_mode == NetworkMode.BLOCKLIST and not blocked_urls: + raise ValueError("blocked_urls must be non-empty for network_mode='blocklist'") + if network_mode != NetworkMode.BLOCKLIST and blocked_urls: + raise ValueError("blocked_urls is only valid for network_mode='blocklist'") class Author(TaskConfigModel): @@ -416,6 +479,14 @@ class VerifierConfig(TaskConfigModel): default=None, description="Hostnames reachable when network_mode='allowlist'.", ) + blocked_urls: list[str] | None = Field( + default=None, + description=( + "URLs unreachable when network_mode='blocklist': each entry is a " + "hostname or host/path-prefix (scheme optional); everything else " + "stays reachable." + ), + ) sandbox_mode: VerifierSandboxMode | None = Field( default=None, description=( @@ -467,6 +538,11 @@ class VerifierConfig(TaskConfigModel): def validate_allowed_hosts(cls, hosts: list[str] | None) -> list[str] | None: return _validate_allowed_hosts(hosts) + @field_validator("blocked_urls") + @classmethod + def validate_blocked_urls(cls, urls: list[str] | None) -> list[str] | None: + return _validate_blocked_urls(urls) + @field_validator("reward_range") @classmethod def validate_reward_range( @@ -488,7 +564,9 @@ def reject_renamed_environment_keys(cls, data: Any) -> Any: @model_validator(mode="after") def validate_verifier_sandbox(self) -> VerifierConfig: - _validate_network_policy_fields(self.network_mode, self.allowed_hosts) + _validate_network_policy_fields( + self.network_mode, self.allowed_hosts, self.blocked_urls + ) if self.sandbox_mode == VerifierSandboxMode.SHARED and self.sandbox is not None: raise ValueError( "[verifier].sandbox_mode='shared' is incompatible with " @@ -529,6 +607,14 @@ class AgentConfig(TaskConfigModel): default=None, description="Hostnames reachable when network_mode='allowlist'.", ) + blocked_urls: list[str] | None = Field( + default=None, + description=( + "URLs unreachable when network_mode='blocklist': each entry is a " + "hostname or host/path-prefix (scheme optional); everything else " + "stays reachable." + ), + ) @field_validator("prompt_prefix") @classmethod @@ -545,9 +631,16 @@ def validate_prompt_prefix(cls, value: str | None) -> str | None: def validate_allowed_hosts(cls, hosts: list[str] | None) -> list[str] | None: return _validate_allowed_hosts(hosts) + @field_validator("blocked_urls") + @classmethod + def validate_blocked_urls(cls, urls: list[str] | None) -> list[str] | None: + return _validate_blocked_urls(urls) + @model_validator(mode="after") def validate_network_policy(self) -> AgentConfig: - _validate_network_policy_fields(self.network_mode, self.allowed_hosts) + _validate_network_policy_fields( + self.network_mode, self.allowed_hosts, self.blocked_urls + ) return self @@ -725,6 +818,14 @@ class SandboxConfig(TaskConfigModel): default=None, description="Hostnames reachable when network_mode='allowlist'.", ) + blocked_urls: list[str] | None = Field( + default=None, + description=( + "URLs unreachable when network_mode='blocklist': each entry is a " + "hostname or host/path-prefix (scheme optional); everything else " + "stays reachable." + ), + ) build_timeout_sec: float = 600.0 docker_image: str | None = Field( default=None, @@ -812,6 +913,11 @@ def _parse_size_to_mb(size_str: str) -> int: def validate_allowed_hosts(cls, hosts: list[str] | None) -> list[str] | None: return _validate_allowed_hosts(hosts) + @field_validator("blocked_urls") + @classmethod + def validate_blocked_urls(cls, urls: list[str] | None) -> list[str] | None: + return _validate_blocked_urls(urls) + @field_validator("os", mode="before") @classmethod def normalize_os(cls, value: Any) -> Any: @@ -821,7 +927,9 @@ def normalize_os(cls, value: Any) -> Any: @model_validator(mode="after") def handle_deprecated_fields_and_network_policy(self) -> SandboxConfig: - _validate_network_policy_fields(self.network_mode, self.allowed_hosts) + _validate_network_policy_fields( + self.network_mode, self.allowed_hosts, self.blocked_urls + ) memory = self.__dict__.get("memory") storage = self.__dict__.get("storage") if memory is not None: @@ -862,7 +970,9 @@ def handle_deprecated_fields_and_network_policy(self) -> SandboxConfig: self.allow_internet = False # Reconciliation must never leave the object in a state that # _validate_network_policy_fields itself rejects. - _validate_network_policy_fields(self.network_mode, self.allowed_hosts) + _validate_network_policy_fields( + self.network_mode, self.allowed_hosts, self.blocked_urls + ) return self diff --git a/src/benchflow/task/runtime_capabilities.py b/src/benchflow/task/runtime_capabilities.py index e624d5441..6f1bfd9e8 100644 --- a/src/benchflow/task/runtime_capabilities.py +++ b/src/benchflow/task/runtime_capabilities.py @@ -281,6 +281,13 @@ def _append_network_issue( reason="network allowlists are parsed but not enforced per sandbox", sandbox=sandbox, ) + if mode == NetworkMode.BLOCKLIST: + _issue( + unsupported, + path=path, + reason="network blocklists are parsed but not enforced per sandbox", + sandbox=sandbox, + ) def _append_document_issues( diff --git a/tests/integration/rubric_checks.py b/tests/integration/rubric_checks.py index d2bff0efd..6e1cd55d0 100644 --- a/tests/integration/rubric_checks.py +++ b/tests/integration/rubric_checks.py @@ -1071,7 +1071,10 @@ def _telemetry_shape(evidence: Evidence) -> set[str]: _NET_NO_NETWORK = "no-network" _NET_ALLOWLIST = "allowlist" _NET_PUBLIC = "public" -_VALID_NETWORK_MODES = frozenset({_NET_NO_NETWORK, _NET_ALLOWLIST, _NET_PUBLIC}) +_NET_BLOCKLIST = "blocklist" +_VALID_NETWORK_MODES = frozenset( + {_NET_NO_NETWORK, _NET_ALLOWLIST, _NET_PUBLIC, _NET_BLOCKLIST} +) def _norm_network_mode(value: Any) -> str | None: @@ -1096,10 +1099,15 @@ def network_hardening( verifier or the sandbox/lockdown surface a ``public`` mode is a hard ``fail`` (blocker) because that surface controls the isolation boundary. + ``blocklist`` (broad egress minus ``blocked_urls``) is graded exactly like + ``public``: the agent still reaches the open internet, so a blocklist is an + experiment-design control (hide specific papers), not an isolation boundary. + Returns the ``V-NETWORK`` gate outcome. ``pass`` for a hardened config, - ``fail`` for an unsafe one (missing allowlist hosts, or ``public`` on a - verifier/sandbox PR), ``quarantine`` for a ``public`` config on an unrelated - PR (documented, needs human sign-off), ``na`` when no policy is declared. + ``fail`` for an unsafe one (missing allowlist hosts, or ``public`` / + ``blocklist`` on a verifier/sandbox PR), ``quarantine`` for a ``public`` / + ``blocklist`` config on an unrelated PR (documented, needs human sign-off), + ``na`` when no policy is declared. """ mode = _norm_network_mode(task_config.get("network_mode")) raw_hosts = task_config.get("allowed_hosts") @@ -1108,21 +1116,40 @@ def network_hardening( for h in (raw_hosts if isinstance(raw_hosts, list) else []) if str(h).strip() ] + raw_blocked = task_config.get("blocked_urls") + blocked_urls = [ + str(u).strip() + for u in (raw_blocked if isinstance(raw_blocked, list) else []) + if str(u).strip() + ] if mode is None: # No declared policy => the runtime default (no-network) applies; an - # explicit allowlist list without a mode is a misconfiguration. + # explicit allowlist/blocklist without a mode is a misconfiguration. if allowed_hosts: return ( "V-NETWORK", "fail", "allowed_hosts declared without network_mode='allowlist'", ) + if blocked_urls: + return ( + "V-NETWORK", + "fail", + "blocked_urls declared without network_mode='blocklist'", + ) return ("V-NETWORK", "na", "no network_mode declared; runtime default applies") if mode not in _VALID_NETWORK_MODES: return ("V-NETWORK", "fail", f"unknown network_mode={mode!r}") + if mode != _NET_BLOCKLIST and blocked_urls: + return ( + "V-NETWORK", + "fail", + "blocked_urls is only valid for network_mode='blocklist'", + ) + if mode == _NET_NO_NETWORK: if allowed_hosts: return ( @@ -1145,17 +1172,24 @@ def network_hardening( f"allowlist hardened: hosts={sorted(allowed_hosts)}", ) - # mode == public + if mode == _NET_BLOCKLIST and not blocked_urls: + return ( + "V-NETWORK", + "fail", + "network_mode='blocklist' requires a non-empty blocked_urls", + ) + + # mode == public, or blocklist (open egress minus blocked_urls) if verifier_or_sandbox_pr: return ( "V-NETWORK", "fail", - "network_mode='public' on a verifier/sandbox PR (isolation boundary)", + f"network_mode={mode!r} on a verifier/sandbox PR (isolation boundary)", ) return ( "V-NETWORK", "quarantine", - "network_mode='public' (no allowlist) — requires human sign-off", + f"network_mode={mode!r} (no allowlist) — requires human sign-off", ) diff --git a/tests/test_rubric_checks.py b/tests/test_rubric_checks.py index 7dbc85f9d..67c2cc6b6 100644 --- a/tests/test_rubric_checks.py +++ b/tests/test_rubric_checks.py @@ -257,6 +257,28 @@ def test_network_hardening_public_is_blocker_on_verifier_sandbox_pr() -> None: assert sandbox[1] == "fail" +def test_network_hardening_blocklist_is_graded_like_public() -> None: + """Guards the blocklist schema PR: open egress minus blocked_urls is not hardened.""" + normal = rubric_checks.network_hardening( + {"network_mode": "blocklist", "blocked_urls": ["arxiv.org/abs/2401.12345"]} + ) + assert normal[1] == "quarantine" + sandbox = rubric_checks.network_hardening( + {"network_mode": "blocklist", "blocked_urls": ["arxiv.org/abs/2401.12345"]}, + verifier_or_sandbox_pr=True, + ) + assert sandbox[1] == "fail" + # An empty blocklist, or blocked_urls under any other mode, is a misconfiguration. + assert rubric_checks.network_hardening({"network_mode": "blocklist"})[1] == "fail" + assert ( + rubric_checks.network_hardening( + {"network_mode": "no-network", "blocked_urls": ["arxiv.org"]} + )[1] + == "fail" + ) + assert rubric_checks.network_hardening({"blocked_urls": ["arxiv.org"]})[1] == "fail" + + # ------------------------------------------------------------------ # Pinned-baseline reward-band parity (subprocess wrapper; needs benchflow). # ------------------------------------------------------------------ diff --git a/tests/test_runtime_capabilities.py b/tests/test_runtime_capabilities.py index 9763edda1..555cc257d 100644 --- a/tests/test_runtime_capabilities.py +++ b/tests/test_runtime_capabilities.py @@ -1127,3 +1127,32 @@ def test_sandbox_launch_allows_supported_legacy_task(tmp_path: Path) -> None: docker_sandbox.assert_called_once() assert result is docker_sandbox.return_value + + +def test_validator_reports_blocklist_as_runtime_gap() -> None: + """Guards the blocklist schema PR: parsed blocklists stay unsupported until enforced.""" + config = TaskConfig.model_validate( + { + "agent": { + "network_mode": "blocklist", + "blocked_urls": ["arxiv.org/abs/2401.12345"], + }, + "sandbox": { + "network_mode": "blocklist", + "blocked_urls": ["openreview.net"], + }, + } + ) + + issues = validate_task_runtime_support(config, sandbox="docker") + + assert [(issue.path, issue.reason) for issue in issues] == [ + ( + "agent.network_mode", + "network blocklists are parsed but not enforced per sandbox", + ), + ( + "sandbox.network_mode", + "network blocklists are parsed but not enforced per sandbox", + ), + ] diff --git a/tests/test_task_config.py b/tests/test_task_config.py index db1dd1646..7ad4509a9 100644 --- a/tests/test_task_config.py +++ b/tests/test_task_config.py @@ -481,3 +481,151 @@ def test_task_config_toml_converts_step_verifier_environment_with_indexed_error( "[steps.verifier.environment]\ncpus = 2\n" "[steps.verifier.sandbox]\ncpus = 3\n" ) + + +# ------------------------------------------------------------------ +# network_mode = "blocklist" / blocked_urls (schema only; enforcement is a +# follow-up PR — see runtime_capabilities for the unsupported-feature report). +# ------------------------------------------------------------------ + + +def test_task_config_accepts_blocklist_network_mode_in_every_section(): + """Guards the blocklist schema PR: the mode parses wherever network_mode does.""" + cfg = TaskConfig.model_validate_toml( + 'version = "1.0"\n' + "[agent]\n" + 'network_mode = "blocklist"\n' + 'blocked_urls = ["arxiv.org/abs/2401.12345"]\n' + "[verifier]\n" + 'network_mode = "blocklist"\n' + 'blocked_urls = ["openreview.net"]\n' + "[sandbox]\n" + 'network_mode = "blocklist"\n' + 'blocked_urls = ["https://arxiv.org/abs/2401.12345", "openreview.net"]\n' + ) + + assert cfg.agent.network_mode == NetworkMode.BLOCKLIST + assert cfg.agent.blocked_urls == ["arxiv.org/abs/2401.12345"] + assert cfg.verifier.network_mode == NetworkMode.BLOCKLIST + assert cfg.verifier.blocked_urls == ["openreview.net"] + assert cfg.sandbox.network_mode == NetworkMode.BLOCKLIST + assert cfg.sandbox.blocked_urls == ["arxiv.org/abs/2401.12345", "openreview.net"] + # A blocklist keeps the internet open — it never collapses to no-network. + assert cfg.sandbox.allow_internet is True + + +@pytest.mark.parametrize( + ("raw", "expected"), + [ + ("https://ArXiv.org/abs/2401.12345", "arxiv.org/abs/2401.12345"), + ("http://arxiv.org/abs/2401.12345/", "arxiv.org/abs/2401.12345"), + (" OpenReview.net. ", "openreview.net"), + ("arxiv.org/pdf/2401.12345v2", "arxiv.org/pdf/2401.12345v2"), + ("Semantic-Scholar.org/", "semantic-scholar.org"), + ], +) +def test_blocked_urls_normalize_to_host_and_path_prefix(raw, expected): + """Pasted paper URLs land in the exact ``host[/path]`` form the egress layer matches.""" + cfg = TaskConfig.model_validate( + {"sandbox": {"network_mode": "blocklist", "blocked_urls": [raw]}} + ) + assert cfg.sandbox.blocked_urls == [expected] + + +def test_blocked_urls_deduplicate_after_normalization(): + cfg = TaskConfig.model_validate( + { + "sandbox": { + "network_mode": "blocklist", + "blocked_urls": [ + "https://arxiv.org/abs/2401.12345", + "ARXIV.ORG/abs/2401.12345/", + "openreview.net", + ], + } + } + ) + assert cfg.sandbox.blocked_urls == ["arxiv.org/abs/2401.12345", "openreview.net"] + + +@pytest.mark.parametrize( + ("raw", "message"), + [ + ("", "non-empty"), + ("ftp://arxiv.org/abs/1", r"http\(s\) URLs"), + ("*.arxiv.org", "wildcards"), + ("arxiv.org/abs?id=1", "query strings"), + ("arxiv.org/abs#frag", "query strings or fragments"), + ("user@arxiv.org", "userinfo"), + ("arxiv.org:443/abs/1", "ports"), + ("/abs/2401.12345", "start with a hostname"), + ("arx_iv.org", "hostnames must contain only"), + ("arxiv.org/abs/2401 12345", "whitespace"), + ], +) +def test_blocked_urls_reject_unmatchable_entries(raw, message): + """Anything the host-suffix/path-prefix matcher cannot honor fails at parse time.""" + with pytest.raises(ValueError, match=message): + TaskConfig.model_validate( + {"sandbox": {"network_mode": "blocklist", "blocked_urls": [raw]}} + ) + + +def test_blocklist_requires_non_empty_blocked_urls(): + with pytest.raises(ValueError, match="blocked_urls must be non-empty"): + TaskConfig.model_validate({"sandbox": {"network_mode": "blocklist"}}) + with pytest.raises(ValueError, match="blocked_urls must be non-empty"): + TaskConfig.model_validate( + {"sandbox": {"network_mode": "blocklist", "blocked_urls": []}} + ) + + +@pytest.mark.parametrize("mode", ["public", "no-network", "allowlist"]) +def test_blocked_urls_only_valid_with_blocklist_mode(mode): + data: dict = {"network_mode": mode, "blocked_urls": ["arxiv.org"]} + if mode == "allowlist": + data["allowed_hosts"] = ["api.example.com"] + with pytest.raises(ValueError, match="only valid for network_mode='blocklist'"): + TaskConfig.model_validate({"sandbox": data}) + + +def test_blocklist_rejects_allowed_hosts(): + """allowed_hosts and blocked_urls are mutually exclusive policy shapes.""" + with pytest.raises(ValueError, match="only valid for network_mode='allowlist'"): + TaskConfig.model_validate( + { + "sandbox": { + "network_mode": "blocklist", + "blocked_urls": ["arxiv.org"], + "allowed_hosts": ["api.example.com"], + } + } + ) + + +def test_blocklist_contradicts_deprecated_allow_internet_false(): + """The deprecated flag must not silently downgrade an explicit blocklist.""" + with pytest.raises(ValueError, match="allow_internet=False contradicts"): + TaskConfig.model_validate( + { + "sandbox": { + "network_mode": "blocklist", + "blocked_urls": ["arxiv.org"], + "allow_internet": False, + } + } + ) + + +def test_blocklist_round_trips_through_toml_dump(): + cfg = TaskConfig.model_validate( + { + "sandbox": { + "network_mode": "blocklist", + "blocked_urls": ["https://arxiv.org/abs/2401.12345"], + } + } + ) + reparsed = TaskConfig.model_validate_toml(cfg.model_dump_toml()) + assert reparsed.sandbox.network_mode == NetworkMode.BLOCKLIST + assert reparsed.sandbox.blocked_urls == ["arxiv.org/abs/2401.12345"] From 97f96a1a74535ccbe9edf0ec3ab134ae5fe013fc Mon Sep 17 00:00:00 2001 From: questiondlmarks <255436763+questiondlmarks@users.noreply.github.com> Date: Tue, 8 Sep 2026 17:15:09 -0700 Subject: [PATCH 2/2] feat(sandbox): enforce agent egress blocklist with in-container proxy and firewall Implement layered, protocol-level egress blocklist enforcement (network_mode="blocklist") to hide target research papers and mirrors from web-enabled agents while preserving native container internet access, binary-safe downloads, and 404 stealth behavior. Key changes: - Sandbox-local filtering proxy (`src/benchflow/sandbox/egress.py`): * Root-run stdlib HTTP/1.1 forward proxy on loopback with MAX_CONNECTIONS=256 backpressure. * Transparent CONNECT tunneling for unblocked hosts; TLS MITM inspection with a per-run CA bundle for hosts carrying path-specific rules. * Stealth 404 Not Found status for blocked endpoints (indistinguishable from missing pages). * SSRF defense: vets all resolved addresses and blocks loopback, link-local (169.254.169.254), and reserved ranges, while preserving RFC1918 private nets for compose side services. * Dual-host checking (URL target + Host header) and path normalization (percent-decoding, dot-segments, slash-collapsing) to prevent WAF evasion. * IPv4-preferred candidate traversal in resolve_upstream for Docker bridge compatibility. - Kernel firewall & container isolation: * Injects docker-compose-net-admin.yaml overlay when an agent network policy is active. * Post-bootstrap iptables rule confines the agent UID to loopback so non-proxy egress fails closed. * Sandbox-user self-check probe runs before the first prompt and fails fast on policy breach. - Model provider & harness synchronization: * LiteLLM pre-call hook rewrites Anthropic server-side web tools with blocked_domains and strips OpenAI hosted search tools. * Harness knobs disable server-side web tools for Codex and Gemini. * Strips rule secret list from agent process env; downloads root-only egress.jsonl audit log. - Batch evaluation & CLI overlays: * Adds --block-url and --block-url-file to bench eval run as C-axis config overlays. * Adds NetworkPolicyPreflightError to validate task compatibility before rollouts start. * Adds ResumeMismatchError guard against resuming jobs with differing network policies. - Cross-platform & test suite: * Enforces encoding="utf-8" across shim readers and config JSONs for Windows compatibility. * Adds comprehensive test suite (tests/test_egress_blocklist.py, 63 tests) covering proxy, TLS inspection, path normalization, ALPN downgrade, and batch preflight. --- CHANGELOG.md | 30 +- docs/reference/cli.md | 2 + docs/sandbox-hardening.md | 122 ++ docs/task-authoring-task-md.md | 2 +- src/benchflow/_utils/config_override.py | 37 + src/benchflow/acp/runtime.py | 25 +- src/benchflow/agents/install.py | 27 +- src/benchflow/agents/manifest.py | 2 + src/benchflow/agents/registry.py | 43 +- src/benchflow/cli/main.py | 23 +- src/benchflow/contracts/planes.py | 4 +- src/benchflow/evaluation.py | 149 +- src/benchflow/providers/litellm_logging.py | 103 +- src/benchflow/providers/litellm_runtime.py | 7 +- src/benchflow/rollout/__init__.py | 101 +- src/benchflow/rollout/_results.py | 5 + src/benchflow/rollout/_setup.py | 102 +- src/benchflow/rollout_planes.py | 22 +- src/benchflow/sandbox/_compose.py | 1 + .../docker-compose-net-admin.yaml | 6 + src/benchflow/sandbox/docker.py | 9 + src/benchflow/sandbox/egress.py | 1159 ++++++++++++ src/benchflow/sandbox/lockdown.py | 24 +- src/benchflow/sandbox/providers.py | 11 + src/benchflow/sandbox/setup.py | 1 + src/benchflow/task/runtime_capabilities.py | 24 +- tests/test_egress_blocklist.py | 1566 +++++++++++++++++ tests/test_internet_policy.py | 10 +- tests/test_runtime_capabilities.py | 57 +- 29 files changed, 3593 insertions(+), 81 deletions(-) create mode 100644 src/benchflow/sandbox/_compose_files/docker-compose-net-admin.yaml create mode 100644 src/benchflow/sandbox/egress.py create mode 100644 tests/test_egress_blocklist.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b7bdfb104..1fbe15e78 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,16 +3,26 @@ ## [Unreleased] ### Added -- **`network_mode = "blocklist"` with `blocked_urls`.** Task configs (`agent`, - `sandbox`, and `verifier` sections) can now declare a list of hosts or - `host/path-prefix` entries that must stay unreachable while every other - destination remains open — the inverse of `allowlist`, for experiments that - hide specific papers or pages from a web-enabled agent. Entries accept pasted - `http(s)://` URLs and are normalized to `host[/path]`; ports, query strings, - fragments, and wildcards are rejected. Like `allowlist`, the mode is parsed - and validated but not yet enforced by any sandbox backend, and - `validate_task_runtime_support` reports it as an unsupported feature until - the egress layer lands. +- **Egress blocklist: `network_mode = "blocklist"` with `blocked_urls`.** The + agent keeps full internet access except for a declared list of `host` or + `host/path-prefix` entries — the inverse of `allowlist`, for experiments that + hide specific papers or pages from a web-enabled research agent. Declare it in + `task.md` (`agent`/`sandbox` sections) or per run with + `bench eval run --block-url https://arxiv.org/abs/2401.12345 --block-url-file hidden.txt` + (a C-axis overlay that replaces task-level `blocked_urls`). Enforcement is + layered so every path an agent has to the web is covered: a root-run + filtering proxy on the sandbox loopback (host rules reject `CONNECT` + tunnels; hosts with path rules are TLS-inspected under a per-run CA installed + into the sandbox trust store), the existing agent-UID iptables rule so tools + that ignore `HTTP(S)_PROXY` fail closed, Anthropic `blocked_domains` + injection plus OpenAI hosted-search stripping in the LiteLLM pre-call hook, + and narrow per-harness knobs (`blocklist_web_tools_*`) that switch off only + server-side search (Codex `web_search`, Gemini grounding). Blocked requests + answer `404`, the rule list never enters the agent process env, a self-check + runs as the sandbox user before the first prompt, and every decision lands in + `agent/egress.jsonl` next to a `network_policy` block in `config.json`. + Supported on docker (stacks a `NET_ADMIN` compose overlay) and daytona; modal, + apple-container, and agentcore refuse blocklist tasks before launch. ## 0.7.6 — 2026-09-04 diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 01c0c2018..04811777b 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -263,6 +263,8 @@ bench eval run --tasks-dir ./tasks --matrix matrix.yaml --trials 3 | `--skill-creator-dir` | — | Path to a `skill-creator` directory (or a skills root containing it); used when `--skill-mode self-gen` | | `--self-gen-no-internet` | `false` | Disable web tools for the self-generated skill run | | `--agent-env` | — | Agent environment variable as `KEY=VALUE`; repeatable | +| `--block-url` | — | Hide a host or `host/path-prefix` from the agent for this run (repeatable; pasted `https://` URLs accepted). Internet stays open otherwise. Sets `sandbox.network_mode=blocklist` via the C-axis overlay and replaces task-level `blocked_urls`; see [Sandbox hardening → Egress blocklist](../sandbox-hardening.md#egress-blocklist-network_mode--blocklist) | +| `--block-url-file` | — | File with one host or `host/path-prefix` per line (`#` comments allowed); merged with `--block-url` | | `--include` | — | Only run these task names; repeatable (e.g. `--include jax-computing-basics --include data-to-d3`) | | `--exclude` | — | Skip these task names; repeatable (e.g. `--exclude quantum-numerical-simulation`) | | `--loop-strategy` | — | Wrap each rollout in a loop, e.g. `verify-retry:k=3,feedback=names` or `self-review:k=3` (omit for single-shot) | diff --git a/docs/sandbox-hardening.md b/docs/sandbox-hardening.md index cdc7ae870..bc7a5c544 100644 --- a/docs/sandbox-hardening.md +++ b/docs/sandbox-hardening.md @@ -52,6 +52,128 @@ Known residual risk: - An agent with sustained access can poison `__pycache__` for files that exist in the baseline (those caches aren't deleted because some tasks diff workspace against `/testbed_verify`). Mitigated by the workspace chown but not eliminated. - Tasks that don't ship a build-config snapshot can still be hijacked via `setup.py` edits. Snapshot is automatic for declared filenames — task authors don't need to opt in. +## Egress blocklist (`network_mode = "blocklist"`) + +`no-network` and `allowlist` answer "may the agent reach the internet at all". +The blocklist answers a different research question: **the agent may use the +whole web except a list of URLs it must not discover** — e.g. hide the paper +under evaluation (and its mirrors) from a deep-research agent while every +other paper stays readable. + +```toml +[sandbox] +network_mode = "blocklist" +blocked_urls = [ + "https://arxiv.org/abs/2401.12345", # normalized to arxiv.org/abs/2401.12345 + "arxiv.org/pdf/2401.12345", # list every mirror path you care about + "openreview.net", # a bare host blocks it and its subdomains +] +``` + +Per run, without editing the task: `bench eval run … --block-url arxiv.org/abs/2401.12345 --block-url-file hidden.txt` +(the run-level list replaces the task's `blocked_urls`; the config is +re-validated, so a `no-network` or `allowlist` task fails loudly, and so does +a task whose `agent` section pins its own `network_mode` — that override would +otherwise shadow the sandbox blocklist). + +### How it is enforced + +A URL blocklist has to be enforced in three places, because no single layer +sees every route an agent has to the web (`src/benchflow/sandbox/egress.py`): + +1. **Sandbox-local filtering proxy.** Before the agent launches, a root-run + stdlib proxy starts on the sandbox loopback and the agent env gets + `HTTP_PROXY`/`HTTPS_PROXY` (plus `NODE_USE_ENV_PROXY=1` for Node fetch). + Plain HTTP exposes the full URL, so host and path rules both apply. HTTPS + `CONNECT` only exposes the host: a host rule rejects the tunnel; a host that + carries **path** rules is TLS-inspected — the proxy terminates TLS with a + leaf certificate signed by a per-run CA (installed into the system trust + store and exported via `SSL_CERT_FILE`, `REQUESTS_CA_BUNDLE`, + `CURL_CA_BUNDLE`, `NODE_EXTRA_CA_CERTS`, `GIT_SSL_CAINFO`), reads the path, + and forwards allowed requests upstream over a fresh verified TLS connection. + The CA key, the rule file, and the log are root-only. +2. **Agent-UID firewall.** The same iptables rule the no-web policy uses + confines the agent UID to loopback, so anything that ignores the proxy env + (a hand-rolled socket, `curl --noproxy`, DoH) fails closed instead of + bypassing the filter. Under the blocklist it is applied **before the agent + process is launched** (the proxy is already listening), so there is no + pre-handshake window; the no-web policy keeps its post-handshake placement. The LiteLLM model proxy is forced + sandbox-local for the same reason. Docker stacks a `NET_ADMIN` compose + overlay for these runs; Daytona allows iptables natively. +3. **Server-side web tools** run at the provider, outside the sandbox. The + LiteLLM pre-call hook merges the rules into Anthropic `web_search_*` / + `web_fetch_*` `blocked_domains` (domains and `domain/path` prefixes) and + strips OpenAI hosted `web_search` tools (allowlist-only filter). Harness + knobs cover the rest: Codex gets `-c tools.web_search=false`, Gemini + excludes `google_web_search` and `web_fetch`. Client-side fetchers + (OpenCode/MiMo `webfetch`, OpenHands browsing, Claude Code `WebFetch`) stay + enabled because their traffic goes through layer 1. + +Blocked requests answer **`404 Not Found`**, so a research agent cannot tell a +hidden page from a missing one. The rule list is deliberately kept out of the +agent process env (`BENCHFLOW_EGRESS_BLOCKED_URLS` reaches only the model +proxy). Before the first prompt a self-check runs **as the sandbox user**: the +first blocked host must answer 404 through the proxy and direct egress must be +rejected; a failed check aborts the rollout rather than running it open. + +### Batch safety + +`bench eval run` resolves every selected task's network posture under the +run's overlay and backend before the first rollout starts; a `--block-url` +against a task that declares `no-network`/`allowlist`, or a blocklist task on +a backend that cannot enforce it, fails the whole batch up front naming the +offending tasks. Resuming a job whose completed tasks recorded a different +`network_policy` (open vs. blocklisted, or a different list) is refused, the +same way an agent mismatch is — those scores belong to different experiments. + +### Auditing a run + +- `config.json` carries a `network_policy` block (`mode`, `blocked_urls`, + `tls_inspection_hosts`, `blocked_status`). +- `agent/egress.jsonl` (downloaded from the root-only log at disconnect) lists + every `allow` / `block` decision with host, path, and matched rule, plus the + `probe` self-check record — the evidence that the agent attempted (or never + attempted) the hidden URLs. +- Server-side search really being off is verified from the request bodies in + `trajectory/llm_trajectory.jsonl`, not from config files. + +### Limits + +- The blocklist hides **URLs**, not knowledge: search-result snippets from + unblocked engines, citations in other papers, and the model's own training + data can still reveal that a paper exists. Block the mirrors you care about + (Semantic Scholar, OpenReview, alphaXiv, HF Papers, …) by host. +- Path rules are prefix matches: `arxiv.org/abs/2401.12345` also hides + `…/abs/2401.123456` (and, usefully, `…/abs/2401.12345v2`). Matching runs on + the canonical path an upstream server would route — percent-decoded, + `..`/`.`/`//` collapsed, case-folded — and on both the URL host and the + `Host` header, so encoding tricks or an IP-literal URL with a spoofed + `Host` do not slip past. Percent-decoding is repeated until stable, so a + double-encoded separator cannot reach an upstream that decodes twice. +- Only the agent phase is covered; `verifier.network_mode = "blocklist"` is + rejected before launch. Modal, Apple Container, and AgentCore cannot run + the root proxy + UID firewall and refuse blocklist tasks. +- The proxy needs `python3` in the task image (and `openssl` for TLS + inspection; it is apt/dnf/apk-installed on demand). +- The proxy runs as root, outside the agent-UID firewall, so it vets every + resolved upstream address and answers `403` for loopback, link-local (cloud + instance metadata such as `169.254.169.254`), unspecified, multicast, + reserved, and private ranges. The only private addresses it will reach are + the container's own directly-connected subnets (from `/proc/net/route`), + which is where compose side-services live; other RFC1918 space — the host + LAN behind the bridge, other projects' networks — is refused. +- Session-factory agents run in-process on the host, outside the sandbox + proxy and firewall; a blocklist run refuses them before connecting. +- TLS inspection speaks HTTP/1.1 only (ALPN advertises just `http/1.1`, so + HTTP/2-capable clients negotiate down). +- The proxy handles at most 256 connections at once; a burst of concurrent + fetches queues in the listen backlog rather than exhausting threads or file + descriptors. +- When the primary agent is the oracle, the oracle itself is exempt from the + blocklist, but the container is still provisioned for it (docker + `NET_ADMIN`, sandbox-local model proxy) so role agents connecting later are + covered. + ## Related - [`progressive-disclosure.md`](./progressive-disclosure.md) — soft-verify (the relaxed hardening used between rounds in multi-round trials). diff --git a/docs/task-authoring-task-md.md b/docs/task-authoring-task-md.md index 27f43c952..cc1d199fb 100644 --- a/docs/task-authoring-task-md.md +++ b/docs/task-authoring-task-md.md @@ -76,7 +76,7 @@ so typos fail at parse time instead of becoming silently-ignored config: | `metadata` | Freeform mapping — difficulty, category, tags, anything descriptive | | `agent` | Agent run policy: `timeout_sec`, `user`, `network_mode`, `allowed_hosts`, `blocked_urls` | | `verifier` | Verifier run policy: `timeout_sec` (default 600), `env`, `user`, `service`, … | -| `sandbox` | Sandbox: `docker_image`, `cpus`, `memory_mb`, `storage_mb`, `network_mode`, `allowed_hosts`, `blocked_urls`, `env`, `workdir`, … (legacy `task.toml` imports convert the Harbor `environment` table to this key; `environment:` in `task.md` is rejected with a rename hint) | +| `sandbox` | Sandbox: `docker_image`, `cpus`, `memory_mb`, `storage_mb`, `network_mode`, `allowed_hosts`, `blocked_urls` (see [Sandbox hardening → Egress blocklist](./sandbox-hardening.md#egress-blocklist-network_mode--blocklist)), `env`, `workdir`, … (legacy `task.toml` imports convert the Harbor `environment` table to this key; `environment:` in `task.md` is rejected with a rename hint) | | `oracle` | Oracle run policy: `env`, `timeout_sec` (import alias: `solution`) | | `source`, `artifacts`, `steps`, `multi_step_reward_strategy`, `reward` | Provenance, artifact, and reward metadata | diff --git a/src/benchflow/_utils/config_override.py b/src/benchflow/_utils/config_override.py index 476070ee0..17339c561 100644 --- a/src/benchflow/_utils/config_override.py +++ b/src/benchflow/_utils/config_override.py @@ -90,6 +90,43 @@ def load_config_override(value: str | None) -> dict[str, Any] | None: return _parse_overlay(value) +def blocklist_override( + raw_override: str | None, + block_urls: list[str] | None, + block_url_file: str | Path | None, +) -> str | None: + """Fold ``--block-url`` / ``--block-url-file`` into a C-axis overlay string. + + The run-level list REPLACES any task-level ``blocked_urls`` (overlay lists + are not unioned) and forces ``sandbox.network_mode = "blocklist"``; the + task config is re-validated at rollout, so a task that declared + ``no-network`` or ``allowlist`` fails loudly instead of silently changing + posture. A task whose ``agent`` section pins its own ``network_mode`` is + refused as well (that override would otherwise shadow the sandbox + blocklist and serve the hidden URLs). Returns ``raw_override`` untouched + when no URLs were given. + """ + entries: list[str] = [] + for url in block_urls or []: + url = url.strip() + if url and url not in entries: + entries.append(url) + if block_url_file is not None: + listing = Path(block_url_file).expanduser().read_text(encoding="utf-8") + for line in listing.splitlines(): + line = line.split("#", 1)[0].strip() + if line and line not in entries: + entries.append(line) + if not entries: + return raw_override + overlay = dict(load_config_override(raw_override) or {}) + sandbox = dict(overlay.get("sandbox") or {}) + sandbox["network_mode"] = "blocklist" + sandbox["blocked_urls"] = entries + overlay["sandbox"] = sandbox + return json.dumps(overlay) + + def deep_merge(base: dict[str, Any], overlay: dict[str, Any]) -> dict[str, Any]: """Recursively merge ``overlay`` into ``base``. diff --git a/src/benchflow/acp/runtime.py b/src/benchflow/acp/runtime.py index 1af7895c8..7195457a3 100644 --- a/src/benchflow/acp/runtime.py +++ b/src/benchflow/acp/runtime.py @@ -51,6 +51,11 @@ TransportClosedDiagnostic, TransportClosedError, ) +from benchflow.sandbox.egress import ( + blocklist_active, + strip_blocklist_secret, + verify_egress_blocklist, +) from benchflow.sandbox.lockdown import ( build_priv_drop_cmd, enforce_agent_egress_firewall, @@ -644,6 +649,20 @@ async def connect_acp( agent_launch = build_priv_drop_cmd(agent_launch, sandbox_user) logger.info(f"Agent sandboxed as: {sandbox_user}") + # Egress blocklist: the filtering proxy was started in the install phase + # (Rollout.install_agent / connect_as); here the rule list is kept out of + # the agent process env and the post-firewall self-check runs below. + process_env = strip_blocklist_secret(agent_env) + # Under the blocklist the proxy is already up and HTTP(S)_PROXY is in the + # agent env, so the UID firewall goes up BEFORE the agent process exists: + # startup traffic that ignores the proxy fails closed instead of enjoying + # a pre-handshake window. The no-web policy keeps its post-handshake + # placement (its agents have no proxy to fall back on during bootstrap). + firewall_applied = False + if blocklist_active(agent_env): + await enforce_agent_egress_firewall(env, sandbox_user, agent_env) + firewall_applied = True + acp_client: ACPClient | None = None session: object | None = None agent_name = agent @@ -668,7 +687,7 @@ async def connect_acp( transport = ContainerTransport( container_process=live_proc, command=agent_launch, - env=agent_env, + env=process_env, cwd=agent_cwd, agent_log_path=agent_log, ) @@ -720,7 +739,9 @@ async def connect_acp( reasoning_effort=reasoning_effort, launch_config_owns_model=launch_config_owns_model, ) - await enforce_agent_egress_firewall(env, sandbox_user, agent_env) + if not firewall_applied: + await enforce_agent_egress_firewall(env, sandbox_user, agent_env) + await verify_egress_blocklist(env, sandbox_user, agent_env) except Exception: with contextlib.suppress(Exception): await acp_client.close() diff --git a/src/benchflow/agents/install.py b/src/benchflow/agents/install.py index a65d8d6df..9beff15b4 100644 --- a/src/benchflow/agents/install.py +++ b/src/benchflow/agents/install.py @@ -265,15 +265,28 @@ async def apply_web_tool_policy( home: str, *, disallow: bool, + blocklist: bool = False, ) -> None: - """Apply an agent-specific hard web-tool disable in the agent home.""" - if not disallow or not agent_cfg or not agent_cfg.disallow_web_tools_setup_cmd: + """Apply an agent-specific web-tool policy in the agent home. + + ``disallow`` applies the hard no-web disable; otherwise ``blocklist`` + applies the narrower egress-blocklist switch (server-side search tools + only). A no-web run always wins over a blocklist. + """ + if not agent_cfg: + return + if disallow: + setup_cmd = agent_cfg.disallow_web_tools_setup_cmd + policy_name = "no-web" + elif blocklist: + setup_cmd = agent_cfg.blocklist_web_tools_setup_cmd + policy_name = "egress-blocklist" + else: + return + if not setup_cmd: return - cmd = ( - f"export BENCHFLOW_AGENT_HOME={shlex.quote(home)}; " - f"{agent_cfg.disallow_web_tools_setup_cmd}" - ) + cmd = f"export BENCHFLOW_AGENT_HOME={shlex.quote(home)}; {setup_cmd}" owner = _owner_from_home(home) if owner: q_owner = shlex.quote(owner) @@ -296,7 +309,7 @@ async def apply_web_tool_policy( if stderr: details.append(f"stderr: {stderr}") raise RuntimeError( - f"Failed to apply no-web policy for {agent}: {'; '.join(details)}" + f"Failed to apply {policy_name} policy for {agent}: {'; '.join(details)}" ) diff --git a/src/benchflow/agents/manifest.py b/src/benchflow/agents/manifest.py index daebc8cc0..2cbeef005 100644 --- a/src/benchflow/agents/manifest.py +++ b/src/benchflow/agents/manifest.py @@ -89,6 +89,8 @@ "disallow_web_tools_setup_cmd", "disallow_web_tools_owned_paths", "disallow_web_tools_launch_suffix", + "blocklist_web_tools_setup_cmd", + "blocklist_web_tools_launch_suffix", "task_mcp_transport", "task_mcp_config_path", } diff --git a/src/benchflow/agents/registry.py b/src/benchflow/agents/registry.py index 127ef423f..9d6ce3f6c 100644 --- a/src/benchflow/agents/registry.py +++ b/src/benchflow/agents/registry.py @@ -282,16 +282,24 @@ def _js_agent_launch(binary: str, args: str = "") -> str: # Path to the openclaw ACP shim script -_OPENCLAW_SHIM = (Path(__file__).parent / "openclaw_acp_shim.py").read_text() +_OPENCLAW_SHIM = (Path(__file__).parent / "openclaw_acp_shim.py").read_text( + encoding="utf-8" +) # Path to the Pi launch wrapper (bridges BENCHFLOW_PROVIDER_* → Pi config) -_PI_LAUNCHER = (Path(__file__).parent / "pi_acp_launcher.py").read_text() +_PI_LAUNCHER = (Path(__file__).parent / "pi_acp_launcher.py").read_text( + encoding="utf-8" +) # Path to the Harvey LAB ACP shim (runs Harvey LAB harness as an ACP agent) -_HARVEY_LAB_SHIM = (Path(__file__).parent / "harvey_lab_acp_shim.py").read_text() +_HARVEY_LAB_SHIM = (Path(__file__).parent / "harvey_lab_acp_shim.py").read_text( + encoding="utf-8" +) # Path to the deepagents ACP shim (runs LangChain's create_deep_agent as an ACP agent) -_DEEPAGENTS_SHIM = (Path(__file__).parent / "deepagents_acp_shim.py").read_text() +_DEEPAGENTS_SHIM = (Path(__file__).parent / "deepagents_acp_shim.py").read_text( + encoding="utf-8" +) def _json_settings_merge(path: str, mutator: str) -> str: @@ -502,6 +510,15 @@ class AgentConfig: # String appended to launch_cmd when BenchFlow's no-web policy is active. # Use for agents whose supported toggle is a launch/config override. disallow_web_tools_launch_suffix: str = "" + # Egress BLOCKLIST policy (network_mode='blocklist'): the sandbox proxy + # filters everything the agent fetches from inside the container, so only + # SERVER-SIDE web tools that never touch the sandbox network need a + # harness-level switch here (Codex web_search at OpenAI, Gemini grounding). + # Anthropic server tools are filtered in the LiteLLM pre-call hook instead. + # Client-side fetch tools stay enabled. Owned paths reuse + # disallow_web_tools_owned_paths. + blocklist_web_tools_setup_cmd: str = "" + blocklist_web_tools_launch_suffix: str = "" # How task-declared MCP servers are delivered to the agent: # "acp" sends them in session/new; "native-config" writes an agent-specific # config file before launch (for agents whose ACP server drops/reformats @@ -650,6 +667,9 @@ class AgentConfig: ], ), disallow_web_tools_launch_suffix=" -c tools.web_search=false", + # OpenAI's hosted web_search only supports an allowlist filter, so a + # blocklist run must switch it off rather than filter it. + blocklist_web_tools_launch_suffix=" -c tools.web_search=false", ), "gemini": AgentConfig( name="gemini", @@ -702,6 +722,15 @@ class AgentConfig: 'if t not in d["tools"]["exclude"]]', ), disallow_web_tools_owned_paths=["$HOME/.gemini"], + # Google grounding has no domain filter and web_fetch prefers the + # server-side urlContext path, so both stay off under a blocklist. + blocklist_web_tools_setup_cmd=_json_settings_merge( + "$BENCHFLOW_AGENT_HOME/.gemini/settings.json", + 'd.setdefault("tools",{}).setdefault("exclude",[]);' + '[d["tools"]["exclude"].append(t) for t in ' + '["google_web_search","web_fetch"] ' + 'if t not in d["tools"]["exclude"]]', + ), ), "opencode": AgentConfig( name="opencode", @@ -1183,6 +1212,8 @@ def _acpx_wrap(config: AgentConfig) -> AgentConfig: disallow_web_tools_setup_cmd=config.disallow_web_tools_setup_cmd, disallow_web_tools_owned_paths=config.disallow_web_tools_owned_paths, disallow_web_tools_launch_suffix=config.disallow_web_tools_launch_suffix, + blocklist_web_tools_setup_cmd=config.blocklist_web_tools_setup_cmd, + blocklist_web_tools_launch_suffix=config.blocklist_web_tools_launch_suffix, task_mcp_transport=config.task_mcp_transport, task_mcp_config_path=config.task_mcp_config_path, ) @@ -1377,6 +1408,8 @@ def register_agent( disallow_web_tools_setup_cmd: str = "", disallow_web_tools_owned_paths: list[str] | None = None, disallow_web_tools_launch_suffix: str = "", + blocklist_web_tools_setup_cmd: str = "", + blocklist_web_tools_launch_suffix: str = "", ) -> AgentConfig: """Register a custom agent at runtime. @@ -1416,6 +1449,8 @@ def register_agent( disallow_web_tools_setup_cmd=disallow_web_tools_setup_cmd, disallow_web_tools_owned_paths=disallow_web_tools_owned_paths or [], disallow_web_tools_launch_suffix=disallow_web_tools_launch_suffix, + blocklist_web_tools_setup_cmd=blocklist_web_tools_setup_cmd, + blocklist_web_tools_launch_suffix=blocklist_web_tools_launch_suffix, ) AGENTS[name] = config AGENT_INSTALLERS[name] = install_cmd diff --git a/src/benchflow/cli/main.py b/src/benchflow/cli/main.py index 266e74ffa..e5da101e7 100644 --- a/src/benchflow/cli/main.py +++ b/src/benchflow/cli/main.py @@ -28,6 +28,7 @@ from benchflow import __version__ from benchflow._utils.config import normalize_sandbox_user +from benchflow._utils.config_override import blocklist_override from benchflow.agents.registry import parse_agent_spec from benchflow.cli._live_progress import ( LiveEvalProgress, @@ -469,6 +470,26 @@ def eval_run( list[str] | None, typer.Option("--agent-env", help="Agent env var (KEY=VALUE)"), ] = None, + block_url: Annotated[ + list[str] | None, + typer.Option( + "--block-url", + help=( + "Hide a host or host/path-prefix from the agent for this run " + "(repeatable; a pasted https:// URL is fine). Internet stays open " + "otherwise. Applied as a C-axis overlay that sets " + "sandbox.network_mode=blocklist and REPLACES task-level blocked_urls." + ), + ), + ] = None, + block_url_file: Annotated[ + Path | None, + typer.Option( + "--block-url-file", + help="File with one host or host/path-prefix per line ('#' comments ok); " + "merged with --block-url.", + ), + ] = None, include: Annotated[ list[str] | None, typer.Option( @@ -641,7 +662,7 @@ def eval_run( usage_tracking=usage_tracking, environment_manifest=environment_manifest, state=state, - config_override=config_override, + config_override=blocklist_override(config_override, block_url, block_url_file), prompt=prompt, concurrency=concurrency, build_concurrency=build_concurrency, diff --git a/src/benchflow/contracts/planes.py b/src/benchflow/contracts/planes.py index 839be0937..6637458ef 100644 --- a/src/benchflow/contracts/planes.py +++ b/src/benchflow/contracts/planes.py @@ -33,7 +33,9 @@ def live_usage_tokens(self) -> int | None: ... class RolloutPlanes(Protocol): """Concrete-plane operations the rollout kernel needs.""" - def agent_launch(self, agent: str, *, disallow_web_tools: bool) -> str: ... + def agent_launch( + self, agent: str, *, disallow_web_tools: bool, blocklist_web_tools: bool = False + ) -> str: ... def agent_config(self, agent: str) -> Any: ... def resolve_agent_env( diff --git a/src/benchflow/evaluation.py b/src/benchflow/evaluation.py index 47d5d466e..850c6f939 100644 --- a/src/benchflow/evaluation.py +++ b/src/benchflow/evaluation.py @@ -160,6 +160,15 @@ class EmptyTaskSelectionError(ValueError): """ +class NetworkPolicyPreflightError(ValueError): + """A selected task's network policy cannot be honored by this run. + + Raised before any rollout starts, so a 50-task batch does not burn 49 + tasks' worth of tokens before the one ``no-network`` task rejects the + ``--block-url`` overlay (or the backend refuses a blocklist). + """ + + class ResumeMismatchError(ValueError): """Raised when resuming a jobs_dir whose completed tasks ran a different agent. @@ -376,7 +385,97 @@ def skip_error(self) -> str: DEFAULT_JOB_MODE = "parallel-independent" -def _check_resume_mismatch(job_dir: Path, config: EvaluationConfig) -> None: +_NETWORK_POLICY_FIELDS = ( + "network_mode", + "blocked_urls", + "allowed_hosts", + "allow_internet", +) + + +def _network_policy_key(policy: dict | None) -> tuple | None: + """The part of a recorded ``network_policy`` that defines the run's posture.""" + if not policy: + return None + return (policy.get("mode"), tuple(policy.get("blocked_urls") or ())) + + +def _expected_network_policies( + task_dirs: list[Path], config: EvaluationConfig +) -> dict[str, dict | None]: + """Resolve, per selected task, the ``network_policy`` this run would bind. + + Applies the C-axis overlay to each task's declared config and asks the + runtime-capability gate about the network posture, BEFORE any rollout + starts. Network-policy conflicts (e.g. ``--block-url`` against a task that + declares ``no-network``/``allowlist``, or a blocklist task on a backend + that cannot enforce it) raise :class:`NetworkPolicyPreflightError` naming + every offending task. Non-network problems are left to the rollout, which + already reports them per task, so this preflight cannot widen the set of + jobs that refuse to start. + """ + from benchflow._utils.config_override import apply_config_override + from benchflow.sandbox.egress import EgressBlocklist + from benchflow.task import Task + from benchflow.task.runtime_capabilities import validate_task_runtime_support + + expected: dict[str, dict | None] = {} + problems: list[str] = [] + for task_dir in task_dirs: + try: + task = Task(task_dir) + except Exception: # malformed task: the batch loader already reported it + continue + try: + merged = apply_config_override(task.config, config.config_override) + except ValueError as exc: + if any(field in str(exc) for field in _NETWORK_POLICY_FIELDS): + problems.append(f"{task_dir.name}: {exc}") + continue + network_issues = [ + issue + for issue in validate_task_runtime_support( + merged, sandbox=config.environment + ) + if "network_mode" in issue.path + ] + if network_issues: + problems.append( + f"{task_dir.name}: " + + "; ".join(issue.reason for issue in network_issues) + ) + continue + # Mirror Rollout's precedence exactly: the oracle is exempt, and a + # no-web run (task allow_internet=false or --self-gen-no-internet) + # wins over a blocklist, recording network_policy=null (review #3). + disallow_web_tools = ( + getattr(merged.sandbox, "allow_internet", True) is False + or config.self_gen_no_internet + ) and config.agent != "oracle" + try: + blocklist = ( + None + if disallow_web_tools or config.agent == "oracle" + else EgressBlocklist.from_task_config(merged) + ) + except ValueError as exc: # agent-level mode shadows the blocklist + problems.append(f"{task_dir.name}: {exc}") + continue + expected[task_dir.name] = blocklist.config_metadata() if blocklist else None + if problems: + raise NetworkPolicyPreflightError( + "network policy preflight failed for " + f"{len(problems)} task(s); refusing to start the batch:\n " + + "\n ".join(problems) + ) + return expected + + +def _check_resume_mismatch( + job_dir: Path, + config: EvaluationConfig, + expected_network_policies: dict[str, dict | None] | None = None, +) -> None: """Guard against resuming a jobs_dir whose completed tasks ran differently. Reads one completed rollout's config.json (written by SDK.run) and @@ -388,7 +487,42 @@ def _check_resume_mismatch(job_dir: Path, config: EvaluationConfig) -> None: An *agent* mismatch raises :class:`ResumeMismatchError` (a blended score is meaningless and silently mixing one in is the bug this guards). A *loop_strategy* mismatch — same agent, different tuning — only warns. + + When ``expected_network_policies`` is given (task name -> the + ``network_policy`` block this run would record, ``None`` for an open + network), every completed rollout's recorded ``network_policy`` is compared + against it and a difference raises :class:`ResumeMismatchError`: scores + taken with and without an egress blocklist belong to different + experiments. Pre-feature config.json files (no key) count as open network. """ + if expected_network_policies is not None and job_dir.exists(): + for cfg_file in sorted(job_dir.rglob("config.json")): + try: + cfg = json.loads(cfg_file.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError, UnicodeDecodeError): + continue + # config.json records the provenance path when the task came from a + # source repo (e.g. "benchmarks/physics/task-1"); the expectation map + # is keyed by task directory name, so fall back to the basename. + task_name = str(cfg.get("task_path") or cfg_file.parent.name) + task_key = ( + task_name + if task_name in expected_network_policies + else Path(task_name).name + ) + if task_key not in expected_network_policies: + continue + prev_key = _network_policy_key(cfg.get("network_policy")) + current_key = _network_policy_key(expected_network_policies[task_key]) + if prev_key != current_key: + raise ResumeMismatchError( + f"refusing to resume: completed task {task_name!r} ran with " + f"network_policy={cfg.get('network_policy')}, but this run " + f"would use network_policy={expected_network_policies[task_key]}. " + "Scores taken with and without an egress blocklist belong to " + "different experiments. Use a fresh --jobs-dir (the existing " + "results are preserved)." + ) sample_dir = ( next((d for d in job_dir.iterdir() if d.is_dir()), None) if job_dir.exists() @@ -399,11 +533,11 @@ def _check_resume_mismatch(job_dir: Path, config: EvaluationConfig) -> None: if sample_dir: for cfg_file in sample_dir.rglob("config.json"): try: - cfg = json.loads(cfg_file.read_text()) + cfg = json.loads(cfg_file.read_text(encoding="utf-8")) prev_agent = cfg.get("agent", "") prev_loop = cfg.get("loop") or loop_block(None) break - except (json.JSONDecodeError, OSError): + except (json.JSONDecodeError, OSError, UnicodeDecodeError): logger.debug("Could not read %s", cfg_file) if prev_agent and prev_agent != config.agent: raise ResumeMismatchError( @@ -1752,6 +1886,9 @@ async def run(self) -> EvaluationResult: f"({', '.join(detail_parts)}). Refusing to publish an " "empty 0/0 summary." ) + # Network-policy preflight: resolve every selected task's posture under + # this run's overlay/backend BEFORE anything runs (review round 3). + expected_network_policies = _expected_network_policies(task_dirs, self._config) completed = self._get_completed_tasks() remaining = [d for d in task_dirs if d.name not in completed] @@ -1782,7 +1919,11 @@ async def run(self) -> EvaluationResult: # Warn if resuming with different config than completed tasks if completed: - _check_resume_mismatch(self._jobs_dir / self._job_name, self._config) + _check_resume_mismatch( + self._jobs_dir / self._job_name, + self._config, + expected_network_policies, + ) self._jobs_dir.mkdir(parents=True, exist_ok=True) self._prune_docker() diff --git a/src/benchflow/providers/litellm_logging.py b/src/benchflow/providers/litellm_logging.py index cb0c7864e..eb31550a7 100644 --- a/src/benchflow/providers/litellm_logging.py +++ b/src/benchflow/providers/litellm_logging.py @@ -222,6 +222,92 @@ def _failure_traceback(detail: Any) -> str: return tb[-2000:] + +#: Anthropic server tools carry a versioned ``type`` (web_search_20250305, +#: web_fetch_20250910, ...). They run at Anthropic, so the sandbox proxy never +#: sees their traffic; Anthropic's own ``blocked_domains`` filter (domains and +#: ``domain/path`` prefixes) is the only lever, injected below. +_ANTHROPIC_SERVER_WEB_TOOL_PREFIXES = ("web_search_20", "web_fetch_20") +#: OpenAI Responses hosted search tools support an ALLOWLIST filter only, so a +#: blocklist run strips them (fail closed) instead of half-filtering. +_OPENAI_HOSTED_SEARCH_TOOL_TYPES = frozenset( + { + "web_search", + "web_search_preview", + "web_search_preview_2025_03_11", + "web_search_2025_08_26", + } +) + + +def _is_anthropic_server_web_tool(tool: dict) -> bool: + tool_type = str(tool.get("type") or "") + return tool_type.startswith(_ANTHROPIC_SERVER_WEB_TOOL_PREFIXES) + + +def _egress_blocked_domains() -> list[str]: + raw = os.environ.get("BENCHFLOW_EGRESS_BLOCKED_URLS", "") + if not raw: + return [] + try: + rules = json.loads(raw) + except ValueError: + return [] + return [r for r in rules if isinstance(r, str) and r] + + +def _apply_egress_blocklist_to_tools(data: dict) -> dict: + # Rewrite server-side web tools under the egress blocklist. Anthropic + # web_search_* / web_fetch_* tools get the rule list merged into + # blocked_domains (an allowed_domains filter, which Anthropic forbids + # alongside blocked_domains, is narrowed instead by dropping the blocked + # entries). OpenAI hosted search tools are removed. Returns the SAME object + # when nothing applies so callers can detect "unchanged" by identity. + blocked = _egress_blocked_domains() + tools = data.get("tools") + if not blocked or not isinstance(tools, list): + return data + changed = False + new_tools: list = [] + for tool in tools: + if not isinstance(tool, dict): + new_tools.append(tool) + continue + tool_type = str(tool.get("type") or "") + if _is_anthropic_server_web_tool(tool): + updated = dict(tool) + if isinstance(updated.get("allowed_domains"), list): + allowed = [ + d + for d in updated["allowed_domains"] + if not any( + str(d).lower() == b or str(d).lower().endswith("." + b) + for b in (r.split("/", 1)[0] for r in blocked) + ) + ] + if allowed != updated["allowed_domains"]: + updated["allowed_domains"] = allowed + changed = True + else: + existing = [ + str(d) for d in (updated.get("blocked_domains") or []) if d + ] + merged = existing + [b for b in blocked if b not in existing] + if merged != existing: + updated["blocked_domains"] = merged + changed = True + new_tools.append(updated) + elif tool_type in _OPENAI_HOSTED_SEARCH_TOOL_TYPES: + changed = True # dropped + else: + new_tools.append(tool) + if not changed: + return data + rewritten = dict(data) + rewritten["tools"] = new_tools + return rewritten + + class BenchFlowLiteLLMLogger(CustomLogger): def _write(self, payload: dict[str, Any]) -> None: path = os.environ.get("BENCHFLOW_LITELLM_LOG_PATH") @@ -278,7 +364,7 @@ def _base_record(self, kwargs: dict[str, Any], start_time: Any, end_time: Any) - "duration_ms": max((getattr(end_time, "timestamp", lambda: time.time())() - getattr(start_time, "timestamp", lambda: time.time())()) * 1000, 0), } - async def async_pre_call_hook( + async def async_pre_call_hook( # noqa: C901 — one linear rewrite pipeline self, user_api_key_dict, cache, data, call_type ): if not isinstance(data, dict): @@ -295,6 +381,12 @@ async def async_pre_call_hook( cleaned = dict(cleaned) cleaned.pop("input", None) + # Egress blocklist: server-side web tools run at the provider, outside + # the sandbox proxy's reach, so filter/strip them here. + rewritten = _apply_egress_blocklist_to_tools(cleaned) + if rewritten is not cleaned: + cleaned = rewritten + # Drop non-"function" tools before they reach a chat-only backend. A # responses-API client (codex) sends tools the Responses wire allows but # chat completions does not, e.g. a {"type": "namespace"} tool. When the @@ -305,10 +397,17 @@ async def async_pre_call_hook( # (shell, file IO, ...) survive untouched. tools = cleaned.get("tools") if isinstance(tools, list): + # Anthropic server web tools survive ONLY under an active egress + # blocklist (they were just given blocked_domains above). In every + # other mode — including the pure no-web policy — they are dropped + # exactly as before. + keep_anthropic_web = bool(_egress_blocked_domains()) kept = [ t for t in tools - if not isinstance(t, dict) or t.get("type", "function") == "function" + if not isinstance(t, dict) + or t.get("type", "function") == "function" + or (keep_anthropic_web and _is_anthropic_server_web_tool(t)) ] if len(kept) != len(tools): if cleaned is data: diff --git a/src/benchflow/providers/litellm_runtime.py b/src/benchflow/providers/litellm_runtime.py index 60c9953c3..787919aca 100644 --- a/src/benchflow/providers/litellm_runtime.py +++ b/src/benchflow/providers/litellm_runtime.py @@ -50,6 +50,7 @@ extract_usage_from_trajectory, trajectory_from_litellm_callback_log, ) +from benchflow.sandbox.egress import strip_proxy_env from benchflow.sandbox.providers import SANDBOX_MODEL_PROXY_PROVIDERS from benchflow.trajectories._llm_capture import LiveLLMTrajectoryWriter from benchflow.trajectories.types import Trajectory @@ -1451,7 +1452,11 @@ def _apply_litellm_agent_env( def _litellm_proxy_env( *, agent: str, agent_env: dict[str, str], required_skill_names: tuple[str, ...] ) -> dict[str, str]: - updated = dict(agent_env) + # The model proxy is a root-run helper that must reach providers + # directly: never route it through the agent's egress filter (nor trust + # the agent-side CA bundle). The blocklist rule list itself stays — the + # pre-call hook reads it to filter server-side web tools. + updated = strip_proxy_env(dict(agent_env)) updated.pop(_SKILL_CATALOG_GATE_AGENT_ENV, None) updated.pop(_REQUIRED_SKILL_NAMES_ENV, None) expected = sorted(set(required_skill_names)) diff --git a/src/benchflow/rollout/__init__.py b/src/benchflow/rollout/__init__.py index f7f6695ca..b35ad419f 100644 --- a/src/benchflow/rollout/__init__.py +++ b/src/benchflow/rollout/__init__.py @@ -142,14 +142,29 @@ from benchflow.rollout._setup import ( _publish_trajectory_for_verifier as _publish_trajectory_for_verifier, ) +from benchflow.rollout._setup import ( + _refuse_session_factory_under_blocklist as _refuse_session_factory_under_blocklist, +) from benchflow.rollout._setup import _resolve_agent_cwd as _resolve_agent_cwd +from benchflow.rollout._setup import ( + _resolve_agent_network_policy as _resolve_agent_network_policy, +) from benchflow.rollout._setup import _resolve_prompts as _resolve_prompts from benchflow.rollout._setup import _run_oracle as _run_oracle from benchflow.rollout._setup import _start_env_and_upload as _start_env_and_upload from benchflow.rollout._setup import ( _task_disallows_internet as _task_disallows_internet, ) +from benchflow.rollout._setup import ( + _task_egress_blocklist as _task_egress_blocklist, +) from benchflow.rollout._setup import _verify_rollout as _verify_rollout +from benchflow.rollout._setup import ( + _web_policy_apply_kwargs as _web_policy_apply_kwargs, +) +from benchflow.rollout._setup import ( + _web_policy_launch_kwargs as _web_policy_launch_kwargs, +) from benchflow.rollout._skills import ( _resolve_skill_creator_root as _resolve_skill_creator_root, ) @@ -197,6 +212,11 @@ from benchflow.rollout.task_runtime import TaskRuntimeResult as TaskRuntimeResult from benchflow.rollout_branch import ChildRunner from benchflow.rollout_branch import branch as _branch_engine +from benchflow.sandbox.egress import ( + EgressBlocklist, + download_egress_log, + start_egress_proxy, +) from benchflow.sandbox.metadata import persist_sandbox_info from benchflow.scenes import compile_scenes_to_steps from benchflow.scenes import scene_step_prompt as scene_step_prompt @@ -952,11 +972,26 @@ async def setup(self) -> None: self._disallow_web_tools = ( _task_disallows_internet(self._task) or cfg.self_gen_no_internet ) and cfg.primary_agent != "oracle" + # Egress blocklist (network_mode='blocklist'): the inverse of the + # no-web policy — internet stays open except for the declared URLs. + # The oracle is exempt like it is from no-web; a no-web run wins. + # Either policy keeps the container online for the sandbox-local model + # proxy and confines the agent UID to loopback instead. The container + # provisioning follows the TASK (an oracle primary must not switch it + # off for the role agents that connect later — review P0 #3). + self._egress_blocklist, self._agent_network_policy = ( + _resolve_agent_network_policy( + self._task, + primary_agent=cfg.primary_agent, + disallow_web_tools=self._disallow_web_tools, + ) + ) self._agent_env = _apply_web_policy( self._planes.resolve_agent_env( cfg.primary_agent, cfg.primary_model, cfg.agent_env ), disallow=self._disallow_web_tools, + blocklist=self._egress_blocklist, ) env_config = getattr(getattr(self._task, "config", None), "sandbox", None) task_skill_policy = resolve_task_skill_policy( @@ -972,7 +1007,10 @@ async def setup(self) -> None: ) self._agent_launch = self._planes.agent_launch( cfg.primary_agent, - disallow_web_tools=self._disallow_web_tools, + **_web_policy_launch_kwargs( + disallow=self._disallow_web_tools, + blocklist=self._egress_blocklist is not None, + ), ) # Copy task dir to temp when Dockerfile mutations are needed @@ -1041,7 +1079,7 @@ async def setup(self) -> None: effective_task_path, self._rollout_name, self._rollout_paths, - preserve_agent_network=self._disallow_web_tools, + preserve_agent_network=self._agent_network_policy, environment_manifest=cfg.environment_manifest, ) # Caller-supplied wall-clock budget (e.g. RuntimeConfig.timeout) @@ -1086,6 +1124,12 @@ async def setup(self) -> None: task_digest=cfg.task_digest, config_override=cfg.config_override, loop_strategy=cfg.loop_strategy_spec, + network_policy=( + egress_blocklist.config_metadata() + if (egress_blocklist := getattr(self, "_egress_blocklist", None)) + is not None + else None + ), ) self._phase = "setup" @@ -1230,8 +1274,18 @@ async def install_agent(self) -> None: agent_name, self._agent_cfg, cred_home, - disallow=self._disallow_web_tools, + **_web_policy_apply_kwargs( + disallow=self._disallow_web_tools, + blocklist=getattr(self, "_egress_blocklist", None) is not None, + ), ) + # Egress blocklist: the loopback filtering proxy is part of the + # sandbox setup, not of any particular connect protocol — it must be + # listening before whichever agent process inherits HTTP(S)_PROXY. + egress_blocklist = getattr(self, "_egress_blocklist", None) + if egress_blocklist is not None: + await start_egress_proxy(self._env, egress_blocklist) + self._egress_proxy_started = True await self._planes.snapshot_build_config(self._env, workspace=self._agent_cwd) await self._planes.seed_verifier_workspace( self._env, workspace=self._agent_cwd, sandbox_user=cfg.sandbox_user @@ -1295,9 +1349,12 @@ async def connect(self) -> None: sandbox_setup_timeout=cfg.sandbox_setup_timeout, required_skill_names=getattr(self, "_required_skill_names", ()), live_trajectory_path=rollout_dir / "trajectory" / "llm_trajectory.jsonl", - force_sandbox_local=getattr(self, "_disallow_web_tools", False), + force_sandbox_local=getattr(self, "_agent_network_policy", False), ) sf_entrypoint = self._session_factory_entrypoint(cfg.primary_agent) + _refuse_session_factory_under_blocklist( + cfg.primary_agent, sf_entrypoint, getattr(self, "_egress_blocklist", None) + ) self._is_session_factory = sf_entrypoint is not None if sf_entrypoint is not None: ( @@ -1391,6 +1448,15 @@ async def disconnect(self) -> None: f"pkill -f {shlex.quote(agent_pattern)} || true", timeout_sec=10, ) + # The audit log exists whenever the proxy ran for ANY role in this + # sandbox — an oracle primary's env carries no blocklist marker, so + # the gate is the proxy-started flag, not self._agent_env (review #1). + if ( + getattr(self, "_egress_proxy_started", False) + and self._env is not None + and getattr(self, "_rollout_paths", None) is not None + ): + await download_egress_log(self._env, self._rollout_paths.agent_dir) self._active_role = None self._session_tool_count = 0 self._session_traj_count = 0 @@ -2272,9 +2338,18 @@ async def connect_as(self, role: Role) -> None: if disallow_web_tools is None: disallow_web_tools = _task_disallows_internet(getattr(self, "_task", None)) disallow_web_tools = bool(disallow_web_tools and role.agent != "oracle") + egress_blocklist = ( + None + if disallow_web_tools or role.agent == "oracle" + else _task_egress_blocklist(getattr(self, "_task", None)) + ) + agent_network_policy = disallow_web_tools or egress_blocklist is not None agent_launch = self._planes.agent_launch( role.agent, - disallow_web_tools=disallow_web_tools, + **_web_policy_launch_kwargs( + disallow=disallow_web_tools, + blocklist=egress_blocklist is not None, + ), ) agent_env = _apply_web_policy( self._planes.resolve_agent_env( @@ -2283,6 +2358,7 @@ async def connect_as(self, role: Role) -> None: {**(cfg.agent_env or {}), **(role.env or {})}, ), disallow=disallow_web_tools, + blocklist=egress_blocklist, ) agent_env, self._usage_runtime = await self._planes.ensure_litellm_runtime( agent=role.agent, @@ -2296,7 +2372,7 @@ async def connect_as(self, role: Role) -> None: sandbox_setup_timeout=cfg.sandbox_setup_timeout, required_skill_names=getattr(self, "_required_skill_names", ()), live_trajectory_path=rollout_dir / "trajectory" / "llm_trajectory.jsonl", - force_sandbox_local=disallow_web_tools, + force_sandbox_local=agent_network_policy, ) role_agent_differs = role.agent != cfg.primary_agent @@ -2341,12 +2417,23 @@ async def connect_as(self, role: Role) -> None: role.agent, agent_cfg, cred_home, - disallow=disallow_web_tools, + **_web_policy_apply_kwargs( + disallow=disallow_web_tools, + blocklist=egress_blocklist is not None, + ), + ) + if egress_blocklist is not None: + await start_egress_proxy( + self._env, EgressBlocklist.from_env(agent_env) or egress_blocklist ) + self._egress_proxy_started = True self._agent_launch = agent_launch sf_entrypoint = self._session_factory_entrypoint(role.agent) + _refuse_session_factory_under_blocklist( + role.agent, sf_entrypoint, egress_blocklist + ) self._is_session_factory = sf_entrypoint is not None if sf_entrypoint is not None: ( diff --git a/src/benchflow/rollout/_results.py b/src/benchflow/rollout/_results.py index 588e5cbd6..a60dd1dc0 100644 --- a/src/benchflow/rollout/_results.py +++ b/src/benchflow/rollout/_results.py @@ -161,6 +161,7 @@ def _write_config( environment_manifest: EnvironmentManifest | None = None, config_override: dict | None = None, loop_strategy: LoopStrategySpec | None = None, + network_policy: dict[str, Any] | None = None, ) -> None: """Write config.json to rollout_dir with secrets filtered out.""" from benchflow.acp.selection import selected_acp_transport @@ -203,6 +204,10 @@ def _write_config( "agent_env": recorded_env, "scenes": _scene_metadata(scenes or []), "loop": loop_block(loop_strategy), + # Agent-phase egress policy actually bound for this run (None when the + # task/run declared no blocklist). Auditors read this — plus the + # downloaded agent/egress.jsonl — to confirm the hidden URLs stayed hidden. + "network_policy": network_policy, } if usage_tracking is not None: config_data["usage_tracking"] = usage_tracking.to_config_artifact() diff --git a/src/benchflow/rollout/_setup.py b/src/benchflow/rollout/_setup.py index 33c95c7e8..d5dd9e517 100644 --- a/src/benchflow/rollout/_setup.py +++ b/src/benchflow/rollout/_setup.py @@ -42,6 +42,7 @@ validate_reward_map, ) from benchflow.rollout._results import _DIAG_TRUNCATE +from benchflow.sandbox.egress import EgressBlocklist, apply_blocklist_env from benchflow.trajectories.types import redact_acp_trajectory_jsonl logger = logging.getLogger(__name__) @@ -82,19 +83,104 @@ def _environment_uses_prebuilt_image( return bool(resolve_manifest_image(environment_manifest)) -def _apply_web_policy(agent_env: dict[str, str], *, disallow: bool) -> dict[str, str]: - """Inject BenchFlow's no-web policy marker into agent env when requested.""" - if not disallow: - return agent_env - return {**agent_env, _DISALLOW_WEB_TOOLS_ENV: "1"} +def _task_egress_blocklist(task: Any) -> EgressBlocklist | None: + """Return the agent-phase egress blocklist declared by the task, if any.""" + config = getattr(task, "config", None) + if config is None: + return None + return EgressBlocklist.from_task_config(config) + + +def _resolve_agent_network_policy( + task: Any, *, primary_agent: str, disallow_web_tools: bool +) -> tuple[EgressBlocklist | None, bool]: + """Return ``(primary_agent_blocklist, container_policy_active)``. + + The two answer different questions and must not be conflated: + + * the first is the blocklist the PRIMARY agent process is routed through — + ``None`` for the oracle (exempt like no-web) and under a no-web run + (which wins); + * the second is whether the CONTAINER must be provisioned for an agent- + layer network policy (docker ``NET_ADMIN`` overlay, sandbox-local model + proxy). It follows the TASK: a task that declares a blocklist needs the + provisioning even when the primary is the oracle, because a later + ``connect_as(role)`` for the agent under test programs iptables in that + same container. + """ + task_blocklist = _task_egress_blocklist(task) + primary_blocklist = ( + None if disallow_web_tools or primary_agent == "oracle" else task_blocklist + ) + policy_active = disallow_web_tools or task_blocklist is not None + return primary_blocklist, policy_active + + +def _refuse_session_factory_under_blocklist( + agent: str, session_factory_entrypoint: str | None, blocklist: Any +) -> None: + """A session-factory agent runs in-process on the HOST: no sandbox proxy + and no agent-UID firewall can cover it, so a blocklist run must refuse it + rather than run open.""" + if session_factory_entrypoint is None or blocklist is None: + return + raise RuntimeError( + f"network_mode='blocklist' cannot be enforced for session-factory agent " + f"{agent!r} ({session_factory_entrypoint}): it runs on the host, outside " + "the sandbox egress proxy and agent-UID firewall. Use an ACP agent or " + "drop the blocklist." + ) + + +def _apply_web_policy( + agent_env: dict[str, str], + *, + disallow: bool, + blocklist: EgressBlocklist | None = None, +) -> dict[str, str]: + """Inject BenchFlow's agent network policy markers into agent env. + + ``disallow`` marks the no-web policy; ``blocklist`` routes the agent through + the sandbox egress proxy and carries the rule list for the model proxy. + """ + updated = agent_env + if disallow: + updated = {**updated, _DISALLOW_WEB_TOOLS_ENV: "1"} + return apply_blocklist_env(updated, blocklist) + + +def _web_policy_launch_kwargs(*, disallow: bool, blocklist: bool) -> dict[str, bool]: + """Keyword args for ``RolloutPlanes.agent_launch``. + + ``blocklist_web_tools`` is only passed when a blocklist is active, so plane + implementations (and test fakes) written against the older + ``agent_launch(agent, *, disallow_web_tools)`` shape keep working for runs + that never use the blocklist. + """ + kwargs: dict[str, bool] = {"disallow_web_tools": disallow} + if blocklist: + kwargs["blocklist_web_tools"] = True + return kwargs + + +def _web_policy_apply_kwargs(*, disallow: bool, blocklist: bool) -> dict[str, bool]: + """Keyword args for ``RolloutPlanes.apply_web_tool_policy`` (same rule).""" + kwargs: dict[str, bool] = {"disallow": disallow} + if blocklist: + kwargs["blocklist"] = True + return kwargs def _agent_launch_with_web_policy( - agent: str, *, disallow: bool, planes: RolloutPlanes | None = None + agent: str, + *, + disallow: bool, + blocklist: bool = False, + planes: RolloutPlanes | None = None, ) -> str: - """Return launch command, appending the agent's no-web launch knob if any.""" + """Return launch command, appending the agent's web-policy launch knob if any.""" return (planes or default_rollout_planes()).agent_launch( - agent, disallow_web_tools=disallow + agent, **_web_policy_launch_kwargs(disallow=disallow, blocklist=blocklist) ) diff --git a/src/benchflow/rollout_planes.py b/src/benchflow/rollout_planes.py index 6d2272d9a..121f67714 100644 --- a/src/benchflow/rollout_planes.py +++ b/src/benchflow/rollout_planes.py @@ -53,13 +53,25 @@ class DefaultRolloutPlanes: """Default bindings for the four concrete planes.""" - def agent_launch(self, agent: str, *, disallow_web_tools: bool) -> str: + def agent_launch( + self, + agent: str, + *, + disallow_web_tools: bool, + blocklist_web_tools: bool = False, + ) -> str: launch = AGENT_LAUNCH.get(agent, agent) - if not disallow_web_tools: - return launch agent_cfg = AGENTS.get(agent) - if agent_cfg and agent_cfg.disallow_web_tools_launch_suffix: - return launch + agent_cfg.disallow_web_tools_launch_suffix + if disallow_web_tools: + if agent_cfg and agent_cfg.disallow_web_tools_launch_suffix: + return launch + agent_cfg.disallow_web_tools_launch_suffix + return launch + if ( + blocklist_web_tools + and agent_cfg + and agent_cfg.blocklist_web_tools_launch_suffix + ): + return launch + agent_cfg.blocklist_web_tools_launch_suffix return launch def agent_config(self, agent: str) -> Any: diff --git a/src/benchflow/sandbox/_compose.py b/src/benchflow/sandbox/_compose.py index 2217aacff..02d1def06 100644 --- a/src/benchflow/sandbox/_compose.py +++ b/src/benchflow/sandbox/_compose.py @@ -9,6 +9,7 @@ COMPOSE_BUILD_PATH = COMPOSE_DIR / "docker-compose-build.yaml" COMPOSE_PREBUILT_PATH = COMPOSE_DIR / "docker-compose-prebuilt.yaml" COMPOSE_NO_NETWORK_PATH = COMPOSE_DIR / "docker-compose-no-network.yaml" +COMPOSE_NET_ADMIN_PATH = COMPOSE_DIR / "docker-compose-net-admin.yaml" # Back-off delays for retrying a `compose up` that hit a daemon-side network # create/attach race. Shared by the host docker.py path and the Daytona DinD diff --git a/src/benchflow/sandbox/_compose_files/docker-compose-net-admin.yaml b/src/benchflow/sandbox/_compose_files/docker-compose-net-admin.yaml new file mode 100644 index 000000000..09a8945a6 --- /dev/null +++ b/src/benchflow/sandbox/_compose_files/docker-compose-net-admin.yaml @@ -0,0 +1,6 @@ +# Agent-layer network policies (no-web, egress blocklist) program iptables +# inside the container, which needs NET_ADMIN. Stacked only for those runs. +services: + main: + cap_add: + - NET_ADMIN diff --git a/src/benchflow/sandbox/docker.py b/src/benchflow/sandbox/docker.py index 4ae9f362d..d1d59eae3 100644 --- a/src/benchflow/sandbox/docker.py +++ b/src/benchflow/sandbox/docker.py @@ -31,6 +31,7 @@ from benchflow.sandbox._compose import ( COMPOSE_BASE_PATH, COMPOSE_BUILD_PATH, + COMPOSE_NET_ADMIN_PATH, COMPOSE_NO_NETWORK_PATH, COMPOSE_PREBUILT_PATH, COMPOSE_UP_RETRY_DELAYS_SEC, @@ -114,6 +115,7 @@ class DockerSandbox(BaseSandbox): _DOCKER_COMPOSE_BUILD_PATH = COMPOSE_BUILD_PATH _DOCKER_COMPOSE_PREBUILT_PATH = COMPOSE_PREBUILT_PATH _DOCKER_COMPOSE_NO_NETWORK_PATH = COMPOSE_NO_NETWORK_PATH + _DOCKER_COMPOSE_NET_ADMIN_PATH = COMPOSE_NET_ADMIN_PATH _image_build_locks: ClassVar[dict[str, asyncio.Lock]] = {} _build_semaphore: ClassVar[asyncio.Semaphore | None] = None @@ -158,6 +160,7 @@ def __init__( task_env_config: SandboxConfig, keep_containers: bool = False, mounts_json: list[dict[str, str]] | None = None, + agent_network_policy: bool = False, *args: Any, **kwargs: Any, ) -> None: @@ -172,6 +175,9 @@ def __init__( self._keep_containers = keep_containers self._mounts_json = mounts_json + # An agent-layer network policy (no-web / egress blocklist) programs + # iptables inside the container, which needs NET_ADMIN. + self._agent_network_policy = agent_network_policy self._mounts_compose_path: Path | None = None self._logs_are_mounted = True @@ -300,6 +306,9 @@ def _docker_compose_paths(self) -> list[Path]: if not self.task_env_config.allow_internet: paths.append(self._DOCKER_COMPOSE_NO_NETWORK_PATH) + if self._agent_network_policy: + paths.append(self._DOCKER_COMPOSE_NET_ADMIN_PATH) + return paths def _docker_compose_env(self) -> dict[str, str]: diff --git a/src/benchflow/sandbox/egress.py b/src/benchflow/sandbox/egress.py new file mode 100644 index 000000000..c2e842dd4 --- /dev/null +++ b/src/benchflow/sandbox/egress.py @@ -0,0 +1,1159 @@ +"""Agent egress blocklist — ``network_mode = "blocklist"`` enforcement. + +The task keeps full internet access, but a declared list of ``host`` / +``host/path-prefix`` entries must stay unreachable from the agent. Enforcement +is layered because a single mechanism cannot see every path an agent has to +the web: + +1. **Sandbox-local filtering proxy** (this module's ``_EGRESS_PROXY_SOURCE``). + A stdlib-only forward proxy started as root on loopback before the agent + launches. Plain HTTP requests carry the full URL, so host *and* path rules + apply. HTTPS ``CONNECT`` tunnels only expose the hostname, so a host rule + rejects the tunnel outright; a host that carries *path* rules is TLS- + inspected instead — the proxy terminates TLS with a leaf certificate signed + by a per-run CA that is installed into the sandbox trust store, reads the + request path, and forwards allowed requests upstream over a fresh TLS + connection. The agent user never sees the CA private key. +2. **Agent-UID firewall** (``lockdown._agent_egress_firewall_cmd``). After the + ACP handshake the agent's UID may only reach loopback, so every tool that + ignores ``HTTP(S)_PROXY`` fails closed instead of bypassing the filter. +3. **Server-side web tools** live outside the sandbox (Claude Code WebSearch, + Codex ``web_search``, Gemini grounding). The LiteLLM pre-call hook injects + Anthropic ``blocked_domains`` and strips OpenAI Responses search tools; the + per-agent ``blocklist_web_tools_*`` registry knobs disable the rest. + +Blocked requests answer ``404 Not Found`` rather than ``403`` so that a +research agent cannot distinguish a hidden paper from a missing one. Every +decision is appended to an egress log the rollout downloads for auditing. + +Because the proxy runs as root (outside the agent-UID firewall) it vets every +resolved upstream address and refuses loopback, link-local (cloud metadata), +unspecified, multicast, reserved, and private ranges with ``403`` — it must +never be the hop that turns the blocklist into an SSRF primitive. The only +private addresses reachable are the container's own directly-connected +subnets (read from ``/proc/net/route``), which is where compose side-services +live; ``allow_private_networks`` re-opens every private range for tasks that +genuinely need it. +""" + +from __future__ import annotations + +import asyncio +import json +import logging +import posixpath +import shlex +import tempfile +import urllib.parse +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +logger = logging.getLogger(__name__) + +#: Agent-env marker carrying the JSON rule list. Present in the LiteLLM proxy +#: process env (it drives the server-side tool rewrite) and stripped from the +#: agent's own env so the agent cannot read which URLs are hidden from it. +EGRESS_BLOCKED_URLS_ENV = "BENCHFLOW_EGRESS_BLOCKED_URLS" +#: Agent-env marker that the blocklist policy is active (safe to expose). +EGRESS_POLICY_ENV = "BENCHFLOW_EGRESS_POLICY" +EGRESS_POLICY_BLOCKLIST = "blocklist" + +DEFAULT_EGRESS_PROXY_PORT = 61380 +EGRESS_RUNTIME_DIR = "/opt/benchflow/egress" +EGRESS_CA_BUNDLE_PATH = f"{EGRESS_RUNTIME_DIR}/ca-bundle.crt" +EGRESS_LOG_PATH = f"{EGRESS_RUNTIME_DIR}/egress.jsonl" +EGRESS_LOG_ARTIFACT_NAME = "egress.jsonl" +#: Status returned for blocked requests — indistinguishable from "no such page". +BLOCKED_STATUS = 404 + +_PROXY_ENV_NAMES = ( + "HTTP_PROXY", + "HTTPS_PROXY", + "http_proxy", + "https_proxy", + "NO_PROXY", + "no_proxy", + "NODE_USE_ENV_PROXY", + "SSL_CERT_FILE", + "REQUESTS_CA_BUNDLE", + "CURL_CA_BUNDLE", + "NODE_EXTRA_CA_CERTS", + "GIT_SSL_CAINFO", +) + + +def _split_rule(rule: str) -> tuple[str, str]: + host, _, path = rule.partition("/") + return host.lower(), path.strip("/") + + +def _host_matches(rule_host: str, host: str) -> bool: + host = host.lower().rstrip(".") + return host == rule_host or host.endswith("." + rule_host) + + +def normalize_request_path(path: str) -> str: + """Canonical form of a request path for rule matching. + + Mirrors what an upstream HTTP server does before routing, so an agent + cannot dodge a rule with ``%32%34...`` percent-encoding, ``//`` runs, or + ``/other/../abs/…`` dot segments: drop the query/fragment, percent-decode, + collapse with ``posixpath.normpath``, strip slashes, lower-case. + """ + raw = path.split("?", 1)[0].split("#", 1)[0] + decoded = _unquote_fully(raw) + collapsed = posixpath.normpath("/" + decoded).strip("/") + return "" if collapsed == "." else collapsed.lower() + + +def _unquote_fully(value: str, *, rounds: int = 4) -> str: + """Percent-decode until stable (bounded) so a double-encoded separator + (``%252f`` → ``%2f`` → ``/``) cannot reach an upstream that decodes twice + while the proxy judged the single-decoded form.""" + for _ in range(rounds): + decoded = urllib.parse.unquote(value) + if decoded == value: + return value + value = decoded + return value + + +def match_blocked( + rules: tuple[str, ...] | list[str], host: str, path: str +) -> str | None: + """Return the first rule that blocks ``host`` + ``path``, else ``None``. + + Host rules match the host and its subdomains. Path rules additionally + require the request path to start with the rule's path prefix (the + ``host/path-prefix`` contract from ``blocked_urls``). ``path`` is the + request target without scheme/host; the query string is ignored. + """ + clean_path = normalize_request_path(path) + for rule in rules: + rule_host, rule_path = _split_rule(rule) + rule_path = rule_path.lower() + if not _host_matches(rule_host, host): + continue + if not rule_path: + return rule + if clean_path == rule_path or clean_path.startswith(rule_path): + return rule + return None + + +@dataclass(frozen=True) +class EgressBlocklist: + """Resolved blocklist for one rollout (already schema-normalized rules).""" + + rules: tuple[str, ...] + proxy_port: int = DEFAULT_EGRESS_PROXY_PORT + + def __post_init__(self) -> None: + if not self.rules: + raise ValueError("EgressBlocklist requires at least one rule") + + @property + def tls_inspection_hosts(self) -> tuple[str, ...]: + """Hosts that carry path rules and therefore need TLS inspection.""" + hosts: list[str] = [] + for rule in self.rules: + host, path = _split_rule(rule) + if path and host not in hosts: + hosts.append(host) + return tuple(hosts) + + @property + def needs_tls_inspection(self) -> bool: + return bool(self.tls_inspection_hosts) + + @property + def proxy_url(self) -> str: + return f"http://127.0.0.1:{self.proxy_port}" + + def to_env_value(self) -> str: + return json.dumps(list(self.rules)) + + @classmethod + def from_env(cls, env: dict[str, str] | None) -> EgressBlocklist | None: + raw = (env or {}).get(EGRESS_BLOCKED_URLS_ENV, "") + if not raw: + return None + rules = json.loads(raw) + if not isinstance(rules, list) or not all(isinstance(r, str) for r in rules): + raise ValueError( + f"{EGRESS_BLOCKED_URLS_ENV} must be a JSON list of strings" + ) + port = int( + (env or {}).get("BENCHFLOW_EGRESS_PROXY_PORT", DEFAULT_EGRESS_PROXY_PORT) + ) + return cls(rules=tuple(rules), proxy_port=port) + + @classmethod + def from_task_config(cls, config: Any) -> EgressBlocklist | None: + """Resolve the agent-phase blocklist from a parsed ``TaskConfig``. + + The ``agent`` section overrides ``sandbox`` when it declares its own + ``network_mode``; otherwise the sandbox policy applies to the agent. + """ + from benchflow.task.config import NetworkMode + + agent = getattr(config, "agent", None) + sandbox = getattr(config, "sandbox", None) + agent_mode = getattr(agent, "network_mode", None) if agent is not None else None + sandbox_mode = ( + getattr(sandbox, "network_mode", None) if sandbox is not None else None + ) + if agent_mode is not None: + section = agent + if ( + sandbox_mode == NetworkMode.BLOCKLIST + and agent_mode != NetworkMode.BLOCKLIST + ): + # A run-level --block-url lands in the sandbox section; an + # explicit agent-level mode would silently win and the hidden + # URLs would be served. Refuse instead of running open. + raise ValueError( + "sandbox.network_mode='blocklist' is overridden by the " + f"agent-level network_mode={agent_mode.value!r}; the " + "blocklist would not apply to the agent. Drop the agent " + "override or set agent.network_mode='blocklist' too." + ) + else: + section = sandbox + if section is None or getattr(section, "network_mode", None) != ( + NetworkMode.BLOCKLIST + ): + return None + rules = tuple(getattr(section, "blocked_urls", None) or ()) + if not rules: + return None + return cls(rules=rules) + + def agent_env(self) -> dict[str, str]: + """Env the *agent* process needs: route through the proxy, trust the CA. + + Deliberately excludes the rule list itself. + """ + env = { + EGRESS_POLICY_ENV: EGRESS_POLICY_BLOCKLIST, + "BENCHFLOW_EGRESS_PROXY_PORT": str(self.proxy_port), + "HTTP_PROXY": self.proxy_url, + "HTTPS_PROXY": self.proxy_url, + "http_proxy": self.proxy_url, + "https_proxy": self.proxy_url, + "NO_PROXY": "127.0.0.1,localhost,::1", + "no_proxy": "127.0.0.1,localhost,::1", + # Node's global fetch (undici) ignores proxy env unless told to. + "NODE_USE_ENV_PROXY": "1", + } + if self.needs_tls_inspection: + env.update( + { + "SSL_CERT_FILE": EGRESS_CA_BUNDLE_PATH, + "REQUESTS_CA_BUNDLE": EGRESS_CA_BUNDLE_PATH, + "CURL_CA_BUNDLE": EGRESS_CA_BUNDLE_PATH, + "NODE_EXTRA_CA_CERTS": EGRESS_CA_BUNDLE_PATH, + "GIT_SSL_CAINFO": EGRESS_CA_BUNDLE_PATH, + } + ) + return env + + def config_metadata(self) -> dict[str, Any]: + """Block recorded in the rollout's ``config.json``.""" + return { + "mode": EGRESS_POLICY_BLOCKLIST, + "blocked_urls": list(self.rules), + "tls_inspection_hosts": list(self.tls_inspection_hosts), + "blocked_status": BLOCKED_STATUS, + "proxy_port": self.proxy_port, + } + + +def apply_blocklist_env( + agent_env: dict[str, str], blocklist: EgressBlocklist | None +) -> dict[str, str]: + """Return ``agent_env`` with the blocklist policy applied (or unchanged).""" + if blocklist is None: + return agent_env + return { + **agent_env, + **blocklist.agent_env(), + EGRESS_BLOCKED_URLS_ENV: blocklist.to_env_value(), + } + + +def strip_blocklist_secret(agent_env: dict[str, str]) -> dict[str, str]: + """Drop the rule list before the env reaches the agent process.""" + if EGRESS_BLOCKED_URLS_ENV not in agent_env: + return agent_env + return {k: v for k, v in agent_env.items() if k != EGRESS_BLOCKED_URLS_ENV} + + +def strip_proxy_env(env: dict[str, str]) -> dict[str, str]: + """Drop proxy/CA routing vars — for root-run helpers (LiteLLM) that must + reach providers directly rather than through the agent's filter.""" + return {k: v for k, v in env.items() if k not in _PROXY_ENV_NAMES} + + +def blocklist_active(agent_env: dict[str, str] | None) -> bool: + return bool((agent_env or {}).get(EGRESS_BLOCKED_URLS_ENV)) + + +# --------------------------------------------------------------------------- +# In-sandbox proxy (stdlib only; must run on whatever python3 the image has). +# --------------------------------------------------------------------------- + +_EGRESS_PROXY_SOURCE = r''' +"""BenchFlow egress blocklist proxy. Runs as root on loopback inside the sandbox.""" +import ipaddress +import json +import os +import posixpath +import select +import socket +import ssl +import subprocess +import sys +import threading +import time +import urllib.parse + +with open(sys.argv[1], encoding="utf-8") as _cfg_fh: + CONFIG = json.load(_cfg_fh) +RULES = list(CONFIG["rules"]) +PORT = int(CONFIG["port"]) +LOG_PATH = CONFIG["log_path"] +BLOCKED_STATUS = int(CONFIG.get("blocked_status", 404)) +CA_DIR = CONFIG.get("ca_dir") # None => no TLS inspection +UPSTREAM_CA_FILE = CONFIG.get("upstream_ca_file") # tests only +# The proxy runs as root and is NOT confined by the agent-UID firewall, so it +# must refuse to become a hop into places the agent could not otherwise reach: +# loopback (root-only local services), link-local (cloud instance metadata, +# 169.254.169.254), unspecified/multicast/reserved ranges. RFC1918 private +# ranges stay reachable by default because task compose side-services (mock +# APIs, vulnerable targets) live there; flip allow_private_networks to block. +# RFC1918/ULA ranges are refused by default: the root proxy is not confined by +# the agent-UID firewall, so it must not become a hop into internal services. +# The container's OWN directly-connected subnets (the compose network with the +# task's side-services) are allowed automatically; allow_private_networks=true +# re-opens every private range for tasks that genuinely need it. +ALLOW_PRIVATE_NETWORKS = bool(CONFIG.get("allow_private_networks", False)) +ALLOW_LOCAL_SUBNETS = bool(CONFIG.get("allow_local_subnets", True)) +ALLOW_LOOPBACK_UPSTREAM = bool(CONFIG.get("allow_loopback_upstream", False)) # tests only +CONNECT_TIMEOUT = float(CONFIG.get("connect_timeout", 20)) +IDLE_TIMEOUT = float(CONFIG.get("idle_timeout", 300)) +MAX_HEAD = 256 * 1024 +# Cap on simultaneously handled connections (one thread each). A burst of +# concurrent fetches from the agent waits in the kernel listen backlog instead +# of exhausting threads or file descriptors in the proxy (review P1 #5). +MAX_CONNECTIONS = int(CONFIG.get("max_connections", 256)) +_SLOTS = threading.BoundedSemaphore(MAX_CONNECTIONS) + +_LOG_LOCK = threading.Lock() +_LEAF_LOCK = threading.Lock() + + +def log(event, **fields): + record = {"ts": time.time(), "event": event} + record.update(fields) + line = json.dumps(record, sort_keys=True) + with _LOG_LOCK: + with open(LOG_PATH, "a") as fh: + fh.write(line + "\n") + + +def split_rule(rule): + host, _, path = rule.partition("/") + return host.lower(), path.strip("/") + + +def host_matches(rule_host, host): + host = host.lower().rstrip(".") + return host == rule_host or host.endswith("." + rule_host) + + +def normalize_request_path(path): + # Same canonicalization an upstream server applies before routing, so + # percent-encoding, "//" runs, and "/x/../" segments cannot dodge a rule. + raw = path.split("?", 1)[0].split("#", 1)[0] + decoded = raw + for _ in range(4): # decode until stable: %252f -> %2f -> / (bounded) + again = urllib.parse.unquote(decoded) + if again == decoded: + break + decoded = again + collapsed = posixpath.normpath("/" + decoded).strip("/") + return "" if collapsed == "." else collapsed.lower() + + +def match_blocked(host, path): + clean = normalize_request_path(path) + for rule in RULES: + rule_host, rule_path = split_rule(rule) + rule_path = rule_path.lower() + if not host_matches(rule_host, host): + continue + if not rule_path: + return rule + if clean == rule_path or clean.startswith(rule_path): + return rule + return None + + +def request_hosts(primary_host, headers): + """Every host name a request names: the URL/CONNECT target AND the Host + header. Upstream routes on the Host header, so an agent that sends + ``GET http:///paper`` with ``Host: arxiv.org`` must be judged on both.""" + hosts = [primary_host] + host_hdr = header(headers, "host") + if host_hdr: + hdr_host = host_hdr.rpartition(":")[0] if ":" in host_hdr and not host_hdr.endswith("]") else host_hdr + hdr_host = hdr_host.strip("[]").strip().lower() + if hdr_host and hdr_host not in hosts: + hosts.append(hdr_host) + return hosts + + +def first_blocking_rule(hosts, path): + for candidate in hosts: + rule = match_blocked(candidate, path) + if rule: + return rule + return None + + +def needs_inspection(host): + if not CA_DIR: + return False + for rule in RULES: + rule_host, rule_path = split_rule(rule) + if rule_path and host_matches(rule_host, host): + return True + return False + + +def blocked_response(head_only=False): + body = b"Not Found\n" if BLOCKED_STATUS == 404 else b"Forbidden\n" + reason = "Not Found" if BLOCKED_STATUS == 404 else "Forbidden" + head = ( + "HTTP/1.1 %d %s\r\nContent-Type: text/plain\r\nContent-Length: %d\r\n" + "Connection: close\r\n\r\n" % (BLOCKED_STATUS, reason, len(body)) + ).encode() + return head if head_only else head + body + + +def simple_response(status, reason, body=b""): + return ( + "HTTP/1.1 %d %s\r\nContent-Type: text/plain\r\nContent-Length: %d\r\n" + "Connection: close\r\n\r\n" % (status, reason, len(body)) + ).encode() + body + + +def read_head(conn): + """Read up to the end of the HTTP head. Returns (head_bytes, leftover).""" + buf = b"" + while b"\r\n\r\n" not in buf: + chunk = conn.recv(65536) + if not chunk: + return buf, b"" + buf += chunk + if len(buf) > MAX_HEAD: + raise ValueError("request head too large") + head, rest = buf.split(b"\r\n\r\n", 1) + return head + b"\r\n\r\n", rest + + +def parse_head(head): + text = head.decode("latin-1") + lines = text.split("\r\n") + request_line = lines[0] + parts = request_line.split(" ") + if len(parts) < 3: + raise ValueError("malformed request line") + method, target, version = parts[0], parts[1], parts[2] + headers = [] + for line in lines[1:]: + if not line: + continue + name, _, value = line.partition(":") + headers.append((name.strip(), value.strip())) + return method, target, version, headers + + +def header(headers, name): + for k, v in headers: + if k.lower() == name.lower(): + return v + return None + + +def relay(a, b): + """Bidirectional copy until either side closes.""" + socks = [a, b] + a.setblocking(True) + b.setblocking(True) + last = time.time() + while True: + readable, _, errored = select.select(socks, [], socks, 5.0) + if errored: + break + if not readable: + if time.time() - last > IDLE_TIMEOUT: + break + continue + last = time.time() + done = False + for s in readable: + other = b if s is a else a + try: + data = s.recv(65536) + except (OSError, ssl.SSLError): + done = True + break + if not data: + done = True + break + try: + other.sendall(data) + except OSError: + done = True + break + if done: + break + + +class UpstreamRefused(Exception): + """The resolved upstream address is one the proxy must not reach for.""" + + +def _local_ipv4_subnets(route_table=None): + """Directly-connected IPv4 networks from /proc/net/route (Linux). Lines + with a gateway of 0 are on-link routes: the container's own subnets.""" + if route_table is None: + try: + with open("/proc/net/route", encoding="ascii") as fh: + route_table = fh.read() + except OSError: + return [] + nets = [] + for line in route_table.splitlines()[1:]: + parts = line.split() + if len(parts) < 8: + continue + dest_hex, gateway_hex, mask_hex = parts[1], parts[2], parts[7] + try: + if int(gateway_hex, 16) != 0: + continue # via a gateway: not on-link + dest = ipaddress.IPv4Address(int.from_bytes(bytes.fromhex(dest_hex)[::-1], "big")) + mask = ipaddress.IPv4Address(int.from_bytes(bytes.fromhex(mask_hex)[::-1], "big")) + net = ipaddress.IPv4Network("%s/%s" % (dest, mask), strict=False) + except (ValueError, OverflowError): + continue + if net.prefixlen == 0 or net.is_loopback: + continue + nets.append(net) + return nets + + +_LOCAL_SUBNETS = _local_ipv4_subnets() if ALLOW_LOCAL_SUBNETS else [] + + +def _address_refusal(ip): + if ip.is_loopback: + return None if ALLOW_LOOPBACK_UPSTREAM else "loopback" + if ip.is_link_local: + return "link-local" + if ip.is_unspecified: + return "unspecified" + if ip.is_multicast: + return "multicast" + if ip.is_reserved: + return "reserved" + if ip.is_private and not ALLOW_PRIVATE_NETWORKS: + if any(ip in net for net in _LOCAL_SUBNETS): + return None # the container's own compose network + return "private" + return None + + +def resolve_upstream(host, port): + """Resolve host and vet EVERY address (DNS-rebinding safe: we connect to + the vetted IPs, never re-resolve). Returns the candidate list ordered + IPv4-first: a default docker bridge has no IPv6 route, and glibc tends to + list AAAA records first, so trying v6 first would fail with ENETUNREACH.""" + try: + infos = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM) + except socket.gaierror as exc: + raise OSError("resolve %s: %s" % (host, exc)) + if not infos: + raise OSError("resolve %s: no addresses" % host) + candidates = [] + for family, _, _, _, sockaddr in infos: + ip = ipaddress.ip_address(sockaddr[0]) + why = _address_refusal(ip) + if why: + raise UpstreamRefused("%s resolves to %s address %s" % (host, why, ip)) + candidates.append((family, sockaddr)) + candidates.sort(key=lambda item: 0 if item[0] == socket.AF_INET else 1) + return candidates + + +def upstream_connect(host, port, tls=False): + sock = None + last_err = None + for family, sockaddr in resolve_upstream(host, port): + candidate = socket.socket(family, socket.SOCK_STREAM) + try: + candidate.settimeout(CONNECT_TIMEOUT) + candidate.connect(sockaddr) + except OSError as exc: # e.g. ENETUNREACH on a v6 address: try the next + last_err = exc + candidate.close() + continue + sock = candidate + break + if sock is None: + raise OSError("connect %s:%s: %s" % (host, port, last_err)) + sock.settimeout(IDLE_TIMEOUT) + if tls: + ctx = ssl.create_default_context() + if UPSTREAM_CA_FILE: + ctx.load_verify_locations(UPSTREAM_CA_FILE) + ctx.set_alpn_protocols(["http/1.1"]) # we only speak HTTP/1.1 in the middle + sock = ctx.wrap_socket(sock, server_hostname=host) + return sock + + +def refused_response(exc): + body = ("Forbidden: %s\n" % exc).encode() + return simple_response(403, "Forbidden", body) + + +def _leaf_key(leaf_dir): + """One shared RSA key for every leaf: keygen is the expensive step, and + per-host uniqueness buys nothing here (the CA is per-run and root-only).""" + key = os.path.join(leaf_dir, "leaf.key") + if not os.path.exists(key): + subprocess.run( + ["openssl", "genrsa", "-out", key, "2048"], check=True, capture_output=True + ) + os.chmod(key, 0o600) + return key + + +def leaf_cert(host): + """Return (cert_path, key_path) for host, signing a new leaf on first use.""" + safe = "".join(ch if ch.isalnum() or ch in ".-" else "_" for ch in host) + leaf_dir = os.path.join(CA_DIR, "leaf") + crt = os.path.join(leaf_dir, safe + ".crt") + with _LEAF_LOCK: + os.makedirs(leaf_dir, exist_ok=True) + key = _leaf_key(leaf_dir) + if os.path.exists(crt): + return crt, key + csr = os.path.join(leaf_dir, safe + ".csr") + ext = os.path.join(leaf_dir, safe + ".ext") + # RFC 5280: an IP-literal host needs an IP: SAN, not DNS: (clients + # reject the name match otherwise). Rules may name bare IPs. + try: + ipaddress.ip_address(host) + san = "IP:%s" % host + except ValueError: + san = "DNS:%s" % host + with open(ext, "w") as fh: + fh.write("subjectAltName=%s\nextendedKeyUsage=serverAuth\n" % san) + subprocess.run( + ["openssl", "req", "-new", "-key", key, "-out", csr, "-subj", "/CN=%s" % host], + check=True, capture_output=True, + ) + subprocess.run( + ["openssl", "x509", "-req", "-in", csr, + "-CA", os.path.join(CA_DIR, "ca.crt"), + "-CAkey", os.path.join(CA_DIR, "ca.key"), "-CAcreateserial", + "-out", crt, "-days", "3", "-sha256", "-extfile", ext], + check=True, capture_output=True, + ) + return crt, key + + +def forward_request(client, upstream, method, target, version, headers, leftover): + """Send one request (origin-form target) upstream and relay the response. + + ``Connection: close`` is forced so the response ends when upstream closes + and per-request policy decisions stay simple; clients reconnect per request. + """ + hop_by_hop = {"proxy-connection", "connection", "keep-alive", "proxy-authorization", + "te", "trailer", "transfer-encoding", "upgrade", "expect"} + out_headers = [(k, v) for k, v in headers if k.lower() not in hop_by_hop] + chunked = (header(headers, "transfer-encoding") or "").lower() == "chunked" + if chunked: + out_headers.append(("Transfer-Encoding", "chunked")) + out_headers.append(("Connection", "close")) + head = "%s %s %s\r\n" % (method, target, version) + head += "".join("%s: %s\r\n" % (k, v) for k, v in out_headers) + head += "\r\n" + upstream.sendall(head.encode("latin-1")) + # Expect: 100-continue — the client waits for an interim response before + # sending its body while we would wait for that body before reading + # upstream: a deadlock (until the client's own timeout). Answer the + # interim ourselves and forward the request WITHOUT the Expect header, so + # the upstream reply is a normal final response. Clients that send the body + # straight away are unaffected (the body arrives as leftover/recv anyway). + if (header(headers, "expect") or "").lower() == "100-continue" and not leftover: + client.sendall(b"HTTP/1.1 100 Continue\r\n\r\n") + # Body: Content-Length or chunked; otherwise none. + length = header(headers, "content-length") + if chunked: + upstream.sendall(leftover) + buf = leftover + while not buf.endswith(b"0\r\n\r\n"): + chunk = client.recv(65536) + if not chunk: + break + upstream.sendall(chunk) + buf = (buf + chunk)[-8:] + elif length: + remaining = int(length) - len(leftover) + upstream.sendall(leftover) + while remaining > 0: + chunk = client.recv(min(65536, remaining)) + if not chunk: + break + upstream.sendall(chunk) + remaining -= len(chunk) + while True: + data = upstream.recv(65536) + if not data: + break + client.sendall(data) + + +def handle_inspected(tls_client, host, port): + """Serve HTTP requests inside a terminated TLS session, one at a time.""" + while True: + try: + head, leftover = read_head(tls_client) + except (OSError, ValueError, ssl.SSLError): + return + if not head.strip(): + return + method, target, version, headers = parse_head(head) + path = target if target.startswith("/") else "/" + target.split("://", 1)[-1].partition("/")[2] + rule = first_blocking_rule(request_hosts(host, headers), path) + if rule: + log("block", scheme="https", host=host, port=port, method=method, path=path, rule=rule) + tls_client.sendall(blocked_response(head_only=(method == "HEAD"))) + return + try: + upstream = upstream_connect(host, port, tls=True) + except UpstreamRefused as exc: + log("refuse", scheme="https", host=host, port=port, method=method, path=path, reason=str(exc)) + tls_client.sendall(refused_response(exc)) + return + except Exception as exc: + tls_client.sendall(simple_response(502, "Bad Gateway", str(exc).encode())) + return + log("allow", scheme="https", host=host, port=port, method=method, path=path) + try: + forward_request(tls_client, upstream, method, path, version, headers, leftover) + finally: + upstream.close() + return # Connection: close semantics + + +def handle_connect(client, target): + host, _, port = target.rpartition(":") + host = host.strip("[]").lower() + port = int(port or 443) + rule = match_blocked(host, "") + if rule: + log("block", scheme="connect", host=host, port=port, rule=rule) + client.sendall(blocked_response()) + return + if needs_inspection(host): + crt, key = leaf_cert(host) + client.sendall(b"HTTP/1.1 200 Connection established\r\n\r\n") + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ctx.load_cert_chain(crt, key) + # We parse HTTP/1.1 text inside the tunnel; refuse h2 at the handshake + # so curl/httpx/Node negotiate down instead of sending binary frames. + ctx.set_alpn_protocols(["http/1.1"]) + try: + tls_client = ctx.wrap_socket(client, server_side=True) + except ssl.SSLError as exc: + log("tls-error", host=host, port=port, error=str(exc)) + return + try: + handle_inspected(tls_client, host, port) + finally: + try: + tls_client.close() + except OSError: + pass + return + try: + upstream = upstream_connect(host, port) + except UpstreamRefused as exc: + log("refuse", scheme="connect", host=host, port=port, reason=str(exc)) + client.sendall(refused_response(exc)) + return + except Exception as exc: + log("error", scheme="connect", host=host, port=port, error=str(exc)) + client.sendall(simple_response(502, "Bad Gateway", str(exc).encode())) + return + log("allow", scheme="connect", host=host, port=port) + client.sendall(b"HTTP/1.1 200 Connection established\r\n\r\n") + try: + relay(client, upstream) + finally: + upstream.close() + + +def handle_plain(client, method, target, version, headers, leftover): + if target.startswith("/"): + if target == "/healthz": + client.sendall(simple_response(200, "OK", b"ok\n")) + else: + client.sendall(simple_response(400, "Bad Request", b"absolute URL required\n")) + return + scheme, _, rest = target.partition("://") + hostport, _, path = rest.partition("/") + path = "/" + path + host, _, port = hostport.rpartition(":") if ":" in hostport else (hostport, "", "") + host = host.strip("[]").lower() + port = int(port) if port else (443 if scheme == "https" else 80) + rule = first_blocking_rule(request_hosts(host, headers), path) + if rule: + log("block", scheme=scheme, host=host, port=port, method=method, path=path, rule=rule) + client.sendall(blocked_response(head_only=(method == "HEAD"))) + return + try: + upstream = upstream_connect(host, port, tls=(scheme == "https")) + except UpstreamRefused as exc: + log("refuse", scheme=scheme, host=host, port=port, method=method, path=path, reason=str(exc)) + client.sendall(refused_response(exc)) + return + except Exception as exc: + client.sendall(simple_response(502, "Bad Gateway", str(exc).encode())) + return + log("allow", scheme=scheme, host=host, port=port, method=method, path=path) + try: + forward_request(client, upstream, method, path, version, headers, leftover) + finally: + upstream.close() + + +def handle(client): + try: + _handle(client) + finally: + _SLOTS.release() + + +def _handle(client): + client.settimeout(IDLE_TIMEOUT) + try: + head, leftover = read_head(client) + if not head.strip(): + return + method, target, version, headers = parse_head(head) + if method == "CONNECT": + handle_connect(client, target) + else: + handle_plain(client, method, target, version, headers, leftover) + except Exception as exc: + log("error", error=str(exc)) + finally: + try: + client.close() + except OSError: + pass + + +def main(): + server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server.bind(("127.0.0.1", PORT)) + server.listen(128) + log( + "start", + port=PORT, + rules=len(RULES), + tls_inspection=bool(CA_DIR), + max_connections=MAX_CONNECTIONS, + ) + while True: + _SLOTS.acquire() # back-pressure: block accept() until a slot frees up + try: + client, _ = server.accept() + except OSError: + _SLOTS.release() + continue + try: + threading.Thread(target=handle, args=(client,), daemon=True).start() + except RuntimeError as exc: # can't start new thread + _SLOTS.release() + log("error", error="thread start failed: %s" % exc) + client.close() + + +if __name__ == "__main__": + main() +''' + + +def egress_proxy_source() -> str: + """The in-sandbox proxy script (exposed for tests).""" + return _EGRESS_PROXY_SOURCE + + +def _ensure_tool_cmd(binary: str, package: str) -> str: + return ( + f"if ! command -v {binary} >/dev/null 2>&1; then " + "if command -v apt-get >/dev/null 2>&1; then " + "export DEBIAN_FRONTEND=noninteractive; " + f"apt-get update -qq && apt-get install -y -qq {package} >/dev/null; " + "elif command -v dnf >/dev/null 2>&1; then " + f"dnf -y install {package} >/dev/null; " + "elif command -v apk >/dev/null 2>&1; then " + f"apk add --no-cache {package} >/dev/null; " + f"else echo 'No supported package manager to install {package}' >&2; exit 86; fi; fi" + ) + + +def _ca_setup_cmd() -> str: + """Generate the per-run CA (root-only key) and publish a trust bundle.""" + d = shlex.quote(EGRESS_RUNTIME_DIR) + ca_dir = f"{d}/ca" + bundle = shlex.quote(EGRESS_CA_BUNDLE_PATH) + return ( + f"mkdir -p {ca_dir}/leaf && chmod 700 {ca_dir} && " + f"if [ ! -s {ca_dir}/ca.key ]; then " + f"openssl req -x509 -newkey rsa:2048 -nodes -keyout {ca_dir}/ca.key " + f"-out {ca_dir}/ca.crt -days 3 -sha256 -subj '/CN=BenchFlow Egress CA' " + "-addext 'basicConstraints=critical,CA:TRUE' " + "-addext 'keyUsage=critical,keyCertSign,cRLSign' >/dev/null 2>&1; fi && " + f"chmod 600 {ca_dir}/ca.key && " + "sys_bundle=''; for c in /etc/ssl/certs/ca-certificates.crt " + "/etc/pki/tls/certs/ca-bundle.crt /etc/ssl/cert.pem; do " + 'if [ -s "$c" ]; then sys_bundle="$c"; break; fi; done; ' + f'{{ [ -n "$sys_bundle" ] && cat "$sys_bundle"; cat {ca_dir}/ca.crt; }} ' + f"> {bundle} && chmod 644 {bundle} && " + "if [ -d /usr/local/share/ca-certificates ]; then " + f"cp {ca_dir}/ca.crt /usr/local/share/ca-certificates/benchflow-egress.crt && " + "(update-ca-certificates >/dev/null 2>&1 || true); " + "elif [ -d /etc/pki/ca-trust/source/anchors ]; then " + f"cp {ca_dir}/ca.crt /etc/pki/ca-trust/source/anchors/benchflow-egress.crt && " + "(update-ca-trust >/dev/null 2>&1 || true); fi" + ) + + +def _exec_return_code(result: Any) -> int: + code = getattr(result, "return_code", None) + return int(code) if code is not None else 1 + + +def _exec_detail(result: Any) -> str: + out = (getattr(result, "stdout", "") or "").strip() + err = (getattr(result, "stderr", "") or "").strip() + parts = [p for p in (err, out) if p] + return (" " + " | ".join(parts)[:800]) if parts else "" + + +async def _upload_text(env: Any, text: str, target_path: str, *, suffix: str) -> None: + with tempfile.NamedTemporaryFile( + "w", suffix=suffix, delete=False, encoding="utf-8" + ) as tmp: + tmp.write(text) + tmp_path = Path(tmp.name) + try: + await env.upload_file(tmp_path, target_path) + finally: + tmp_path.unlink(missing_ok=True) + + +async def start_egress_proxy( + env: Any, + blocklist: EgressBlocklist, + *, + timeout_sec: int = 180, +) -> None: + """Install and start the filtering proxy as root; idempotent per sandbox. + + Runs before the agent launches so ``HTTP(S)_PROXY`` already resolves. The + rule file and log are root-only (mode 0600): the agent must not learn the + hidden URLs by reading the proxy's configuration. + """ + runtime_dir = EGRESS_RUNTIME_DIR + q_dir = shlex.quote(runtime_dir) + script_path = f"{runtime_dir}/egress_proxy.py" + config_path = f"{runtime_dir}/policy.json" + pid_path = f"{runtime_dir}/proxy.pid" + stderr_path = f"{runtime_dir}/proxy.stderr" + + health = ( + 'python3 -c "import urllib.request,sys;' + f"sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:{blocklist.proxy_port}/healthz'," + ' timeout=2).status==200 else 1)"' + ) + already = await env.exec(health, user="root", timeout_sec=10) + if _exec_return_code(already) == 0: + logger.info("Egress blocklist proxy already running on %s", blocklist.proxy_url) + return + + prep = ( + "set -e; " + f"mkdir -p {q_dir} && chmod 700 {q_dir}; " + "if ! command -v python3 >/dev/null 2>&1; then " + "echo 'python3 is required in the task image for the egress blocklist proxy' >&2; " + "exit 87; fi" + ) + if blocklist.needs_tls_inspection: + prep += "; " + _ensure_tool_cmd("openssl", "openssl") + "; " + _ca_setup_cmd() + result = await env.exec(prep, user="root", timeout_sec=timeout_sec) + if _exec_return_code(result) != 0: + raise RuntimeError( + f"Failed to prepare the egress blocklist runtime.{_exec_detail(result)}" + ) + + config = { + "rules": list(blocklist.rules), + "port": blocklist.proxy_port, + "log_path": EGRESS_LOG_PATH, + "blocked_status": BLOCKED_STATUS, + "ca_dir": f"{runtime_dir}/ca" if blocklist.needs_tls_inspection else None, + } + await _upload_text(env, _EGRESS_PROXY_SOURCE, script_path, suffix=".py") + await _upload_text(env, json.dumps(config), config_path, suffix=".json") + + start = ( + f"chmod 600 {shlex.quote(config_path)} && " + f"touch {shlex.quote(EGRESS_LOG_PATH)} && chmod 600 {shlex.quote(EGRESS_LOG_PATH)} && " + f"(nohup python3 {shlex.quote(script_path)} {shlex.quote(config_path)} " + f">/dev/null 2>{shlex.quote(stderr_path)} {shlex.quote(pid_path)})" + ) + result = await env.exec(start, user="root", timeout_sec=30) + if _exec_return_code(result) != 0: + raise RuntimeError( + f"Failed to start the egress blocklist proxy.{_exec_detail(result)}" + ) + + last = "" + for _ in range(80): + probe = await env.exec(health, user="root", timeout_sec=10) + if _exec_return_code(probe) == 0: + logger.info( + "Egress blocklist proxy active on %s (%d rules, tls_inspection=%s)", + blocklist.proxy_url, + len(blocklist.rules), + blocklist.needs_tls_inspection, + ) + return + last = _exec_detail(probe) + await asyncio.sleep(0.25) + stderr = await env.exec( + f"cat {shlex.quote(stderr_path)} 2>/dev/null | tail -n 20", + user="root", + timeout_sec=10, + ) + raise RuntimeError( + f"Egress blocklist proxy did not become healthy.{last}{_exec_detail(stderr)}" + ) + + +def _probe_cmd(blocklist: EgressBlocklist) -> str: + """Shell probe run AS THE AGENT USER: the first blocked host **or path** + must answer the blocked status through the proxy, and direct egress must + be rejected. + + The probe targets the rule's full ``host/path`` — a path-only rule such as + ``arxiv.org/abs/2401.12345`` leaves ``arxiv.org/`` legitimately open, so + probing the root would report 200 and fail a correctly configured run. + """ + rule_host, rule_path = _split_rule(blocklist.rules[0]) + probe_target = f"/{rule_path}" if rule_path else "/" + return ( + "python3 - <<'PY'\n" + "import json, socket, sys, urllib.error, urllib.request\n" + f"proxy = {blocklist.proxy_url!r}\n" + f"host = {rule_host!r}\n" + f"target = {probe_target!r}\n" + f"expected = {BLOCKED_STATUS}\n" + "out = {'blocked_target': host + target}\n" + "opener = urllib.request.build_opener(urllib.request.ProxyHandler({'http': proxy, 'https': proxy}))\n" + "try:\n" + " resp = opener.open('http://' + host + target, timeout=15)\n" + " out['via_proxy_status'] = resp.status\n" + "except urllib.error.HTTPError as exc:\n" + " out['via_proxy_status'] = exc.code\n" + "except Exception as exc:\n" + " out['via_proxy_status'] = None\n" + " out['via_proxy_error'] = str(exc)[:200]\n" + "try:\n" + " s = socket.create_connection(('1.1.1.1', 443), timeout=5)\n" + " s.close()\n" + " out['direct_egress'] = 'open'\n" + "except Exception as exc:\n" + " out['direct_egress'] = 'blocked'\n" + " out['direct_egress_error'] = str(exc)[:200]\n" + "out['ok'] = out['via_proxy_status'] == expected and out['direct_egress'] == 'blocked'\n" + "print(json.dumps(out))\n" + "sys.exit(0 if out['ok'] else 1)\n" + "PY\n" + ) + + +async def verify_egress_blocklist( + env: Any, sandbox_user: str | None, agent_env: dict[str, str] +) -> dict[str, Any] | None: + """Post-firewall self-check, run as the sandbox user. Returns the probe. + + Fails loudly when the blocked host is reachable or direct egress is still + open: an experiment that silently ran without its blocklist is worthless. + """ + blocklist = EgressBlocklist.from_env(agent_env) + if blocklist is None: + return None + if not sandbox_user: + raise RuntimeError( + "network_mode='blocklist' requires a sandbox_user: the agent-UID " + "firewall is what makes the proxy mandatory" + ) + result = await env.exec(_probe_cmd(blocklist), user=sandbox_user, timeout_sec=60) + stdout = (getattr(result, "stdout", "") or "").strip().splitlines() + probe: dict[str, Any] = {} + for line in reversed(stdout): + try: + probe = json.loads(line) + break + except ValueError: + continue + record = json.dumps({"event": "probe", "user": sandbox_user, **probe}) + await env.exec( + f"printf '%s\\n' {shlex.quote(record)} >> {shlex.quote(EGRESS_LOG_PATH)}", + user="root", + timeout_sec=10, + ) + if _exec_return_code(result) != 0 or not probe.get("ok"): + raise RuntimeError( + "Egress blocklist self-check failed: " + f"{json.dumps(probe) if probe else _exec_detail(result)}" + ) + logger.info("Egress blocklist verified for %s: %s", sandbox_user, probe) + return probe + + +async def download_egress_log(env: Any, target_dir: Path) -> None: + """Copy the root-only egress log into the rollout's agent artifacts. + + Callers gate on whether the proxy was ever started for this sandbox (any + role, any scene) — not on the primary agent's env, which an oracle primary + never carries even when a role agent ran under the blocklist. + """ + target_dir.mkdir(parents=True, exist_ok=True) + target = target_dir / EGRESS_LOG_ARTIFACT_NAME + try: + await env.download_file(EGRESS_LOG_PATH, target) + except Exception as exc: # audit artifact, never fatal + logger.warning("Could not download egress log: %s", exc) diff --git a/src/benchflow/sandbox/lockdown.py b/src/benchflow/sandbox/lockdown.py index 74eff863e..849d9a348 100644 --- a/src/benchflow/sandbox/lockdown.py +++ b/src/benchflow/sandbox/lockdown.py @@ -143,13 +143,30 @@ def build_priv_drop_cmd(agent_launch: str, sandbox_user: str) -> str: ) +def agent_network_policy_active(agent_env: dict[str, str]) -> bool: + """True when an agent-layer network policy is on: the no-web policy + (``BENCHFLOW_DISALLOW_WEB_TOOLS``) or the egress blocklist. Both keep the + container online for the sandbox-local model proxy and confine the agent + UID to loopback instead.""" + from benchflow.sandbox.egress import blocklist_active + + return agent_env.get("BENCHFLOW_DISALLOW_WEB_TOOLS") == "1" or blocklist_active( + agent_env + ) + + async def enforce_agent_egress_firewall( env: Any, sandbox_user: str | None, agent_env: dict[str, str], ) -> None: - """Block sandbox-user external egress after ACP bootstrap, before prompting.""" - if not sandbox_user or agent_env.get("BENCHFLOW_DISALLOW_WEB_TOOLS") != "1": + """Block sandbox-user external egress after ACP bootstrap, before prompting. + + Under the egress blocklist the same rule makes the loopback filtering + proxy the agent's only route out, so tools that ignore ``HTTP(S)_PROXY`` + fail closed rather than bypassing the blocklist. + """ + if not sandbox_user or not agent_network_policy_active(agent_env): return base_url = agent_env.get("BENCHFLOW_PROVIDER_BASE_URL") or agent_env.get( @@ -162,7 +179,8 @@ async def enforce_agent_egress_firewall( or parsed.port is None ): raise RuntimeError( - "No-web agent requires an HTTP loopback provider base URL with a port" + "Agent network policy requires an HTTP loopback provider base URL " + "with a port" ) result = await env.exec( diff --git a/src/benchflow/sandbox/providers.py b/src/benchflow/sandbox/providers.py index f26fbcaa3..b78c30d59 100644 --- a/src/benchflow/sandbox/providers.py +++ b/src/benchflow/sandbox/providers.py @@ -41,6 +41,11 @@ class SandboxProvider: #: registry instead of growing a ``sandbox == ""`` special case per #: backend that cannot isolate the network. enforces_no_network: bool = True + #: Whether the backend can enforce the agent-phase egress blocklist + #: (``network_mode = "blocklist"``): a root-run loopback proxy plus an + #: agent-UID iptables rule inside the sandbox. Needs NET_ADMIN (docker + #: stacks a compose overlay; Daytona VMs allow it natively). + enforces_egress_blocklist: bool = False #: Whether the backend can run a task's docker-compose side services. #: ``False`` means a multi-service task must be refused, not run partially. supports_compose: bool = False @@ -58,12 +63,14 @@ def off_box_model(self) -> bool: "docker", extra=None, model_proxy=ModelProxyLocation.HOST, + enforces_egress_blocklist=True, supports_compose=True, ), SandboxProvider( "daytona", extra="sandbox-daytona", model_proxy=ModelProxyLocation.SANDBOX, + enforces_egress_blocklist=True, # The DinD strategy runs compose inside the sandbox VM. supports_compose=True, ), @@ -115,6 +122,10 @@ def off_box_model(self) -> bool: NO_NETWORK_UNSUPPORTED_PROVIDERS: frozenset[str] = frozenset( p.name for p in _PROVIDERS if not p.enforces_no_network ) +#: Providers that cannot enforce ``network_mode = "blocklist"``. +EGRESS_BLOCKLIST_UNSUPPORTED_PROVIDERS: frozenset[str] = frozenset( + p.name for p in _PROVIDERS if not p.enforces_egress_blocklist +) def is_known_provider(name: str) -> bool: diff --git a/src/benchflow/sandbox/setup.py b/src/benchflow/sandbox/setup.py index 87f2e18ee..c491c32c4 100644 --- a/src/benchflow/sandbox/setup.py +++ b/src/benchflow/sandbox/setup.py @@ -749,6 +749,7 @@ def _create_sandbox_environment( rollout_paths=rollout_paths, task_env_config=env_config, persistent_env=manifest_env or None, + agent_network_policy=preserve_agent_network, ) elif sandbox_type == "daytona": try: diff --git a/src/benchflow/task/runtime_capabilities.py b/src/benchflow/task/runtime_capabilities.py index 6f1bfd9e8..d9647024c 100644 --- a/src/benchflow/task/runtime_capabilities.py +++ b/src/benchflow/task/runtime_capabilities.py @@ -17,6 +17,7 @@ from benchflow.rewards.rubric_config import criteria_aggregate_policy_from_rubric from benchflow.sandbox._compose import compose_definition_path from benchflow.sandbox.providers import ( + EGRESS_BLOCKLIST_UNSUPPORTED_PROVIDERS, NO_NETWORK_UNSUPPORTED_PROVIDERS, SANDBOX_PROVIDER_SET, SINGLE_CONTAINER_PROVIDERS, @@ -282,12 +283,23 @@ def _append_network_issue( sandbox=sandbox, ) if mode == NetworkMode.BLOCKLIST: - _issue( - unsupported, - path=path, - reason="network blocklists are parsed but not enforced per sandbox", - sandbox=sandbox, - ) + if path.startswith("verifier"): + _issue( + unsupported, + path=path, + reason=( + "network_mode='blocklist' applies to the agent phase only; " + "the verifier has no egress blocklist" + ), + sandbox=sandbox, + ) + elif sandbox in EGRESS_BLOCKLIST_UNSUPPORTED_PROVIDERS: + _issue( + unsupported, + path=path, + reason=f"network_mode='blocklist' is not enforced by {sandbox}", + sandbox=sandbox, + ) def _append_document_issues( diff --git a/tests/test_egress_blocklist.py b/tests/test_egress_blocklist.py new file mode 100644 index 000000000..03915f820 --- /dev/null +++ b/tests/test_egress_blocklist.py @@ -0,0 +1,1566 @@ +"""Egress blocklist (``network_mode = "blocklist"``) enforcement tests. + +Guards the egress-blocklist PR: policy resolution, agent-env shaping, the +per-harness web-tool knobs, the LiteLLM server-tool rewrite, the lockdown +gate, the mocked start/verify flows, and — on Linux with openssl — a real +run of the in-sandbox proxy covering plain HTTP, CONNECT tunnels, and TLS +inspection with path rules. +""" + +from __future__ import annotations + +import http.server +import json +import os +import shutil +import socket +import ssl +import subprocess +import sys +import threading +import time +import urllib.error +import urllib.request +from datetime import datetime +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from benchflow.sandbox.egress import ( + BLOCKED_STATUS, + EGRESS_BLOCKED_URLS_ENV, + EGRESS_LOG_PATH, + EgressBlocklist, + apply_blocklist_env, + blocklist_active, + egress_proxy_source, + match_blocked, + start_egress_proxy, + strip_blocklist_secret, + strip_proxy_env, + verify_egress_blocklist, +) +from benchflow.task.config import TaskConfig + +LINUX_ONLY = pytest.mark.skipif(sys.platform == "win32", reason="needs bash/Linux") +NEEDS_OPENSSL = pytest.mark.skipif( + sys.platform == "win32" or shutil.which("openssl") is None, + reason="needs openssl + Linux sockets", +) + + +# ------------------------------------------------------------------ matching + + +@pytest.mark.parametrize( + ("host", "path", "expected"), + [ + ("arxiv.org", "/abs/2401.12345", "arxiv.org/abs/2401.12345"), + ("ARXIV.ORG", "/abs/2401.12345v2", "arxiv.org/abs/2401.12345"), + ("export.arxiv.org", "/abs/2401.12345?x=1", "arxiv.org/abs/2401.12345"), + ("arxiv.org", "/abs/2402.00001", None), + ("arxiv.org", "/", None), + ("notarxiv.org", "/abs/2401.12345", None), + ("openreview.net", "/forum?id=abc", "openreview.net"), + ("api.openreview.net", "/", "openreview.net"), + ("openreview.net.evil.com", "/", None), + ], +) +def test_match_blocked_host_suffix_and_path_prefix(host, path, expected): + rules = ("arxiv.org/abs/2401.12345", "openreview.net") + assert match_blocked(rules, host, path) == expected + + +@pytest.mark.parametrize( + "path", + [ + "/abs/%32%34%30%31%2e%31%32%33%34%35", # percent-encoded digits + "//abs///2401.12345", # slash runs + "/other/../abs/2401.12345", # dot segments + "/./abs/./2401.12345v2?download=1#x", # dot-segments + query + fragment + "/ABS/2401.12345", # case + "/abs/2401.12345/", # trailing slash + ], +) +def test_match_blocked_normalizes_like_an_upstream_server(path): + """Review round 5 (WAF bypass): matching must see the path the upstream + server will route, not the raw bytes the agent typed.""" + rules = ("arxiv.org/abs/2401.12345",) + assert match_blocked(rules, "arxiv.org", path) == "arxiv.org/abs/2401.12345" + assert match_blocked(rules, "arxiv.org", "/abs/%32%34%30%32.1") is None + + +def test_normalize_request_path_edge_cases(): + from benchflow.sandbox.egress import normalize_request_path + + assert normalize_request_path("/") == "" + assert normalize_request_path("") == "" + assert normalize_request_path("/../../etc") == "etc" + assert normalize_request_path("/a/%2e%2e/b") == "b" + # Double / triple encoding decodes until stable (Devin review). + assert normalize_request_path("/abs%252f2401.12345") == "abs/2401.12345" + assert normalize_request_path("/abs/%2532%2534") == "abs/24" + assert match_blocked( + ("arxiv.org/abs/2401.12345",), "arxiv.org", "/abs%252f2401.12345" + ) + + +def test_blocklist_tls_inspection_hosts_are_only_hosts_with_path_rules(): + b = EgressBlocklist(rules=("arxiv.org/abs/1", "arxiv.org/pdf/1", "openreview.net")) + assert b.tls_inspection_hosts == ("arxiv.org",) + assert b.needs_tls_inspection is True + assert EgressBlocklist(rules=("openreview.net",)).needs_tls_inspection is False + + +def test_blocklist_requires_rules(): + with pytest.raises(ValueError, match="at least one rule"): + EgressBlocklist(rules=()) + + +# ------------------------------------------------------------------ resolution + + +def test_from_task_config_uses_sandbox_then_agent_override(): + sandbox_only = TaskConfig.model_validate( + {"sandbox": {"network_mode": "blocklist", "blocked_urls": ["openreview.net"]}} + ) + assert EgressBlocklist.from_task_config(sandbox_only).rules == ("openreview.net",) + + agent_override = TaskConfig.model_validate( + { + "agent": {"network_mode": "blocklist", "blocked_urls": ["arxiv.org/abs/1"]}, + "sandbox": { + "network_mode": "blocklist", + "blocked_urls": ["openreview.net"], + }, + } + ) + assert EgressBlocklist.from_task_config(agent_override).rules == ( + "arxiv.org/abs/1", + ) + + agent_public = TaskConfig.model_validate( + { + "agent": {"network_mode": "public"}, + "sandbox": { + "network_mode": "blocklist", + "blocked_urls": ["openreview.net"], + }, + } + ) + # Devin review: an explicit agent-level mode must not silently shadow a + # sandbox blocklist (which is where --block-url lands) — it is a conflict. + with pytest.raises(ValueError, match="overridden by the agent-level"): + EgressBlocklist.from_task_config(agent_public) + assert EgressBlocklist.from_task_config(TaskConfig.model_validate({})) is None + agent_blocklist_only = TaskConfig.model_validate( + {"agent": {"network_mode": "blocklist", "blocked_urls": ["x.org"]}} + ) + assert EgressBlocklist.from_task_config(agent_blocklist_only).rules == ("x.org",) + + +def test_agent_env_routes_through_proxy_without_revealing_rules(): + b = EgressBlocklist(rules=("arxiv.org/abs/2401.12345",), proxy_port=61399) + env = b.agent_env() + assert env["HTTP_PROXY"] == env["HTTPS_PROXY"] == "http://127.0.0.1:61399" + assert env["NO_PROXY"] == "127.0.0.1,localhost,::1" + assert env["NODE_USE_ENV_PROXY"] == "1" + assert env["SSL_CERT_FILE"] == env["NODE_EXTRA_CA_CERTS"] + # The hidden URLs must never be readable from the agent process env. + assert EGRESS_BLOCKED_URLS_ENV not in env + assert "2401.12345" not in json.dumps(env) + + no_tls = EgressBlocklist(rules=("openreview.net",)).agent_env() + assert "SSL_CERT_FILE" not in no_tls + + +def test_apply_and_strip_blocklist_env_round_trip(): + b = EgressBlocklist(rules=("openreview.net",)) + applied = apply_blocklist_env({"FOO": "1"}, b) + assert blocklist_active(applied) + assert EgressBlocklist.from_env(applied) == b + assert apply_blocklist_env({"FOO": "1"}, None) == {"FOO": "1"} + + process_env = strip_blocklist_secret(applied) + assert EGRESS_BLOCKED_URLS_ENV not in process_env + assert process_env["HTTP_PROXY"] == b.proxy_url + assert process_env["FOO"] == "1" + assert not blocklist_active(process_env) + + +def test_strip_proxy_env_keeps_rules_for_model_proxy(): + b = EgressBlocklist(rules=("arxiv.org/abs/1",)) + env = strip_proxy_env(apply_blocklist_env({"OPENAI_API_KEY": "k"}, b)) + assert "HTTP_PROXY" not in env and "SSL_CERT_FILE" not in env + assert env[EGRESS_BLOCKED_URLS_ENV] == b.to_env_value() + assert env["OPENAI_API_KEY"] == "k" + + +# ------------------------------------------------------------------ rollout policy + + +def test_apply_web_policy_marks_blocklist_and_no_web_independently(): + from benchflow.rollout._setup import _apply_web_policy, _task_egress_blocklist + + b = EgressBlocklist(rules=("openreview.net",)) + env = _apply_web_policy({}, disallow=False, blocklist=b) + assert "BENCHFLOW_DISALLOW_WEB_TOOLS" not in env + assert blocklist_active(env) + both = _apply_web_policy({}, disallow=True, blocklist=None) + assert both == {"BENCHFLOW_DISALLOW_WEB_TOOLS": "1"} + + task = SimpleNamespace( + config=TaskConfig.model_validate( + { + "sandbox": { + "network_mode": "blocklist", + "blocked_urls": ["openreview.net"], + } + } + ) + ) + assert _task_egress_blocklist(task) == b + assert _task_egress_blocklist(SimpleNamespace()) is None + + +def test_agent_launch_applies_blocklist_suffix_only_for_server_side_search(): + from benchflow.agents.registry import AGENT_LAUNCH + from benchflow.rollout._setup import _agent_launch_with_web_policy + + codex = _agent_launch_with_web_policy("codex-acp", disallow=False, blocklist=True) + assert codex == AGENT_LAUNCH["codex-acp"] + " -c tools.web_search=false" + # A no-web run keeps its own (identical here) knob and wins over blocklist. + assert ( + _agent_launch_with_web_policy("codex-acp", disallow=True, blocklist=True) + == codex + ) + # Client-side fetch tools go through the proxy, so nothing is appended. + for agent in ("claude-agent-acp", "opencode", "openhands", "gemini"): + assert ( + _agent_launch_with_web_policy(agent, disallow=False, blocklist=True) + == AGENT_LAUNCH[agent] + ) + + +def test_registry_blocklist_knobs_cover_only_unfilterable_server_tools(): + from benchflow.agents.registry import AGENTS + + assert ( + AGENTS["codex-acp"].blocklist_web_tools_launch_suffix + == " -c tools.web_search=false" + ) + assert "google_web_search" in AGENTS["gemini"].blocklist_web_tools_setup_cmd + # Anthropic server tools are filtered in the LiteLLM hook; client-side + # fetchers (OpenCode/MiMo webfetch, OpenHands browsing) stay enabled. + for agent in ("claude-agent-acp", "opencode", "mimo", "openhands", "pi-acp"): + cfg = AGENTS[agent] + assert cfg.blocklist_web_tools_setup_cmd == "" + assert cfg.blocklist_web_tools_launch_suffix == "" + + +def test_manifest_contract_keeps_blocklist_knobs_shim_only(): + from benchflow.agents.manifest import _SHIM_ONLY + + assert { + "blocklist_web_tools_setup_cmd", + "blocklist_web_tools_launch_suffix", + } <= _SHIM_ONLY + + +@LINUX_ONLY +def test_gemini_blocklist_setup_cmd_excludes_server_side_tools(tmp_path): + from benchflow.agents.registry import AGENTS + + home = tmp_path / "home" + subprocess.run( + ["bash", "-c", AGENTS["gemini"].blocklist_web_tools_setup_cmd], + check=True, + env={**os.environ, "BENCHFLOW_AGENT_HOME": str(home)}, + ) + settings = json.loads((home / ".gemini" / "settings.json").read_text()) + assert settings["tools"]["exclude"] == ["google_web_search", "web_fetch"] + + +@pytest.mark.asyncio +async def test_apply_web_tool_policy_blocklist_runs_narrow_cmd_and_no_web_wins(): + from benchflow.agents.install import apply_web_tool_policy + from benchflow.agents.registry import AgentConfig + + cfg = AgentConfig( + name="x", + install_cmd="true", + launch_cmd="x", + disallow_web_tools_setup_cmd="echo no-web", + blocklist_web_tools_setup_cmd="echo blocklist", + ) + env = MagicMock() + env.exec = AsyncMock(return_value=MagicMock(return_code=0)) + + await apply_web_tool_policy(env, "x", cfg, "/root", disallow=False, blocklist=True) + assert "echo blocklist" in env.exec.await_args.args[0] + + env.exec.reset_mock() + await apply_web_tool_policy(env, "x", cfg, "/root", disallow=True, blocklist=True) + assert "echo no-web" in env.exec.await_args.args[0] + + env.exec.reset_mock() + await apply_web_tool_policy(env, "x", cfg, "/root", disallow=False, blocklist=False) + env.exec.assert_not_awaited() + + +# ------------------------------------------------------------------ lockdown gate + + +@pytest.mark.asyncio +async def test_firewall_fires_under_blocklist_and_requires_loopback_model_proxy(): + from benchflow.sandbox.lockdown import ( + agent_network_policy_active, + enforce_agent_egress_firewall, + ) + + b = EgressBlocklist(rules=("openreview.net",)) + env = MagicMock() + env.exec = AsyncMock(return_value=MagicMock(return_code=0)) + agent_env = apply_blocklist_env( + {"BENCHFLOW_PROVIDER_BASE_URL": "http://127.0.0.1:4000/v1"}, b + ) + assert agent_network_policy_active(agent_env) + assert not agent_network_policy_active({"HTTP_PROXY": b.proxy_url}) + + await enforce_agent_egress_firewall(env, "agent", agent_env) + env.exec.assert_awaited_once() + assert '--uid-owner "$agent_uid" -j REJECT' in env.exec.await_args.args[0] + + with pytest.raises(RuntimeError, match="loopback provider base URL"): + await enforce_agent_egress_firewall( + env, + "agent", + apply_blocklist_env({"OPENAI_BASE_URL": "https://api.openai.com/v1"}, b), + ) + + +# ------------------------------------------------------------------ start / verify (mocked) + + +def _exec_recorder(responses): + """Build an ``env.exec`` mock whose result depends on the command text.""" + calls: list[tuple[str, dict]] = [] + + async def _exec(cmd, **kwargs): + calls.append((cmd, kwargs)) + for needle, result in responses: + if needle in cmd: + return result + return MagicMock(return_code=0, stdout="", stderr="") + + return AsyncMock(side_effect=_exec), calls + + +@pytest.mark.asyncio +async def test_start_egress_proxy_uploads_policy_and_waits_for_health(): + b = EgressBlocklist(rules=("arxiv.org/abs/2401.12345",)) + health_states = iter( + [1, 1, 0] + ) # first probe (already-running check) fails, then up + + async def _exec(cmd, **kwargs): + if "/healthz" in cmd: + return MagicMock(return_code=next(health_states), stdout="", stderr="") + return MagicMock(return_code=0, stdout="", stderr="") + + env = MagicMock() + env.exec = AsyncMock(side_effect=_exec) + env.upload_file = AsyncMock() + + await start_egress_proxy(env, b) + + uploaded = { + str(call.args[1]): Path(call.args[0]) + for call in env.upload_file.await_args_list + } + assert "/opt/benchflow/egress/egress_proxy.py" in uploaded + assert "/opt/benchflow/egress/policy.json" in uploaded + root_cmds = [ + c.args[0] for c in env.exec.await_args_list if c.kwargs.get("user") == "root" + ] + assert all(c.kwargs.get("user") == "root" for c in env.exec.await_args_list) + # TLS inspection (path rule) provisions openssl + the per-run CA. + assert any("openssl req -x509" in c for c in root_cmds) + assert any("chmod 600 /opt/benchflow/egress/policy.json" in c for c in root_cmds) + assert any( + "nohup python3 /opt/benchflow/egress/egress_proxy.py" in c for c in root_cmds + ) + + +@pytest.mark.asyncio +async def test_start_egress_proxy_is_idempotent_when_already_healthy(): + env = MagicMock() + env.exec = AsyncMock(return_value=MagicMock(return_code=0, stdout="", stderr="")) + env.upload_file = AsyncMock() + + await start_egress_proxy(env, EgressBlocklist(rules=("openreview.net",))) + + env.upload_file.assert_not_awaited() + env.exec.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_start_egress_proxy_fails_loudly_without_python3(): + async def _exec(cmd, **kwargs): + if "/healthz" in cmd: + return MagicMock(return_code=1, stdout="", stderr="") + return MagicMock(return_code=87, stdout="", stderr="python3 is required") + + env = MagicMock() + env.exec = AsyncMock(side_effect=_exec) + env.upload_file = AsyncMock() + with pytest.raises(RuntimeError, match="python3 is required"): + await start_egress_proxy(env, EgressBlocklist(rules=("openreview.net",))) + + +@pytest.mark.asyncio +async def test_verify_egress_blocklist_runs_probe_as_sandbox_user_and_logs_it(): + b = EgressBlocklist(rules=("openreview.net",)) + probe = {"via_proxy_status": BLOCKED_STATUS, "direct_egress": "blocked", "ok": True} + env = MagicMock() + env.exec, calls = _exec_recorder( + [ + ( + "python3 - <<'PY'", + MagicMock(return_code=0, stdout=json.dumps(probe), stderr=""), + ) + ] + ) + + result = await verify_egress_blocklist(env, "agent", apply_blocklist_env({}, b)) + + assert result == probe + probe_call = next(c for c in calls if "python3 - <<'PY'" in c[0]) + assert probe_call[1]["user"] == "agent" + assert "target = '/'" in probe_call[0] + log_call = next(c for c in calls if EGRESS_LOG_PATH in c[0] and "printf" in c[0]) + assert log_call[1]["user"] == "root" + assert '"event": "probe"' in log_call[0] + + +def test_probe_targets_the_rule_path_not_the_host_root(): + """Review bug: a path-only rule leaves the host root open, so the self-check + must probe host/path (which the proxy hides) — probing "/" reported 200 + and aborted correctly configured rollouts.""" + from benchflow.sandbox.egress import _probe_cmd + + cmd = _probe_cmd(EgressBlocklist(rules=("arxiv.org/abs/2401.12345", "x.org"))) + assert "host = 'arxiv.org'" in cmd + assert "target = '/abs/2401.12345'" in cmd + assert "opener.open('http://' + host + target" in cmd + assert "'/'" not in cmd.split("target = ")[1].split("\n")[0] + + +@NEEDS_OPENSSL +def test_probe_passes_end_to_end_against_real_proxy_for_path_only_rule( + running_proxy, monkeypatch +): + """Run the actual self-check script against the live proxy: the first rule + is path-only, so only the exact blocked path yields the blocked status.""" + from benchflow.sandbox.egress import _probe_cmd + + p = running_proxy + # The fixture's first rule is a host rule; build a path-only blocklist that + # points at the same proxy and matches the fixture's "localhost/secret". + b = EgressBlocklist(rules=("localhost/secret",), proxy_port=p.proxy_port) + script = _probe_cmd(b).split("<<'PY'\n", 1)[1].rsplit("\nPY", 1)[0] + # The direct-egress leg needs the UID firewall; emulate it by making the + # raw socket connect fail the way REJECT does. + script = script.replace( + "socket.create_connection(('1.1.1.1', 443), timeout=5)", + "(_ for _ in ()).throw(OSError('Connection refused'))", + ) + # localhost: is the inspected origin; plain-HTTP probe goes to the + # HTTP origin on the same host name so the path rule applies. + script = script.replace( + "'http://' + host + target", f"'http://' + host + ':{p.http_port}' + target" + ) + result = subprocess.run( + [sys.executable, "-c", script], capture_output=True, text=True, timeout=60 + ) + out = json.loads(result.stdout.strip().splitlines()[-1]) + assert out["blocked_target"] == "localhost/secret" + assert out["via_proxy_status"] == BLOCKED_STATUS + assert out["ok"] is True, out + assert result.returncode == 0 + + +@pytest.mark.asyncio +async def test_verify_egress_blocklist_fails_when_hidden_host_is_reachable(): + b = EgressBlocklist(rules=("openreview.net",)) + probe = {"via_proxy_status": 200, "direct_egress": "open", "ok": False} + env = MagicMock() + env.exec, _ = _exec_recorder( + [ + ( + "python3 - <<'PY'", + MagicMock(return_code=1, stdout=json.dumps(probe), stderr=""), + ) + ] + ) + with pytest.raises(RuntimeError, match="self-check failed"): + await verify_egress_blocklist(env, "agent", apply_blocklist_env({}, b)) + + +@pytest.mark.asyncio +async def test_verify_egress_blocklist_is_noop_without_policy_and_needs_sandbox_user(): + env = MagicMock() + env.exec = AsyncMock() + assert await verify_egress_blocklist(env, "agent", {}) is None + env.exec.assert_not_awaited() + with pytest.raises(RuntimeError, match="requires a sandbox_user"): + await verify_egress_blocklist( + env, None, apply_blocklist_env({}, EgressBlocklist(rules=("x.org",))) + ) + + +# ------------------------------------------------------------------ LiteLLM hook + + +def _hook_namespace(monkeypatch, rules): + from benchflow.providers.litellm_logging import callback_module_source + + monkeypatch.setenv(EGRESS_BLOCKED_URLS_ENV, json.dumps(rules)) + namespace: dict[str, object] = {} + exec(callback_module_source(), namespace) + return namespace + + +@pytest.mark.asyncio +async def test_pre_call_hook_injects_anthropic_blocked_domains(monkeypatch): + ns = _hook_namespace(monkeypatch, ["arxiv.org/abs/2401.12345", "openreview.net"]) + logger = ns["proxy_handler_instance"] + data = { + "model": "claude-fable-5-1", + "messages": [{"role": "user", "content": "hi"}], + "tools": [ + {"type": "web_search_20250305", "name": "web_search", "max_uses": 3}, + { + "type": "web_fetch_20250910", + "name": "web_fetch", + "blocked_domains": ["x.org"], + }, + {"name": "bash", "input_schema": {"type": "object"}}, + ], + } + + cleaned = await logger.async_pre_call_hook(None, None, data, "anthropic_messages") + + assert cleaned is not data + search, fetch, bash = cleaned["tools"] + assert search["blocked_domains"] == ["arxiv.org/abs/2401.12345", "openreview.net"] + assert fetch["blocked_domains"] == [ + "x.org", + "arxiv.org/abs/2401.12345", + "openreview.net", + ] + assert bash == {"name": "bash", "input_schema": {"type": "object"}} + # The original request is not mutated in place. + assert "blocked_domains" not in data["tools"][0] + + +@pytest.mark.asyncio +async def test_pre_call_hook_narrows_anthropic_allowed_domains_instead_of_mixing( + monkeypatch, +): + ns = _hook_namespace(monkeypatch, ["openreview.net"]) + logger = ns["proxy_handler_instance"] + data = { + "model": "claude-fable-5-1", + "messages": [], + "tools": [ + { + "type": "web_search_20250305", + "name": "web_search", + "allowed_domains": ["arxiv.org", "api.openreview.net"], + } + ], + } + cleaned = await logger.async_pre_call_hook(None, None, data, "anthropic_messages") + tool = cleaned["tools"][0] + assert tool["allowed_domains"] == ["arxiv.org"] + assert "blocked_domains" not in tool + + +@pytest.mark.asyncio +async def test_pre_call_hook_strips_openai_hosted_search_under_blocklist(monkeypatch): + ns = _hook_namespace(monkeypatch, ["openreview.net"]) + logger = ns["proxy_handler_instance"] + data = { + "model": "gpt-5.5", + "input": [{"role": "user", "content": "hi"}], + "tools": [ + {"type": "web_search"}, + {"type": "function", "name": "shell", "parameters": {}}, + ], + } + cleaned = await logger.async_pre_call_hook(None, None, data, "aresponses") + assert [t["type"] for t in cleaned["tools"]] == ["function"] + + +@pytest.mark.asyncio +async def test_pre_call_hook_still_drops_anthropic_server_tools_without_blocklist( + monkeypatch, +): + """Review fix: the blocklist exemption must not relax the pure no-web mode. + + Without an active blocklist the pre-existing filter drops every non-function + tool — Anthropic ``web_search_*`` included — exactly as before this PR. + """ + from benchflow.providers.litellm_logging import callback_module_source + + monkeypatch.delenv(EGRESS_BLOCKED_URLS_ENV, raising=False) + monkeypatch.setenv("BENCHFLOW_DISALLOW_WEB_TOOLS", "1") + ns: dict[str, object] = {} + exec(callback_module_source(), ns) + logger = ns["proxy_handler_instance"] + data = { + "model": "claude-fable-5-1", + "messages": [], + "tools": [ + {"type": "web_search_20250305", "name": "web_search"}, + {"name": "bash", "input_schema": {"type": "object"}}, + ], + } + cleaned = await logger.async_pre_call_hook(None, None, data, "anthropic_messages") + assert [t["name"] for t in cleaned["tools"]] == ["bash"] + + +# ------------------------------------------------------------------ rollout seams + + +def test_session_factory_agents_are_refused_under_blocklist(): + """Review fix: a host-side session-factory agent is outside the sandbox + proxy and firewall, so a blocklist run must refuse it, not run open.""" + from benchflow.rollout._setup import _refuse_session_factory_under_blocklist + + b = EgressBlocklist(rules=("openreview.net",)) + _refuse_session_factory_under_blocklist("claude-agent-acp", None, b) + _refuse_session_factory_under_blocklist("omnigent", "omnigent.acp:factory", None) + with pytest.raises(RuntimeError, match="runs on the host"): + _refuse_session_factory_under_blocklist("omnigent", "omnigent.acp:factory", b) + + +@pytest.mark.asyncio +async def test_install_phase_starts_proxy_after_web_policy(tmp_path, monkeypatch): + """Review fix: the proxy is started in the generic install phase (before + any connect protocol), right after the harness web-tool policy.""" + import benchflow.rollout as rollout_mod + from benchflow.rollout import Rollout, RolloutConfig + + config = RolloutConfig.from_legacy( + task_path=tmp_path / "task", + agent="claude-agent-acp", + prompts=[None], + sandbox_user="agent", + ) + trial = Rollout(config) + trial._env = MagicMock() + trial._env.exec = AsyncMock(return_value=MagicMock(stdout="/workspace\n")) + trial._rollout_dir = tmp_path / "trial" + trial._rollout_dir.mkdir() + trial._rollout_paths = MagicMock() + trial._task = MagicMock() + trial._effective_locked = [] + trial._agent_cwd = "/app" + trial._agent_env = {} + trial._disallow_web_tools = False + trial._egress_blocklist = EgressBlocklist(rules=("openreview.net",)) + order: list[str] = [] + + async def _policy(*args, **kwargs): + order.append("web-policy") + + async def _start(env, blocklist): + assert blocklist == trial._egress_blocklist + order.append("egress-proxy") + + monkeypatch.setattr(rollout_mod, "start_egress_proxy", _start) + planes = trial._planes + monkeypatch.setattr(planes, "install_agent", AsyncMock(return_value=MagicMock())) + monkeypatch.setattr(planes, "setup_sandbox_user", AsyncMock(return_value="/app")) + monkeypatch.setattr(planes, "write_credential_files", AsyncMock()) + monkeypatch.setattr(planes, "upload_subscription_auth", AsyncMock()) + monkeypatch.setattr(planes, "apply_web_tool_policy", _policy) + monkeypatch.setattr(planes, "snapshot_build_config", AsyncMock()) + monkeypatch.setattr(planes, "seed_verifier_workspace", AsyncMock()) + monkeypatch.setattr(planes, "deploy_skills", AsyncMock()) + monkeypatch.setattr(planes, "lockdown_paths", AsyncMock()) + monkeypatch.setattr(planes, "link_skill_paths", AsyncMock()) + + await trial.install_agent() + + assert order == ["web-policy", "egress-proxy"] + + +def test_container_policy_follows_task_even_when_primary_is_oracle(): + """Review P0 #3: an oracle primary is exempt from the blocklist itself, but + the container must still be provisioned (NET_ADMIN, sandbox-local model + proxy) for the role agents that connect_as() later.""" + from benchflow.rollout._setup import _resolve_agent_network_policy + + task = SimpleNamespace( + config=TaskConfig.model_validate( + { + "sandbox": { + "network_mode": "blocklist", + "blocked_urls": ["openreview.net"], + } + } + ) + ) + b = EgressBlocklist(rules=("openreview.net",)) + + assert _resolve_agent_network_policy( + task, primary_agent="claude-agent-acp", disallow_web_tools=False + ) == (b, True) + # Oracle primary: no proxy routing for the oracle, container still provisioned. + assert _resolve_agent_network_policy( + task, primary_agent="oracle", disallow_web_tools=False + ) == (None, True) + # A no-web run wins over the blocklist and is itself a container policy. + assert _resolve_agent_network_policy( + task, primary_agent="claude-agent-acp", disallow_web_tools=True + ) == (None, True) + # No policy declared anywhere. + plain = SimpleNamespace(config=TaskConfig.model_validate({})) + assert _resolve_agent_network_policy( + plain, primary_agent="oracle", disallow_web_tools=False + ) == (None, False) + + +def test_proxy_source_caps_concurrent_connections(): + """Review P1 #5: the accept loop is bounded by a connection semaphore.""" + source = egress_proxy_source() + assert 'MAX_CONNECTIONS = int(CONFIG.get("max_connections", 256))' in source + assert "_SLOTS = threading.BoundedSemaphore(MAX_CONNECTIONS)" in source + assert "_SLOTS.acquire()" in source and "_SLOTS.release()" in source + + +# ------------------------------------------------------------------ batch preflight + resume + + +def _write_task(tasks_dir: Path, name: str, sandbox_toml: str = "") -> Path: + task_dir = tasks_dir / name + task_dir.mkdir(parents=True) + (task_dir / "instruction.md").write_text("do it\n") + (task_dir / "task.toml").write_text( + 'version = "1.0"\n[verifier]\ntimeout_sec = 60\n[agent]\ntimeout_sec = 60\n' + f"[environment]\n{sandbox_toml}" + ) + return task_dir + + +def test_preflight_names_every_task_that_conflicts_with_block_url(tmp_path): + """Review round 3: --block-url against a no-network/allowlist task must fail + before ANY rollout starts, naming the offenders, not after 49 tasks ran.""" + from benchflow.evaluation import ( + EvaluationConfig, + NetworkPolicyPreflightError, + _expected_network_policies, + ) + + tasks = tmp_path / "tasks" + open_task = _write_task(tasks, "task-open") + closed = _write_task(tasks, "task-closed", 'network_mode = "no-network"\n') + listed = _write_task( + tasks, "task-allow", 'network_mode = "allowlist"\nallowed_hosts = ["x.org"]\n' + ) + overlay = { + "sandbox": {"network_mode": "blocklist", "blocked_urls": ["openreview.net"]} + } + cfg = EvaluationConfig(agent="claude-agent-acp", config_override=overlay) + + with pytest.raises(NetworkPolicyPreflightError) as info: + _expected_network_policies([open_task, closed, listed], cfg) + message = str(info.value) + assert "2 task(s)" in message + assert "task-closed" in message and "task-allow" in message + assert "task-open" not in message + + # Devin review: a task that pins agent.network_mode would have swallowed + # the run-level blocklist silently; preflight must name it instead. + pinned = _write_task(tasks, "task-agent-public") + (pinned / "task.toml").write_text( + (pinned / "task.toml") + .read_text() + .replace( + "[agent]\ntimeout_sec = 60\n", + '[agent]\ntimeout_sec = 60\nnetwork_mode = "public"\n', + ) + ) + with pytest.raises(NetworkPolicyPreflightError, match="task-agent-public"): + _expected_network_policies([open_task, pinned], cfg) + + # Without the conflicting tasks the overlay resolves to a per-task policy. + expected = _expected_network_policies([open_task], cfg) + assert expected["task-open"]["mode"] == "blocklist" + assert expected["task-open"]["blocked_urls"] == ["openreview.net"] + # The oracle records no policy of its own (it is exempt), like the rollout. + oracle = EvaluationConfig(agent="oracle", config_override=overlay) + assert _expected_network_policies([open_task], oracle) == {"task-open": None} + # A no-web run wins over the blocklist in Rollout, so the expectation must + # be null too — otherwise resume would refuse a perfectly consistent job. + no_web = EvaluationConfig( + agent="claude-agent-acp", config_override=overlay, self_gen_no_internet=True + ) + assert _expected_network_policies([open_task], no_web) == {"task-open": None} + + +def test_preflight_refuses_blocklist_task_on_non_enforcing_backend(tmp_path): + from benchflow.evaluation import ( + EvaluationConfig, + NetworkPolicyPreflightError, + _expected_network_policies, + ) + + tasks = tmp_path / "tasks" + task = _write_task( + tasks, + "task-bl", + 'network_mode = "blocklist"\nblocked_urls = ["openreview.net"]\n', + ) + with pytest.raises(NetworkPolicyPreflightError, match="not enforced by modal"): + _expected_network_policies([task], EvaluationConfig(environment="modal")) + assert _expected_network_policies( + [task], EvaluationConfig(environment="docker") + ) == {"task-bl": EgressBlocklist(rules=("openreview.net",)).config_metadata()} + + +def test_resume_refuses_mixing_open_and_blocklisted_scores(tmp_path): + """Review round 3: a job resumed with a different network policy must + refuse, exactly like an agent mismatch.""" + from benchflow.evaluation import ( + EvaluationConfig, + ResumeMismatchError, + _check_resume_mismatch, + ) + + job_dir = tmp_path / "jobs" / "job" + rollout = job_dir / "task-a__r1" + rollout.mkdir(parents=True) + policy = EgressBlocklist(rules=("openreview.net",)).config_metadata() + (rollout / "config.json").write_text( + json.dumps( + { + "agent": "claude-agent-acp", + "task_path": "task-a", + "network_policy": policy, + } + ) + ) + cfg = EvaluationConfig(agent="claude-agent-acp") + + # Same posture: fine. Open network on resume: refused. Unknown task: ignored. + _check_resume_mismatch(job_dir, cfg, {"task-a": policy}) + with pytest.raises(ResumeMismatchError, match="network_policy"): + _check_resume_mismatch(job_dir, cfg, {"task-a": None}) + _check_resume_mismatch(job_dir, cfg, {"task-other": None}) + + # A pre-feature config.json (no key) counts as open network. + (rollout / "config.json").write_text( + json.dumps({"agent": "claude-agent-acp", "task_path": "task-a"}) + ) + _check_resume_mismatch(job_dir, cfg, {"task-a": None}) + with pytest.raises(ResumeMismatchError, match="different experiments"): + _check_resume_mismatch(job_dir, cfg, {"task-a": policy}) + # A provenance-recorded task_path ("benchmarks/physics/task-a") still maps + # to the expectation keyed by directory name (review micro-fix). + (rollout / "config.json").write_text( + json.dumps( + { + "agent": "claude-agent-acp", + "task_path": "benchmarks/physics/task-a", + "network_policy": policy, + } + ) + ) + _check_resume_mismatch(job_dir, cfg, {"task-a": policy}) + with pytest.raises(ResumeMismatchError, match="task-a"): + _check_resume_mismatch(job_dir, cfg, {"task-a": None}) + # No expectations supplied: the legacy agent/loop-only behaviour is kept. + _check_resume_mismatch(job_dir, cfg) + + +@pytest.mark.asyncio +async def test_disconnect_downloads_egress_log_whenever_proxy_ran( + tmp_path, monkeypatch +): + """Review #1: an oracle primary's env carries no blocklist marker, yet a + role agent may have run under the blocklist via connect_as(); the audit + log download is gated on the proxy having started, not on the env.""" + import benchflow.rollout as rollout_mod + from benchflow.rollout import Rollout + + downloads: list[Path] = [] + + async def _download(env, target_dir): + downloads.append(target_dir) + + monkeypatch.setattr(rollout_mod, "download_egress_log", _download) + trial = Rollout.__new__(Rollout) + trial._is_session_factory = False + trial._capture_partial_acp_trajectory = lambda: None + trial._collect_native_acp_usage = None + trial._acp_client = None + trial._session = None + trial._session_adapter = None + trial._agent_launch = "agent-binary" + trial._env = MagicMock() + trial._env.exec = AsyncMock(return_value=MagicMock(return_code=0)) + trial._rollout_paths = SimpleNamespace(agent_dir=tmp_path / "agent") + trial._agent_env = {} # oracle primary: no blocklist marker here + trial._active_role = None + trial._phase = "connected" + + trial._egress_proxy_started = True + await trial.disconnect() + assert downloads == [tmp_path / "agent"] + + downloads.clear() + trial._egress_proxy_started = False + await trial.disconnect() + assert downloads == [] + + +def test_proxy_source_tries_ipv4_first_and_falls_back_across_candidates(): + """Review #2: docker's default bridge has no IPv6 route; a v6-first + resolve must not turn into a 502 for dual-stack hosts.""" + source = egress_proxy_source() + assert ( + "candidates.sort(key=lambda item: 0 if item[0] == socket.AF_INET else 1)" + in source + ) + assert "for family, sockaddr in resolve_upstream(host, port):" in source + assert "last_err = exc" in source + + +def test_proxy_source_uses_ip_san_for_ip_literal_hosts(): + """Review #5: RFC 5280 requires IP: SANs for IP-literal names.""" + source = egress_proxy_source() + assert 'san = "IP:%s" % host' in source + assert 'san = "DNS:%s" % host' in source + assert 'open(sys.argv[1], encoding="utf-8")' in source + + +@pytest.mark.asyncio +async def test_connect_acp_applies_firewall_before_launch_under_blocklist(tmp_path): + """Devin review: with the proxy already up, the UID firewall must precede + the agent process so startup traffic cannot escape before the handshake.""" + from unittest.mock import patch + + from benchflow.acp.client import ACPClient + from benchflow.acp.runtime import connect_acp + + events: list[str] = [] + mock_session = MagicMock(session_id="s1") + mock_init = MagicMock(agent_info=None) + mock_acp = AsyncMock(spec=ACPClient) + mock_acp.connect = AsyncMock(side_effect=lambda: events.append("connect")) + mock_acp.initialize = AsyncMock(return_value=mock_init) + mock_acp.session_new = AsyncMock( + side_effect=lambda *a, **k: events.append("session_new") or mock_session + ) + mock_acp.set_config_option = AsyncMock() + mock_acp.close = AsyncMock() + + async def enforce(*args, **kwargs): + events.append("firewall") + + async def verify(*args, **kwargs): + events.append("verify") + + agent_env = apply_blocklist_env( + {"BENCHFLOW_PROVIDER_BASE_URL": "http://127.0.0.1:4000/v1"}, + EgressBlocklist(rules=("openreview.net",)), + ) + with ( + patch("benchflow.acp.runtime.ContainerTransport", return_value=MagicMock()), + patch("benchflow.acp.runtime.ACPClient", return_value=mock_acp), + patch( + "benchflow.acp.runtime.enforce_agent_egress_firewall", + new_callable=AsyncMock, + side_effect=enforce, + ) as mock_firewall, + patch( + "benchflow.acp.runtime.verify_egress_blocklist", + new_callable=AsyncMock, + side_effect=verify, + ), + ): + await connect_acp( + env=AsyncMock(), + agent="openhands", + agent_launch="openhands acp", + agent_env=agent_env, + sandbox_user="agent", + model=None, + rollout_dir=tmp_path, + environment="docker", + agent_cwd="/app", + ) + + assert events == ["firewall", "connect", "session_new", "verify"] + mock_firewall.assert_awaited_once() + + +# ------------------------------------------------------------------ CLI / docker / config + + +def test_blocklist_override_folds_urls_into_c_axis_overlay(tmp_path): + from benchflow._utils.config_override import blocklist_override + + assert blocklist_override(None, None, None) is None + assert blocklist_override('{"agent":{"timeout_sec":5}}', [], None) == ( + '{"agent":{"timeout_sec":5}}' + ) + listing = tmp_path / "hidden.txt" + listing.write_text( + "# hidden papers\nhttps://arxiv.org/abs/2401.12345 # v1\n\nopenreview.net\n" + ) + + merged = json.loads( + blocklist_override( + '{"agent":{"timeout_sec":5}}', + ["openreview.net", " arxiv.org/pdf/2401.12345 "], + listing, + ) + ) + assert merged["agent"] == {"timeout_sec": 5} + assert merged["sandbox"]["network_mode"] == "blocklist" + assert merged["sandbox"]["blocked_urls"] == [ + "openreview.net", + "arxiv.org/pdf/2401.12345", + "https://arxiv.org/abs/2401.12345", + ] + # The overlay re-validates through the task schema at rollout time. + cfg = TaskConfig.model_validate({"sandbox": merged["sandbox"]}) + assert cfg.sandbox.blocked_urls[-1] == "arxiv.org/abs/2401.12345" + + +def test_docker_stacks_net_admin_overlay_only_under_agent_network_policy(tmp_path): + from benchflow.sandbox._compose import COMPOSE_NET_ADMIN_PATH + from benchflow.sandbox.docker import DockerSandbox + from benchflow.task.config import SandboxConfig + + env_dir = tmp_path / "environment" + env_dir.mkdir() + (env_dir / "Dockerfile").write_text("FROM scratch\n") + + def _paths(policy: bool): + sandbox = DockerSandbox( + environment_dir=env_dir, + environment_name="t", + session_id="s", + rollout_paths=None, + task_env_config=SandboxConfig(), + agent_network_policy=policy, + ) + return sandbox._docker_compose_paths + + assert COMPOSE_NET_ADMIN_PATH in _paths(True) + assert COMPOSE_NET_ADMIN_PATH not in _paths(False) + assert "NET_ADMIN" in COMPOSE_NET_ADMIN_PATH.read_text() + + +def test_write_config_records_network_policy(tmp_path): + from benchflow.rollout._results import _write_config + from benchflow.skill_policy import resolve_task_skill_policy + + b = EgressBlocklist(rules=("arxiv.org/abs/2401.12345",)) + task = tmp_path / "task" + task.mkdir() + _write_config( + tmp_path, + task_path=task, + agent="claude-agent-acp", + model="m", + environment="docker", + skill_policy=resolve_task_skill_policy( + task_path=task, + skill_mode="no-skill", + runtime_skills_dir=None, + declared_sandbox_skills_dir=None, + ), + sandbox_user="agent", + context_root=None, + timeout=60, + started_at=datetime(2026, 1, 1), + agent_env={}, + network_policy=b.config_metadata(), + ) + recorded = json.loads((tmp_path / "config.json").read_text())["network_policy"] + assert recorded["mode"] == "blocklist" + assert recorded["blocked_urls"] == ["arxiv.org/abs/2401.12345"] + assert recorded["tls_inspection_hosts"] == ["arxiv.org"] + assert recorded["blocked_status"] == BLOCKED_STATUS + + +def test_proxy_source_defaults_to_refusing_private_networks_outside_local_subnets(): + """Devin review: RFC1918 is refused by default; only the container's own + on-link subnets (compose network) are allowed, plus an explicit opt-in.""" + source = egress_proxy_source() + assert ( + 'ALLOW_PRIVATE_NETWORKS = bool(CONFIG.get("allow_private_networks", False))' + in source + ) + assert "_local_ipv4_subnets" in source and "/proc/net/route" in source + assert "any(ip in net for net in _LOCAL_SUBNETS)" in source + assert 'header(headers, "expect") or "").lower() == "100-continue"' in source + + +def test_local_subnet_parser_reads_on_link_routes_only(): + """Exercise the embedded route parser on a captured /proc/net/route.""" + ns: dict = {} + src = egress_proxy_source() + start = src.index("def _local_ipv4_subnets(") + end = src.index("\n_LOCAL_SUBNETS = ") + exec("import ipaddress\n" + src[start:end], ns) + table = ( + "Iface\tDestination\tGateway\tFlags\tRefCnt\tUse\tMetric\tMask\tMTU\tWindow\tIRTT\n" + "eth0\t00000000\t010011AC\t0003\t0\t0\t0\t00000000\t0\t0\t0\n" # default via gw + "eth0\t000011AC\t00000000\t0001\t0\t0\t0\t0000FFFF\t0\t0\t0\n" # 172.17.0.0/16 on-link + "lo\t0000007F\t00000000\t0001\t0\t0\t0\t000000FF\t0\t0\t0\n" # 127.0.0.0/8 + ) + nets = ns["_local_ipv4_subnets"](table) + assert [str(n) for n in nets] == ["172.17.0.0/16"] + + +# ------------------------------------------------------------------ real proxy (Linux) + + +def _free_port() -> int: + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +class _Origin(http.server.BaseHTTPRequestHandler): + def do_GET(self): + body = f"origin:{self.path}".encode() + self.send_response(200) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_POST(self): + length = int(self.headers.get("Content-Length") or 0) + received = self.rfile.read(length) + body = f"origin:{self.path}:{len(received)}".encode() + self.send_response(200) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): # silence + pass + + +def _serve(handler, port, tls_cert=None, tls_key=None): + server = http.server.ThreadingHTTPServer(("127.0.0.1", port), handler) + if tls_cert: + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ctx.load_cert_chain(tls_cert, tls_key) + server.socket = ctx.wrap_socket(server.socket, server_side=True) + threading.Thread(target=server.serve_forever, daemon=True).start() + return server + + +def _openssl_selfsigned( + directory: Path, name: str, subj: str, *ext: str +) -> tuple[Path, Path]: + key, crt = directory / f"{name}.key", directory / f"{name}.crt" + cmd = [ + "openssl", + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + str(key), + "-out", + str(crt), + "-days", + "2", + "-subj", + subj, + ] + for e in ext: + cmd += ["-addext", e] + subprocess.run(cmd, check=True, capture_output=True) + return crt, key + + +def _wait_port(port: int, timeout: float = 10.0) -> None: + deadline = time.time() + timeout + while time.time() < deadline: + try: + with socket.create_connection(("127.0.0.1", port), timeout=0.5): + return + except OSError: + time.sleep(0.05) + raise RuntimeError(f"port {port} never opened") + + +@pytest.fixture +def running_proxy(tmp_path): + """Start the real proxy script with a CA, a plain origin, and a TLS origin.""" + ca_dir = tmp_path / "ca" + ca_dir.mkdir() + _openssl_selfsigned( + ca_dir, + "ca", + "/CN=BenchFlow Test CA", + "basicConstraints=critical,CA:TRUE", + "keyUsage=critical,keyCertSign,cRLSign", + ) + srv_crt, srv_key = _openssl_selfsigned( + tmp_path, "origin", "/CN=localhost", "subjectAltName=DNS:localhost,IP:127.0.0.1" + ) + http_port, tls_port, proxy_port = _free_port(), _free_port(), _free_port() + origin = _serve(_Origin, http_port) + tls_origin = _serve(_Origin, tls_port, srv_crt, srv_key) + + script = tmp_path / "egress_proxy.py" + script.write_text(egress_proxy_source()) + log_path = tmp_path / "egress.jsonl" + config = tmp_path / "policy.json" + config.write_text( + json.dumps( + { + "rules": ["blocked.example", "localhost/secret"], + "port": proxy_port, + "log_path": str(log_path), + "blocked_status": BLOCKED_STATUS, + "ca_dir": str(ca_dir), + "upstream_ca_file": str(srv_crt), + # Test origins live on 127.0.0.1; production refuses loopback. + "allow_loopback_upstream": True, + # Small cap: every request in these tests must release its slot. + "max_connections": 3, + } + ) + ) + proc = subprocess.Popen( + [sys.executable, str(script), str(config)], + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + ) + try: + _wait_port(proxy_port) + yield SimpleNamespace( + proxy_port=proxy_port, + http_port=http_port, + tls_port=tls_port, + ca_crt=ca_dir / "ca.crt", + origin_crt=srv_crt, + log_path=log_path, + ) + finally: + proc.terminate() + try: + stderr = proc.communicate(timeout=5)[1].decode() + except subprocess.TimeoutExpired: + proc.kill() + stderr = "" + origin.shutdown() + tls_origin.shutdown() + if stderr.strip(): + print("proxy stderr:", stderr) + + +def _via_proxy(p, url: str, *, trust: Path | None = None) -> tuple[int, bytes]: + proxy = f"http://127.0.0.1:{p.proxy_port}" + handlers: list = [urllib.request.ProxyHandler({"http": proxy, "https": proxy})] + if trust is not None: + ctx = ssl.create_default_context(cafile=str(trust)) + handlers.append(urllib.request.HTTPSHandler(context=ctx)) + opener = urllib.request.build_opener(*handlers) + try: + with opener.open(url, timeout=15) as resp: + return resp.status, resp.read() + except urllib.error.HTTPError as exc: + return exc.code, exc.read() + + +@NEEDS_OPENSSL +def test_proxy_filters_plain_http_by_host_and_path(running_proxy): + p = running_proxy + assert _via_proxy(p, f"http://127.0.0.1:{p.http_port}/ok") == (200, b"origin:/ok") + assert _via_proxy(p, f"http://localhost:{p.http_port}/ok")[0] == 200 + # Path rule on localhost: /secret* hidden, everything else served. + assert ( + _via_proxy(p, f"http://localhost:{p.http_port}/secret/paper.pdf")[0] + == BLOCKED_STATUS + ) + # Host rule: no DNS lookup is even attempted for a blocked host. + status, body = _via_proxy(p, "http://blocked.example/anything") + assert status == BLOCKED_STATUS + assert body == b"Not Found\n" + + +@NEEDS_OPENSSL +def test_proxy_rejects_connect_to_blocked_host(running_proxy): + p = running_proxy + with socket.create_connection(("127.0.0.1", p.proxy_port), timeout=5) as s: + s.sendall( + b"CONNECT sub.blocked.example:443 HTTP/1.1\r\nHost: sub.blocked.example:443\r\n\r\n" + ) + reply = s.recv(4096) + assert reply.startswith(f"HTTP/1.1 {BLOCKED_STATUS} ".encode()) + + +@NEEDS_OPENSSL +def test_proxy_tunnels_https_without_inspection_for_hosts_without_path_rules( + running_proxy, +): + p = running_proxy + # 127.0.0.1 carries no path rule => opaque tunnel; the client sees the + # ORIGIN certificate (trusting it directly), not a proxy-minted one, and + # a /secret path is NOT filtered because the rule is bound to "localhost". + status, body = _via_proxy( + p, f"https://127.0.0.1:{p.tls_port}/secret", trust=p.origin_crt + ) + assert (status, body) == (200, b"origin:/secret") + + +@NEEDS_OPENSSL +def test_proxy_inspects_tls_for_hosts_with_path_rules(running_proxy): + p = running_proxy + # localhost carries a path rule => TLS is terminated with a leaf signed by + # the run CA; the client trusts that CA (as the sandbox bundle would). + assert _via_proxy(p, f"https://localhost:{p.tls_port}/ok", trust=p.ca_crt) == ( + 200, + b"origin:/ok", + ) + status, body = _via_proxy( + p, f"https://localhost:{p.tls_port}/secret/x", trust=p.ca_crt + ) + assert (status, body) == (BLOCKED_STATUS, b"Not Found\n") + # Without the CA the inspected host must fail verification — proof the + # tunnel really was terminated rather than passed through. + with pytest.raises(urllib.error.URLError): + _via_proxy(p, f"https://localhost:{p.tls_port}/ok", trust=p.origin_crt) + + events = [json.loads(line) for line in p.log_path.read_text().splitlines()] + blocked = [e for e in events if e["event"] == "block"] + assert {(e["host"], e.get("rule")) for e in blocked} >= { + ("localhost", "localhost/secret") + } + assert all("secret" not in json.dumps(e) or e["event"] == "block" for e in events) + + +@NEEDS_OPENSSL +def test_proxy_refuses_link_local_and_loopback_upstreams(tmp_path): + """Review fix: the root-run proxy must not be an SSRF hop. Link-local + (cloud metadata) is always refused; loopback is refused unless a test + explicitly allows it.""" + port = _free_port() + origin_port = _free_port() + origin = _serve(_Origin, origin_port) + script = tmp_path / "egress_proxy.py" + script.write_text(egress_proxy_source()) + log_path = tmp_path / "egress.jsonl" + config = tmp_path / "policy.json" + config.write_text( + json.dumps( + { + "rules": ["blocked.example"], + "port": port, + "log_path": str(log_path), + "blocked_status": BLOCKED_STATUS, + "ca_dir": None, + } + ) + ) + proc = subprocess.Popen( + [sys.executable, str(script), str(config)], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + try: + _wait_port(port) + p = SimpleNamespace(proxy_port=port) + status, body = _via_proxy(p, "http://169.254.169.254/latest/meta-data/") + assert status == 403 + assert b"link-local" in body + status, body = _via_proxy(p, f"http://127.0.0.1:{origin_port}/ok") + assert status == 403 + assert b"loopback" in body + with socket.create_connection(("127.0.0.1", port), timeout=5) as s: + s.sendall(b"CONNECT 169.254.169.254:443 HTTP/1.1\r\n\r\n") + assert s.recv(4096).startswith(b"HTTP/1.1 403 ") + events = [json.loads(line) for line in log_path.read_text().splitlines()] + assert {e["event"] for e in events} >= {"refuse"} + assert all(e["event"] != "allow" for e in events) + finally: + proc.terminate() + proc.wait(timeout=5) + origin.shutdown() + + +def test_proxy_source_pins_http11_alpn_and_vets_upstream_addresses(): + """Review fixes #2/#3: ALPN is limited to HTTP/1.1 on both TLS legs and + every resolved upstream address is vetted before connecting.""" + source = egress_proxy_source() + assert source.count('set_alpn_protocols(["http/1.1"])') == 2 + assert "ipaddress.ip_address(sockaddr[0])" in source + assert "is_link_local" in source and "is_loopback" in source + + +@NEEDS_OPENSSL +def test_proxy_reuses_one_leaf_key_across_inspected_hosts(running_proxy): + """Review fix #5: leaf certs share one key; only the x509 signing is per host.""" + p = running_proxy + assert _via_proxy(p, f"https://localhost:{p.tls_port}/ok", trust=p.ca_crt)[0] == 200 + leaf_dir = p.ca_crt.parent / "leaf" + assert (leaf_dir / "leaf.key").exists() + assert (leaf_dir / "localhost.crt").exists() + assert not (leaf_dir / "localhost.key").exists() + + +@NEEDS_OPENSSL +def test_proxy_blocks_encoded_and_dotted_paths_and_host_header_spoofs(running_proxy): + """Review round 5 (WAF bypass): the live proxy must hide ``localhost/secret`` + through percent-encoding, slash runs, dot segments, and an IP-literal URL + that smuggles the real host in the Host header.""" + p = running_proxy + for path in ( + "/%73ecret/paper.pdf", + "//secret///paper.pdf", + "/other/../secret/paper.pdf", + "/SECRET/x", + ): + status, _ = _via_proxy(p, f"http://localhost:{p.http_port}{path}") + assert status == BLOCKED_STATUS, path + # Control: a neighbouring path is still served. + assert _via_proxy(p, f"http://localhost:{p.http_port}/secre/t")[0] == 200 + + # Host-header spoof: URL names the IP (no rule), header names localhost. + with socket.create_connection(("127.0.0.1", p.proxy_port), timeout=5) as sock: + sock.sendall( + f"GET http://127.0.0.1:{p.http_port}/secret/paper.pdf HTTP/1.1\r\n" + "Host: localhost\r\nConnection: close\r\n\r\n".encode() + ) + reply = sock.recv(4096) + assert reply.startswith(f"HTTP/1.1 {BLOCKED_STATUS} ".encode()) + # Same request with an honest Host header is not affected by the rule. + with socket.create_connection(("127.0.0.1", p.proxy_port), timeout=5) as sock: + sock.sendall( + f"GET http://127.0.0.1:{p.http_port}/secret/paper.pdf HTTP/1.1\r\n" + f"Host: 127.0.0.1:{p.http_port}\r\nConnection: close\r\n\r\n".encode() + ) + reply = sock.recv(4096) + # The stdlib test origin answers HTTP/1.0; only the status matters here. + assert reply.split(b" ", 2)[1] == b"200" + + +@NEEDS_OPENSSL +def test_proxy_inspected_tls_also_normalizes_paths(running_proxy): + p = running_proxy + for path in ("/%73ecret/x", "/a/../secret/x", "//secret//x"): + status, _ = _via_proxy( + p, f"https://localhost:{p.tls_port}{path}", trust=p.ca_crt + ) + assert status == BLOCKED_STATUS, path + assert _via_proxy(p, f"https://localhost:{p.tls_port}/ok", trust=p.ca_crt)[0] == 200 + + +@NEEDS_OPENSSL +def test_inspected_tls_negotiates_http11_with_an_h2_capable_client(running_proxy): + """Functional ALPN check: a client that offers h2 first must be negotiated + down to HTTP/1.1 inside the inspected tunnel, and a plain HTTP/1.1 request + over that session must succeed. Guards the review fix for h2 clients.""" + p = running_proxy + ctx = ssl.create_default_context(cafile=str(p.ca_crt)) + ctx.set_alpn_protocols(["h2", "http/1.1"]) + with socket.create_connection(("127.0.0.1", p.proxy_port), timeout=10) as raw: + raw.sendall( + f"CONNECT localhost:{p.tls_port} HTTP/1.1\r\n" + f"Host: localhost:{p.tls_port}\r\n\r\n".encode() + ) + assert raw.recv(4096).startswith(b"HTTP/1.1 200 ") + with ctx.wrap_socket(raw, server_hostname="localhost") as tls: + assert tls.selected_alpn_protocol() == "http/1.1" + tls.sendall( + b"GET /ok HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n" + ) + reply = b"" + while True: + chunk = tls.recv(65536) + if not chunk: + break + reply += chunk + assert reply.split(b" ", 2)[1] == b"200" + assert reply.endswith(b"origin:/ok") + + +@NEEDS_OPENSSL +def test_proxy_refuses_private_addresses_outside_local_subnets(tmp_path): + """Devin review: a private address that is not on one of the container's + own subnets is refused without any connect attempt.""" + port = _free_port() + script = tmp_path / "egress_proxy.py" + script.write_text(egress_proxy_source()) + config = tmp_path / "policy.json" + config.write_text( + json.dumps( + { + "rules": ["blocked.example"], + "port": port, + "log_path": str(tmp_path / "egress.jsonl"), + "blocked_status": BLOCKED_STATUS, + "ca_dir": None, + # TEST-NET-3 style private target that no lab subnet uses. + "allow_local_subnets": True, + } + ) + ) + proc = subprocess.Popen( + [sys.executable, str(script), str(config)], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + try: + _wait_port(port) + p = SimpleNamespace(proxy_port=port) + status, body = _via_proxy(p, "http://10.255.255.254/") + assert status == 403 and b"private" in body + status, body = _via_proxy(p, "http://192.168.255.254/") + assert status == 403 and b"private" in body + finally: + proc.terminate() + proc.wait(timeout=5) + + +@NEEDS_OPENSSL +def test_proxy_answers_expect_100_continue_and_relays_the_body(running_proxy): + """Devin review: a client that waits for 100 Continue must get it from the + proxy (with Expect stripped upstream) and then have its body relayed.""" + p = running_proxy + body = b"x" * 4096 + with socket.create_connection(("127.0.0.1", p.proxy_port), timeout=10) as sock: + sock.sendall( + f"POST http://localhost:{p.http_port}/upload HTTP/1.1\r\n" + f"Host: localhost:{p.http_port}\r\nContent-Length: {len(body)}\r\n" + "Expect: 100-continue\r\nConnection: close\r\n\r\n".encode() + ) + interim = sock.recv(4096) + assert interim.startswith(b"HTTP/1.1 100 Continue") + sock.sendall(body) + reply = b"" + while True: + chunk = sock.recv(65536) + if not chunk: + break + reply += chunk + assert reply.split(b" ", 2)[1] == b"200" + assert reply.endswith(b"origin:/upload:4096") diff --git a/tests/test_internet_policy.py b/tests/test_internet_policy.py index f34e2daaf..8fb378461 100644 --- a/tests/test_internet_policy.py +++ b/tests/test_internet_policy.py @@ -18,8 +18,14 @@ def _wire_fake_planes(trial: Rollout) -> MagicMock: planes = MagicMock() - planes.agent_launch.side_effect = lambda agent, *, disallow_web_tools: ( - f"{agent} --no-web" if disallow_web_tools else agent + # Mirrors RolloutPlanes.agent_launch: the blocklist keyword landed with + # the egress-blocklist PR, so the fake accepts both policy knobs. + planes.agent_launch.side_effect = ( + lambda agent, *, disallow_web_tools, blocklist_web_tools=False: ( + f"{agent} --no-web" + if disallow_web_tools + else (f"{agent} --blocklist" if blocklist_web_tools else agent) + ) ) planes.resolve_agent_env.side_effect = lambda _agent, _model, env: env or {} planes.ensure_litellm_runtime = AsyncMock( diff --git a/tests/test_runtime_capabilities.py b/tests/test_runtime_capabilities.py index 555cc257d..872beffda 100644 --- a/tests/test_runtime_capabilities.py +++ b/tests/test_runtime_capabilities.py @@ -1129,30 +1129,53 @@ def test_sandbox_launch_allows_supported_legacy_task(tmp_path: Path) -> None: assert result is docker_sandbox.return_value -def test_validator_reports_blocklist_as_runtime_gap() -> None: - """Guards the blocklist schema PR: parsed blocklists stay unsupported until enforced.""" - config = TaskConfig.model_validate( - { - "agent": { - "network_mode": "blocklist", - "blocked_urls": ["arxiv.org/abs/2401.12345"], - }, - "sandbox": { - "network_mode": "blocklist", - "blocked_urls": ["openreview.net"], - }, - } - ) +_BLOCKLIST_TASK = { + "agent": { + "network_mode": "blocklist", + "blocked_urls": ["arxiv.org/abs/2401.12345"], + }, + "sandbox": { + "network_mode": "blocklist", + "blocked_urls": ["openreview.net"], + }, +} - issues = validate_task_runtime_support(config, sandbox="docker") + +@pytest.mark.parametrize("sandbox", ["docker", "daytona"]) +def test_validator_accepts_blocklist_on_enforcing_backends(sandbox: str) -> None: + """Guards the egress-blocklist PR: docker and daytona enforce network_mode='blocklist'.""" + config = TaskConfig.model_validate(_BLOCKLIST_TASK) + + assert validate_task_runtime_support(config, sandbox=sandbox) == [] + + +@pytest.mark.parametrize("sandbox", ["modal", "apple-container", "agentcore"]) +def test_validator_reports_blocklist_gap_on_other_backends(sandbox: str) -> None: + """Guards the egress-blocklist PR: backends without a root proxy + agent-UID + firewall must refuse a blocklist task instead of running it open.""" + config = TaskConfig.model_validate(_BLOCKLIST_TASK) + + issues = validate_task_runtime_support(config, sandbox=sandbox) assert [(issue.path, issue.reason) for issue in issues] == [ ( "agent.network_mode", - "network blocklists are parsed but not enforced per sandbox", + f"network_mode='blocklist' is not enforced by {sandbox}", ), ( "sandbox.network_mode", - "network blocklists are parsed but not enforced per sandbox", + f"network_mode='blocklist' is not enforced by {sandbox}", ), ] + + +def test_validator_reports_verifier_blocklist_as_unsupported() -> None: + """Guards the egress-blocklist PR: the blocklist is an agent-phase control only.""" + config = TaskConfig.model_validate( + {"verifier": {"network_mode": "blocklist", "blocked_urls": ["arxiv.org"]}} + ) + + issues = validate_task_runtime_support(config, sandbox="docker") + + assert [issue.path for issue in issues] == ["verifier.network_mode"] + assert "agent phase only" in issues[0].reason