From da06b727c557fe28d189174c38c2d7de5ce499f8 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 12 Nov 2026 17:03:12 -0800 Subject: [PATCH 1/7] fix(gateway): take the control token for the data plane and each MCP server's own endpoint A gateway booted with a control token now refuses, with 401, a call without it to /agentenv, to /svc//agentenv and its extensions, and to an MCP server's own endpoint behind /svc (/svc/mcp-/mcp, or a site's MCP endpoint when the site is an MCP server). Paths are matched after dot-segment normalization. The rest of the proxy, the website routes and the gateway's own /mcp stay open, and a gateway without a token guards nothing. MCP server and website loads, and a server reset, send the token their deploy record carries, so a load outside a task run (no grant) still reaches the data plane. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/env/envs/_deployment.py | 8 +- src/agent_env/env/envs/mcp_server.py | 6 +- src/agent_env/env/envs/website.py | 5 +- src/agent_env/env/gateway/constants.py | 5 +- src/agent_env/env/gateway/gateway.py | 17 ++- .../env/envs/multi_env_local_test.py | 4 +- tst/integration/env/gateway/gateway_test.py | 53 +++++----- tst/unit/env/envs/test_env_card_loads.py | 16 +++ tst/unit/env/envs/test_server_reset.py | 9 ++ tst/unit/env/gateway/control_token_test.py | 100 +++++++++++++++++- 10 files changed, 184 insertions(+), 39 deletions(-) diff --git a/src/agent_env/env/envs/_deployment.py b/src/agent_env/env/envs/_deployment.py index 428c88528..1ae76c656 100644 --- a/src/agent_env/env/envs/_deployment.py +++ b/src/agent_env/env/envs/_deployment.py @@ -141,8 +141,9 @@ async def load_by_signed_url(env: MCPServerEnv | WebsiteEnv, environment_artifac if urls[path] is None: raise RuntimeError(f"{deployed_by}, so its server fetches {file.object_url} itself, and {type(stores[path]).__name__} " "can't sign a URL for it; loading into such an env needs an object store that signs URLs (S3, Cloud Storage)") - await protocol_v1.reset_data(base_url, timeout=timeout) - await protocol_v1.add_data(base_url, file_parts(environment_artifact, files, urls), timeout=timeout) + headers = env._deployed.control_headers_for(base_url) + await protocol_v1.reset_data(base_url, timeout=timeout, headers=headers) + await protocol_v1.add_data(base_url, file_parts(environment_artifact, files, urls), timeout=timeout, headers=headers) async def reset_server_data(env: MCPServerEnv | WebsiteEnv, *, mcp: bool) -> bool: @@ -161,7 +162,8 @@ async def reset_server_data(env: MCPServerEnv | WebsiteEnv, *, mcp: bool) -> boo child = env._deployed.child_env(name, mcp=mcp) if env._deployed is not None and env._deployed.environment_card else None if child is not None and child.serves(METHOD_RESET) is False: return False - await protocol_v1.reset_data(base_url, timeout=DATA_PLANE_LOAD_TIMEOUT_S) + await protocol_v1.reset_data(base_url, timeout=DATA_PLANE_LOAD_TIMEOUT_S, + headers=env._deployed.control_headers_for(base_url) if env._deployed is not None else None) elif builtin is None: return False else: diff --git a/src/agent_env/env/envs/mcp_server.py b/src/agent_env/env/envs/mcp_server.py index bb959deb3..891d94bef 100644 --- a/src/agent_env/env/envs/mcp_server.py +++ b/src/agent_env/env/envs/mcp_server.py @@ -186,8 +186,10 @@ async def load_environment_artifact(self, environment_artifact: EnvironmentArtif # reset_data previously took the client default (30s), tighter than the load it # precedes -- dropping and recreating a large service's schemas can plausibly # exceed that. Give it the same budget as the load it belongs to. - await protocol_v1.reset_data(base_url, timeout=timeout) - await protocol_v1.add_data(base_url, file_parts(environment_artifact, files, staged), timeout=timeout) + headers = self._deployed.control_headers_for(base_url) if self._deployed is not None else None + await protocol_v1.reset_data(base_url, timeout=timeout, headers=headers) + await protocol_v1.add_data(base_url, file_parts(environment_artifact, files, staged), timeout=timeout, + headers=headers) finally: if folder is not None: await self._remove_from_container(folder) diff --git a/src/agent_env/env/envs/website.py b/src/agent_env/env/envs/website.py index 001ab5d46..c0991513d 100644 --- a/src/agent_env/env/envs/website.py +++ b/src/agent_env/env/envs/website.py @@ -321,9 +321,10 @@ async def load_environment_artifact(self, environment_artifact: EnvironmentArtif try: staged = {path: f"file://{await self._copy_artifact_into_container(file, paths[path])}" for path, file in files.items()} - await protocol_v1.reset_data(base_url) + headers = self._deployed.control_headers_for(base_url) if self._deployed is not None else None + await protocol_v1.reset_data(base_url, headers=headers) await protocol_v1.add_data(base_url, file_parts(environment_artifact, files, staged), - timeout=DATA_PLANE_LOAD_TIMEOUT_S) + timeout=DATA_PLANE_LOAD_TIMEOUT_S, headers=headers) finally: if folder is not None: await self._remove_from_container(folder) diff --git a/src/agent_env/env/gateway/constants.py b/src/agent_env/env/gateway/constants.py index 06ecbcfd3..d2425becf 100644 --- a/src/agent_env/env/gateway/constants.py +++ b/src/agent_env/env/gateway/constants.py @@ -176,8 +176,9 @@ def sources(names: list[str]) -> str: TOOL_DISABLE_ACTION = "disable" TOOL_ENABLE_ACTION = "enable" # The harness's routes that take the deploy's control token: the ones that change tool access, triggers and -# notifications. Every other route stays open for now, the clock's writes included: callers outside agent-env set the -# clock over plain HTTP and don't send the token yet. +# notifications. The gateway also takes it for the data plane and for an MCP server's own endpoint behind its proxy +# (Gateway._takes_control_token). Every other route stays open for now, the clock's writes included: callers outside +# agent-env set the clock over plain HTTP and don't send the token yet. CONTROL_ROUTES = frozenset({ f"/tools/{TOOL_DISABLE_ACTION}", f"/tools/{TOOL_ENABLE_ACTION}", "/triggers/register", "/triggers/remove", "/triggers/clear", diff --git a/src/agent_env/env/gateway/gateway.py b/src/agent_env/env/gateway/gateway.py index 20ae28735..e953b4c32 100644 --- a/src/agent_env/env/gateway/gateway.py +++ b/src/agent_env/env/gateway/gateway.py @@ -85,6 +85,7 @@ declared_site_names, fetch_child_card, forward, + is_child_mcp_path, listed_url, lowered, operations, @@ -542,7 +543,7 @@ def _asgi_app(self): async def app(scope, receive, send): if self._control_bearer and scope["type"] in ("http", "websocket"): headers = scope.get("headers", ()) - if scope["type"] == "http" and scope["path"] in CONTROL_ROUTES and not hmac.compare_digest( + if scope["type"] == "http" and self._takes_control_token(scope["path"]) and not hmac.compare_digest( dict(headers).get(b"authorization", b""), self._control_bearer): response = self._json_response({"ok": False, "error": "this route takes the env's control token"}, 401) await response(scope, receive, send) @@ -566,6 +567,20 @@ async def app(scope, receive, send): return app + def _takes_control_token(self, path: str) -> bool: + """Whether a request to ``path`` takes the deploy's control token: a control route; the data plane, its + extensions included, the env's own or a service's behind /svc; or an MCP server's own endpoint behind /svc, + which skips the gateway's tool rules, trajectory and triggers. Judged as the proxy forwards a path, after + dot-segment normalisation. A site's /website route never reaches its data plane.""" + if path in CONTROL_ROUTES or is_child_mcp_path(path, RPC_PATH): + return True + prefix, _, rest = path.lstrip("/").partition("/") + service, _, rest = rest.partition("/") + if prefix != "svc" or not service: + return False + endpoint = self._site_mcp_endpoints.get(service) or ("mcp" if service.startswith("mcp-") else None) + return is_child_mcp_path(rest, RPC_PATH) or (endpoint is not None and is_child_mcp_path(rest, endpoint)) + @asynccontextmanager async def _session_lifespan(self, mcp: FastMCP): """Once per inbound MCP session: the session's key joins the caller, and a connect gets to heal a failed diff --git a/tst/integration/env/envs/multi_env_local_test.py b/tst/integration/env/envs/multi_env_local_test.py index 92535eede..50c369c60 100644 --- a/tst/integration/env/envs/multi_env_local_test.py +++ b/tst/integration/env/envs/multi_env_local_test.py @@ -53,14 +53,14 @@ async def test_a_multi_env_deploys_restores_loads_and_tears_down_on_local_compos assert isinstance(restored, MultiEnv) and restored._sandbox.work_dir == work_dir await restored.load_environment_universe_artifact(_items_universe(uid)) base_url = await legacy_protocol.v1_base_url(deployed, deployed.gateway_url, "items") - assert (await protocol_v1.get_data(base_url)).model_dump()["parts"][0]["data"] == {"items": ["snap-x", "snap-y"]} + assert (await protocol_v1.get_data(base_url, headers=deployed.control_headers)).model_dump()["parts"][0]["data"] == {"items": ["snap-x", "snap-y"]} slack_state = f"{legacy_protocol.environment_base_url(deployed.gateway_url, 'slack')}/export-state" assert len(httpx.get(slack_state, timeout=30).json()["channels"]) == 10 # A multi env without computers resets its servers, each as it resets alone, and fails if one doesn't. await ResetEnvTaskStep(id="reset", version=None, env_id=env.id, fail_task_on_error=True).execute( TaskStepContext(deployed_envs=[deployed])) - assert (await protocol_v1.get_data(base_url)).model_dump()["parts"][0]["data"] == {"items": []} + assert (await protocol_v1.get_data(base_url, headers=deployed.control_headers)).model_dump()["parts"][0]["data"] == {"items": []} assert httpx.get(slack_state, timeout=30).json()["channels"] == [] await restored.close() diff --git a/tst/integration/env/gateway/gateway_test.py b/tst/integration/env/gateway/gateway_test.py index f381bba7c..e02e295c9 100644 --- a/tst/integration/env/gateway/gateway_test.py +++ b/tst/integration/env/gateway/gateway_test.py @@ -984,7 +984,8 @@ async def assert_generated_data_via_step(client, gateway_url, tools, tool_names) build_step = BuildMcpCliTaskStep(id=f"build-cli-{email_env.id}", version=None, env_id=email_env.id, command_name="email") ctx = TaskStepContext(deployed_envs=[result]) - await build_step.execute(ctx) + with protocol_v1.authorized({result.environment_url: result.control_headers}): + await build_step.execute(ctx) cli_ref = ctx.metadata["cli_artifact"] cli_artifact = Artifact.get(cli_ref["id"], version=cli_ref["version"]) @@ -1915,7 +1916,7 @@ async def test_gateway_changes_name_the_request_behind_a_write_through_the_rest_ async with httpx.AsyncClient() as client: seen = (await client.get(f"{gw}/changes", params={"limit": 10000}, timeout=30)).json()["changes"] after_id = seen[-1]["id"] if seen else 0 - async with httpx.AsyncClient(headers={AGENT_ENV_ROLE_HEADER: "carol@example.com"}, timeout=60) as as_carol: + async with httpx.AsyncClient(headers={AGENT_ENV_ROLE_HEADER: "carol@example.com", **result.control_headers}, timeout=60) as as_carol: async with streamable_http_client(f"{gw}/svc/mcp-email/mcp", http_client=as_carol) as (read, write, _): async with ClientSession(read, write) as session: await session.initialize() @@ -2149,7 +2150,7 @@ async def recording_send(self, request, *args, **kwargs): monkeypatch.undo() routes = [r for r in sent if r[1].startswith("/svc/")] assert routes == [("POST", "/svc/mcp-renameditems/agentenv")] * 2, routes - assert (await protocol_v1.get_data(base)).parts[0].data == {"items": ["r"]} + assert (await protocol_v1.get_data(base, headers=result.control_headers)).parts[0].data == {"items": ["r"]} finally: await env.close() @@ -2200,23 +2201,23 @@ async def test_gateway_with_agentenv_environment(agentenv_items_env, monkeypatch } }, } - assert (await protocol_v1.get_data(base)).parts[0].data == {"items": []} - await protocol_v1.add_data(base, [DataPart(data={"items": ["a", "b"]})]) - assert (await protocol_v1.get_data(base)).parts[0].data == {"items": ["a", "b"]} + assert (await protocol_v1.get_data(base, headers=result.control_headers)).parts[0].data == {"items": []} + await protocol_v1.add_data(base, [DataPart(data={"items": ["a", "b"]})], headers=result.control_headers) + assert (await protocol_v1.get_data(base, headers=result.control_headers)).parts[0].data == {"items": ["a", "b"]} - await protocol_v1.reset_data(base) - await protocol_v1.add_data(base, [FilePart(file={"uri": "file:///data/seed.json", "mimeType": "application/json"})]) - assert (await protocol_v1.get_data(base)).parts[0].data == {"items": ["seeded-a", "seeded-b"]} + await protocol_v1.reset_data(base, headers=result.control_headers) + await protocol_v1.add_data(base, [FilePart(file={"uri": "file:///data/seed.json", "mimeType": "application/json"})], headers=result.control_headers) + assert (await protocol_v1.get_data(base, headers=result.control_headers)).parts[0].data == {"items": ["seeded-a", "seeded-b"]} - async with httpx.AsyncClient(timeout=30) as client: + async with httpx.AsyncClient(timeout=30, headers=result.control_headers) as client: r = await client.post(f"{base}/agentenv", json={"jsonrpc": "2.0", "id": 1, "method": "data/add", "params": {"parts": []}}) assert r.status_code == 200 and r.json()["error"]["code"] == -32602 r = await client.post(f"{base}/agentenv", json={"jsonrpc": "2.0", "id": 1, "method": "data/bogus"}) assert r.json()["error"]["code"] == -32601 - await protocol_v1.reset_data(base) - assert (await protocol_v1.get_data(base)).parts[0].data == {"items": []} - await protocol_v1.add_data(base, [DataPart(data={"items": ["x"]})]) + await protocol_v1.reset_data(base, headers=result.control_headers) + assert (await protocol_v1.get_data(base, headers=result.control_headers)).parts[0].data == {"items": []} + await protocol_v1.add_data(base, [DataPart(data={"items": ["x"]})], headers=result.control_headers) async def assert_tool_sees_items(session, tools_result, tool_names): assert "list_items" in tool_names, f"Expected list_items tool, got {tool_names}" @@ -2254,7 +2255,7 @@ async def assert_tool_sees_items(session, tools_result, tool_names): ), ) await env.load_environment_artifact(items_artifact) - assert (await protocol_v1.get_data(base)).parts[0].data == {"items": ["y", "z"]} + assert (await protocol_v1.get_data(base, headers=result.control_headers)).parts[0].data == {"items": ["y", "z"]} # Non-JSON EnvironmentArtifact loads via the file:// FilePart branch (no content_type special-case). csv_artifact = EnvironmentArtifact.put( @@ -2269,11 +2270,11 @@ async def assert_tool_sees_items(session, tools_result, tool_names): ), ) await env.load_environment_artifact(csv_artifact) - assert (await protocol_v1.get_data(base)).parts[0].data == {"items": ["file:text/csv:a,b\n1,2"]} + assert (await protocol_v1.get_data(base, headers=result.control_headers)).parts[0].data == {"items": ["file:text/csv:a,b\n1,2"]} # A file name with shell syntax in it is staged and loaded as it is, and runs nothing on the sandbox host. await _load_shell_named_artifact(env, "items", monkeypatch, tmp_path) - assert (await protocol_v1.get_data(base)).parts[0].data == {"items": ["shell-named"]} + assert (await protocol_v1.get_data(base, headers=result.control_headers)).parts[0].data == {"items": ["shell-named"]} assert not (tmp_path / "pwned").exists() # Environment Card extension end-to-end (urn:agentenv:set-errors/v1): a server-local @@ -2302,7 +2303,7 @@ async def _count_list_items_errors(n: int) -> int: assert await _count_list_items_errors(20) == 0 # no injection yet ack = await protocol_v1.invoke_extension( base, card, "urn:agentenv:set-errors/v1", - {"tool_name": "list_items", "error_rate": 0.75}, + {"tool_name": "list_items", "error_rate": 0.75}, headers=result.control_headers, ) assert ack == {"tool_name": "list_items", "error_rate": 0.75} assert await _count_list_items_errors(20) >= 1 # P(0 errors) = 0.25**20 ~ 1e-12 @@ -2352,13 +2353,13 @@ async def assert_list_items_disabled(session, tools_result, tool_names): ack = await protocol_v1.invoke_extension( result.gateway_url, items_child, "urn:agentenv:set-errors/v1", - {"tool_name": "list_items", "error_rate": 0.5}, + {"tool_name": "list_items", "error_rate": 0.5}, headers=result.control_headers, ) assert ack == {"tool_name": "list_items", "error_rate": 0.5} - await protocol_v1.reset_data(result.gateway_url) - await protocol_v1.add_data(result.gateway_url, [DataPart(data={"items": ["env-added"]})]) - assert (await protocol_v1.get_data(result.gateway_url)).parts[0].data == {"items": ["env-added"]} + await protocol_v1.reset_data(result.gateway_url, headers=result.control_headers) + await protocol_v1.add_data(result.gateway_url, [DataPart(data={"items": ["env-added"]})], headers=result.control_headers) + assert (await protocol_v1.get_data(result.gateway_url, headers=result.control_headers)).parts[0].data == {"items": ["env-added"]} finally: await env.close() @@ -2386,7 +2387,7 @@ async def test_snapshot_agent_state_v1_and_fallback(multi_env_items_email, items env_card = await protocol_v1.get_card(result.gateway_url) assert protocol_v1.find_child(env_card, "items") is not None assert protocol_v1.find_child(env_card, "email") is None - async with httpx.AsyncClient(timeout=30) as client: + async with httpx.AsyncClient(timeout=30, headers=result.control_headers) as client: r = await client.post(f"{result.gateway_url}/agentenv", json={"jsonrpc": "2.0", "id": 1, "method": "data/reset"}) assert "exactly one" in r.json()["error"]["message"] @@ -2453,7 +2454,7 @@ async def test_gateway_with_website_env_v1(agentenv_website_env, monkeypatch, tm base = f"{result.gateway_url}/svc/webitems" assert await protocol_v1.supports_v1(base) is True - assert (await protocol_v1.get_data(base)).parts[0].data == {"items": []} + assert (await protocol_v1.get_data(base, headers=result.control_headers)).parts[0].data == {"items": []} assert result.environment_card_url == f"{result.gateway_url}/.well-known/agent-env.json" env_card = await protocol_v1.get_card(result.gateway_url) @@ -2472,7 +2473,7 @@ async def test_gateway_with_website_env_v1(agentenv_website_env, monkeypatch, tm ), ) await env.load_environment_artifact(web_artifact) - assert (await protocol_v1.get_data(base)).parts[0].data == {"items": ["w1", "w2"]} + assert (await protocol_v1.get_data(base, headers=result.control_headers)).parts[0].data == {"items": ["w1", "w2"]} # Non-JSON EnvironmentArtifact loads via the file:// FilePart branch. csv_artifact = EnvironmentArtifact.put( @@ -2487,10 +2488,10 @@ async def test_gateway_with_website_env_v1(agentenv_website_env, monkeypatch, tm ), ) await env.load_environment_artifact(csv_artifact) - assert (await protocol_v1.get_data(base)).parts[0].data == {"items": ["file:text/csv:a,b\n1,2"]} + assert (await protocol_v1.get_data(base, headers=result.control_headers)).parts[0].data == {"items": ["file:text/csv:a,b\n1,2"]} await _load_shell_named_artifact(env, "webitems", monkeypatch, tmp_path) - assert (await protocol_v1.get_data(base)).parts[0].data == {"items": ["shell-named"]} + assert (await protocol_v1.get_data(base, headers=result.control_headers)).parts[0].data == {"items": ["shell-named"]} assert not (tmp_path / "pwned").exists() finally: await env.close() diff --git a/tst/unit/env/envs/test_env_card_loads.py b/tst/unit/env/envs/test_env_card_loads.py index 4e2b4301a..5ef0bdb13 100644 --- a/tst/unit/env/envs/test_env_card_loads.py +++ b/tst/unit/env/envs/test_env_card_loads.py @@ -56,6 +56,22 @@ async def test_a_child_env_on_the_card_loads_through_its_v1_data_plane_with_no_c assert json.loads(sent[1].content)["params"]["parts"][0]["file"]["uri"] == "file:///data/email.json" +@pytest.mark.asyncio +async def test_a_load_sends_its_records_control_token_with_no_run_to_grant_it(sent): + deployed = _record(await _card("mcp-email", "shop")) + deployed.control_token = "s3cret" + web = _website() + web._sandbox, web._gateway_url, web._deployed = MagicMock(), _GW, deployed + web._copy_artifact_into_container = AsyncMock(return_value="/tmp/data/shop.json") + web._env_provider = MagicMock(spec=EnvironmentGatewayProvider, install_changelog_triggers=AsyncMock()) + + await _mcp(deployed).load_environment_artifact(_artifact("email")) + await web.load_environment_artifact(_artifact("shop")) + + assert [(str(r.url), r.headers.get("authorization")) for r in sent] == [ + *[(f"{_CARD}/svc/mcp-email/agentenv", "Bearer s3cret")] * 2, *[(f"{_CARD}/svc/shop/agentenv", "Bearer s3cret")] * 2] + + @pytest.mark.asyncio async def test_a_child_env_missing_from_the_card_takes_the_legacy_path(sent): # email serves no card, so the gateway leaves it out of the composed card. diff --git a/tst/unit/env/envs/test_server_reset.py b/tst/unit/env/envs/test_server_reset.py index a73236dc1..42adeec6b 100644 --- a/tst/unit/env/envs/test_server_reset.py +++ b/tst/unit/env/envs/test_server_reset.py @@ -66,6 +66,15 @@ async def test_a_server_serving_data_reset_is_emptied_through_it_and_its_changel server.triggers.assert_awaited_once_with("email") +@pytest.mark.asyncio +async def test_a_reset_sends_its_records_control_token_with_no_run_to_grant_it(server): + env = _server(_card("data/reset", "data/add", "data/get")) + env._deployed.control_token = "s3cret" + + assert await _deployment.reset_server_data(env, mcp=True) is True + assert [request.headers.get("authorization") for request in server.requests] == ["Bearer s3cret"] + + @pytest.mark.asyncio async def test_a_server_whose_card_declares_no_data_reset_takes_no_load_and_is_left_as_it_is(server): assert await _deployment.reset_server_data(_server(_card()), mcp=True) is False diff --git a/tst/unit/env/gateway/control_token_test.py b/tst/unit/env/gateway/control_token_test.py index b561ba1c9..eeaa24b16 100644 --- a/tst/unit/env/gateway/control_token_test.py +++ b/tst/unit/env/gateway/control_token_test.py @@ -1,10 +1,15 @@ -"""The gateway's control routes take the deploy's control token when it has one; every other route stays open.""" +"""The gateway's control routes, the data plane and an MCP server's own endpoint behind its proxy take the deploy's +control token when it has one; every other route stays open.""" from __future__ import annotations +import asyncio +from urllib.parse import unquote, urlsplit + import httpx import pytest from starlette.responses import JSONResponse +from agent_env.env.gateway import rest_proxy as rest_proxy_module from agent_env.env.gateway.constants import CONTROL_ROUTES from agent_env.env.gateway.gateway import Gateway @@ -19,6 +24,18 @@ "/clock/set-time": "PUT", "/clock/advance": "PUT", "/clock/clear": "POST"} +# Behind the proxy: the data plane, its extensions, each service's included, and an MCP server's own endpoint, however +# the path is spelt on its way to the gateway. +GUARDED = [("POST", "/agentenv"), ("POST", "/svc/mcp-svc/agentenv"), ("POST", "/svc/mcp-svc/agentenv/ext/grant"), + ("POST", "/svc/mcp-svc/x/../agentenv"), ("POST", "/svc/mcp-svc/%61gentenv"), ("POST", "/svc/shop/agentenv"), + ("POST", "/svc/mcp-svc/mcp"), ("GET", "/svc/mcp-svc/mcp/"), + ("DELETE", "/svc/mcp-svc/./mcp"), ("POST", "/svc/mcp-svc//mcp")] +PROXIED_OPEN = [("GET", "/svc/mcp-svc/.well-known/agent-env.json"), ("GET", "/svc/mcp-svc/openapi.yaml"), + ("GET", "/svc/mcp-svc/export-state"), ("POST", "/svc/shop/api/orders"), ("POST", "/svc/shop/mcp"), + ("GET", "/website/shop/index.html")] +_RealAsyncClient = httpx.AsyncClient + + def _client(token: str | None) -> httpx.AsyncClient: gw = Gateway(host="127.0.0.1", port=0, server_name="unit", internal_mcp_servers=[], control_token=token) return httpx.AsyncClient(transport=httpx.ASGITransport(app=gw._asgi_app()), base_url="http://gateway") @@ -88,3 +105,84 @@ async def test_a_gateway_started_without_a_token_checks_none(path): async with _client(None) as client: response = await client.request(CONTROL[path], path, json={}) assert response.status_code != 401 + + +class _Backing: + """The services behind the gateway's proxy: each answers every request, and the paths reaching them are kept.""" + + def __init__(self, monkeypatch): + self.paths: list[str] = [] + transport = httpx.MockTransport(self._answer) + monkeypatch.setattr(rest_proxy_module.httpx, "AsyncClient", lambda **kw: _RealAsyncClient(transport=transport, **kw)) + + def _answer(self, request: httpx.Request) -> httpx.Response: + self.paths.append(request.url.path) + + async def body(): + yield b'{"jsonrpc": "2.0", "id": 1, "result": {}}' + + return httpx.Response(200, headers={"content-type": "application/json"}, content=body()) + + +async def _proxied(token: str | None, method: str, path: str, headers: dict[str, str]) -> int: + """The status of one request through the gateway's served app, its path as uvicorn hands it: percent-decoded, its + dot segments and doubled slashes kept.""" + gw = Gateway(host="127.0.0.1", port=0, server_name="unit", internal_mcp_servers=[], control_token=token, + rest_proxy_urls={"mcp-svc": "http://svc", "shop": "http://shop-backend"}, + website_urls={"shop": "http://shop-backend"}) + url = urlsplit(path) + scope = {"type": "http", "asgi": {"version": "3.0", "spec_version": "2.3"}, "http_version": "1.1", "method": method, + "scheme": "http", "path": unquote(url.path), "raw_path": url.path.encode(), "query_string": b"", + "root_path": "", "server": ("gateway", 80), "client": ("agent", 1), + "headers": [(b"content-type", b"application/json"), + *((name.lower().encode(), value.encode()) for name, value in headers.items())]} + received, started = False, [] + + async def receive(): + nonlocal received + if not received: + received = True + return {"type": "http.request", "body": b'{"jsonrpc": "2.0", "id": 1, "method": "data/reset"}', + "more_body": False} + await asyncio.Event().wait() + + async def send(message): + if message["type"] == "http.response.start": + started.append(message["status"]) + + await gw._asgi_app()(scope, receive, send) + return started[0] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("method, path", GUARDED) +@pytest.mark.parametrize("headers", [{}, {"Authorization": "Bearer wrong"}], ids=["none", "wrong"]) +async def test_the_data_plane_and_a_servers_own_endpoint_behind_the_proxy_refuse_a_call_without_the_token( + monkeypatch, method, path, headers): + backing = _Backing(monkeypatch) + assert await _proxied(TOKEN, method, path, headers) == 401 + assert backing.paths == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("method, path", GUARDED) +async def test_the_data_plane_and_a_servers_own_endpoint_behind_the_proxy_answer_a_call_with_the_token( + monkeypatch, method, path): + _Backing(monkeypatch) + assert await _proxied(TOKEN, method, path, {"Authorization": f"Bearer {TOKEN}"}) != 401 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("method, path", PROXIED_OPEN) +async def test_the_rest_of_the_proxy_stays_open(monkeypatch, method, path): + backing = _Backing(monkeypatch) + assert await _proxied(TOKEN, method, path, {}) != 401 + assert backing.paths + + +@pytest.mark.asyncio +@pytest.mark.parametrize("method, path", GUARDED) +async def test_a_gateway_started_without_a_token_guards_nothing_behind_its_proxy(monkeypatch, method, path): + _Backing(monkeypatch) + assert await _proxied(None, method, path, {}) != 401 + From 2f95335a4c1ad7d217c8cfb2b8f38067d031698d Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 13 Nov 2026 09:30:29 -0800 Subject: [PATCH 2/7] fix(gateway): guard only the servers the gateway fronts, at each MCP server's own endpoint The guard took the gateway's token for /svc//agentenv, so a child env holding its own control token, a computer, refused the calls agent-env makes with that token through the gateway (the CDP connect step's browser/v1 cdp). It now covers each MCP server and website the gateway fronts (token_paths): its data plane, and an MCP server's own endpoint at the path its mcp_url gives instead of assuming /mcp. A deploy asks each site's website/v1 ready through the gateway before any run grants the token, so it now sends the gateway's control headers. MultiEnv grants its record's token, at the card's address and the gateway's, around its children's loads and resets, which covers a name an MCP server and a website share, whose children keep no record. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/env/envs/multi_env.py | 18 ++++++++-- src/agent_env/env/gateway/gateway.py | 35 +++++++++++++----- .../env_providers/env_gateway_provider.py | 16 +++++---- tst/unit/env/envs/test_env_card_loads.py | 21 +++++++++++ tst/unit/env/gateway/control_token_test.py | 36 +++++++++++++++---- .../env_gateway_provider_test.py | 8 +++-- 6 files changed, 108 insertions(+), 26 deletions(-) diff --git a/src/agent_env/env/envs/multi_env.py b/src/agent_env/env/envs/multi_env.py index e8c7f0ca1..3239c4ed3 100644 --- a/src/agent_env/env/envs/multi_env.py +++ b/src/agent_env/env/envs/multi_env.py @@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Any, Callable, ClassVar, Optional import yaml +from agentenv_protocol import client as protocol_v1 logger = logging.getLogger(__name__) @@ -461,11 +462,13 @@ async def load_environment_artifact(self, environment_artifact: EnvironmentArtif # MCP servers are checked first; if a website shares the same environment_name, the MCP server wins. for env in self.mcp_server_envs: if env.environment_name == environment_artifact.environment_name: - await env.load_environment_artifact(environment_artifact) + with _record_grant(self._deployed): + await env.load_environment_artifact(environment_artifact) return for env in self.website_envs: if env.environment_name == environment_artifact.environment_name: - await env.load_environment_artifact(environment_artifact) + with _record_grant(self._deployed): + await env.load_environment_artifact(environment_artifact) return if (computer := self._computers.holding(environment_artifact.environment_name)) is not None: await computer.load_environment_artifact(environment_artifact) @@ -1067,7 +1070,8 @@ async def reset(self, deployed: DeployedEnv) -> None: for env in services.mcp_server_envs), *((f"website {env.environment_name!r}", reset_server_data(env, mcp=False)) for env in services.website_envs)] - results = await _multi_computers.each(self.id, "reset", resets) + with _record_grant(deployed): + results = await _multi_computers.each(self.id, "reset", resets) for (child, _), result in zip(resets, results): if result is False: logger.info(f"multi env '{self.id}': {child} serves no reset and takes no load, so it stays as it is") @@ -1122,6 +1126,14 @@ def _gateway_state_provider(env: MultiEnv) -> DatabaseStateProvider | None: return env._env_provider._state_provider if isinstance(env._env_provider, EnvironmentGatewayProvider) else None +def _record_grant(deployed: Optional[DeployedEnv]): + """The grant of ``deployed``'s control token to calls under its address and its gateway's, for a child env that a + name it shares with another keeps from the record (``_hand_record_to_child_envs``), whose calls go to the gateway.""" + if deployed is None: + return protocol_v1.authorized({}) + return protocol_v1.authorized({url: deployed.control_headers for url in (deployed.environment_url, gateway_url_of(deployed))}) + + def _hand_record_to_child_envs(env: MultiEnv, deployed: DeployedEnv) -> None: """Give each child env the record its loads read the card from; a name an MCP server and a website share keeps the live probe, since the card can't tell them apart by name.""" shared = {c.environment_name for c in env.mcp_server_envs} & {c.environment_name for c in env.website_envs} diff --git a/src/agent_env/env/gateway/gateway.py b/src/agent_env/env/gateway/gateway.py index e953b4c32..55c3f458d 100644 --- a/src/agent_env/env/gateway/gateway.py +++ b/src/agent_env/env/gateway/gateway.py @@ -177,6 +177,7 @@ def __init__( self._site_names = {site: lowered(names) for site, names in (site_names or {}).items()} self._listed_by_name = frozenset(listed_by_name) self._site_mcp_endpoints = site_mcp_endpoints(internal_mcp_servers, self.website_urls, self.rest_proxy_urls) + self._token_paths = token_paths(internal_mcp_servers, self.website_urls, self.rest_proxy_urls) self._site_cards: dict[str, SiteCard] = {} self._withheld = frozenset(name.strip().lower().encode() for name in provider_headers if name.strip()) self._browsers = BrowserRoutes(self.rest_proxy_urls, self.REST_PROXY_TIMEOUT, self._log_event, @@ -568,18 +569,14 @@ async def app(scope, receive, send): return app def _takes_control_token(self, path: str) -> bool: - """Whether a request to ``path`` takes the deploy's control token: a control route; the data plane, its - extensions included, the env's own or a service's behind /svc; or an MCP server's own endpoint behind /svc, - which skips the gateway's tool rules, trajectory and triggers. Judged as the proxy forwards a path, after - dot-segment normalisation. A site's /website route never reaches its data plane.""" + """Whether a request to ``path`` takes the deploy's control token: a control route, the env's data plane, or + one of ``token_paths`` behind /svc. Judged as the proxy forwards a path, after dot-segment normalisation. A + site's /website route never reaches its data plane.""" if path in CONTROL_ROUTES or is_child_mcp_path(path, RPC_PATH): return True prefix, _, rest = path.lstrip("/").partition("/") service, _, rest = rest.partition("/") - if prefix != "svc" or not service: - return False - endpoint = self._site_mcp_endpoints.get(service) or ("mcp" if service.startswith("mcp-") else None) - return is_child_mcp_path(rest, RPC_PATH) or (endpoint is not None and is_child_mcp_path(rest, endpoint)) + return prefix == "svc" and any(is_child_mcp_path(rest, guarded) for guarded in self._token_paths.get(service, ())) @asynccontextmanager async def _session_lifespan(self, mcp: FastMCP): @@ -1732,6 +1729,28 @@ def site_mcp_endpoints(servers: list[InternalMCPServer], website_urls: dict[str, return endpoints +def token_paths(servers: list[InternalMCPServer], website_urls: dict[str, str], + rest_proxy_urls: dict[str, str]) -> dict[str, tuple[str, ...]]: + """The paths under /svc// that take the gateway's control token, by key, for each MCP server and website the + gateway fronts: its data plane, extensions included, and an MCP server's own MCP endpoint, at the path its + ``mcp_url`` gives (``/mcp`` when none is under the route's address), which skips the gateway's tool rules, + trajectory and triggers. A child env of its own, a computer, takes its own token, so none of its paths are here.""" + sites = site_mcp_endpoints(servers, website_urls, rest_proxy_urls) + paths = {} + for key, base in rest_proxy_urls.items(): + if key in website_urls: + paths[key] = (RPC_PATH, *((sites[key],) if key in sites else ())) + continue + name = route_environment_name(key, base, servers, website=False) + if name == key: + continue + endpoints = [resolved.route_path for server in servers if server.name == name + if (resolved := card_url(base, server.mcp_url)) is not None and resolved.own_server + and resolved.route_path is not None] + paths[key] = (RPC_PATH, *(endpoints or ["/mcp"])) + return paths + + class CardURL(NamedTuple): """A URL a site's card names, as the site at an address serves it: its path and query; the path a request on the site's route reaches it at, its path under the address's (None when it isn't under it); and whether it is on the diff --git a/src/agent_env/providers/env_providers/env_gateway_provider.py b/src/agent_env/providers/env_providers/env_gateway_provider.py index 16437dc0d..40b9953d0 100644 --- a/src/agent_env/providers/env_providers/env_gateway_provider.py +++ b/src/agent_env/providers/env_providers/env_gateway_provider.py @@ -18,6 +18,7 @@ from typing import TYPE_CHECKING, Optional import httpx +from agentenv_protocol.client import control_headers from agentenv_protocol.website import ( SESSION_SECRET_ENV, SITE_READY_TIMEOUT_S, @@ -984,7 +985,7 @@ async def _stand_up(_spec): if card is None: raise RuntimeError("Tunnel did not become ready in time") if website_configs: - await _wait_for_websites(gateway_url, website_configs) + await _wait_for_websites(gateway_url, website_configs, headers=control_headers(control_token)) card = await self._wait_for_tunnel(gateway_url, timeout=120) or card card_read_at_utc = datetime.now(timezone.utc).isoformat() await self._refuse_tool_name_clashes(gateway_url) @@ -1781,8 +1782,10 @@ class WebsiteContractError(RuntimeError): async def _wait_for_websites(gateway_url: str, website_configs: list[WebsiteConfig], *, - timeout_s: float = WEBSITE_READY_TIMEOUT_S, poll_s: float = WEBSITE_READY_POLL_S) -> None: - """Wait until each site is ready, as its card says through ``/svc//``: by ``website/v1``'s ``ready``, or, + headers: dict[str, str] | None = None, timeout_s: float = WEBSITE_READY_TIMEOUT_S, + poll_s: float = WEBSITE_READY_POLL_S) -> None: + """Wait until each site is ready, as its card says through ``/svc//``, asked with the gateway's control + ``headers``, since its data plane takes them: by ``website/v1``'s ``ready``, or, for a card without it or a site serving none, by its stored health path answering 200, which is deprecated. A site whose answer says how long it may take (``timeout_s``) is waited that long, or its own ``ready_timeout_s`` if longer, and ``WEBSITE_READY_MARGIN_S``; any other its ``ready_timeout_s`` and the margin, or ``timeout_s`` if @@ -1796,7 +1799,7 @@ async def _wait_for_websites(gateway_url: str, website_configs: list[WebsiteConf warned: set[str] = set() while waiting: for name, wc in list(waiting.items()): - ready, why[name], declared = await _website_ready(f"{gateway_url}/svc/{name}", wc.health_path) + ready, why[name], declared = await _website_ready(f"{gateway_url}/svc/{name}", wc.health_path, headers=headers) if declared is not None: limit[name] = max(declared, own[name]) + WEBSITE_READY_MARGIN_S if ready is None and name not in warned: @@ -1814,7 +1817,8 @@ async def _wait_for_websites(gateway_url: str, website_configs: list[WebsiteConf await asyncio.sleep(poll_s) -async def _website_ready(base_url: str, health_path: str) -> tuple[Optional[bool], str, Optional[float]]: +async def _website_ready(base_url: str, health_path: str, *, + headers: dict[str, str] | None = None) -> tuple[Optional[bool], str, Optional[float]]: """Whether the site at ``base_url`` is ready, why not, and how long it says it may take: True or False by the ``WebsiteReadiness`` its card's ``website/v1`` ``ready`` answers, as the protocol's ``website_readiness`` reads it, an answer that isn't one False; with no card or no ``ready`` on it, None once its health path answers 200 and False @@ -1822,7 +1826,7 @@ async def _website_ready(base_url: str, health_path: str) -> tuple[Optional[bool async with httpx.AsyncClient() as client: try: card = _card_of(await client.get(base_url + WELL_KNOWN_PATH, timeout=10), base_url) - readiness = None if card is None else await website_readiness(base_url, card, timeout=10) + readiness = None if card is None else await website_readiness(base_url, card, timeout=10, headers=headers) if readiness is not None: return readiness.ready, "" if readiness.ready else _not_ready_why(readiness), readiness.timeout_s health = await client.get(base_url + health_path, timeout=10) diff --git a/tst/unit/env/envs/test_env_card_loads.py b/tst/unit/env/envs/test_env_card_loads.py index 5ef0bdb13..90a0961d6 100644 --- a/tst/unit/env/envs/test_env_card_loads.py +++ b/tst/unit/env/envs/test_env_card_loads.py @@ -249,6 +249,27 @@ async def test_multi_env_withholds_the_record_from_a_name_an_mcp_server_and_a_we assert [email._deployed, slack._deployed, slack_web._deployed] == [deployed, None, None] +@pytest.mark.asyncio +async def test_a_name_an_mcp_server_and_a_website_share_loads_and_resets_with_the_records_token(sent): + deployed = _record(await _card("mcp-email", "slack")) + deployed.control_token = "s3cret" + provider = MagicMock(get_sandbox=AsyncMock(return_value=MagicMock(mode="vm"))) + with patch("agent_env.env.env.Env.get", side_effect=lambda *_: MultiEnv( + id="multi", version=1, mcp_server_envs=[_mcp_env(), _mcp_env("slack")], website_envs=[_website("slack")])), \ + patch("agent_env.providers.get_env_sandbox_provider", return_value=provider), \ + patch.object(EnvironmentGatewayProvider, "install_changelog_triggers", AsyncMock()): + env = await MultiEnv.from_deployed_env(deployed) + slack = env.mcp_server_envs[1] + slack._copy_artifact_into_container = AsyncMock(return_value="/data/slack.json") + slack._staged_artifact_size = AsyncMock(return_value=None) + await env.load_environment_artifact(_artifact("slack")) + await env.reset(deployed) + + assert sorted((r.url.path, r.headers.get("authorization")) for r in sent if r.url.path.endswith("/agentenv")) == [ + ("/sandbox/sb-1-18765/svc/mcp-email/agentenv", "Bearer s3cret"), + *[("/svc/mcp-slack/agentenv", "Bearer s3cret")] * 3, ("/svc/slack/agentenv", "Bearer s3cret")] + + async def _card(*keys: str) -> dict: """The real gateway's composed card, with a v1 backing server behind each key.""" gw = Gateway(host="127.0.0.1", port=0, server_name="env1234", internal_mcp_servers=[], diff --git a/tst/unit/env/gateway/control_token_test.py b/tst/unit/env/gateway/control_token_test.py index eeaa24b16..00a9cc259 100644 --- a/tst/unit/env/gateway/control_token_test.py +++ b/tst/unit/env/gateway/control_token_test.py @@ -9,8 +9,9 @@ import pytest from starlette.responses import JSONResponse +from agent_env.env.gateway import InternalMCPServer from agent_env.env.gateway import rest_proxy as rest_proxy_module -from agent_env.env.gateway.constants import CONTROL_ROUTES +from agent_env.env.gateway.constants import CONTROL_ROUTES, WELL_KNOWN_PATH from agent_env.env.gateway.gateway import Gateway TOKEN = "s3cret" @@ -112,11 +113,13 @@ class _Backing: def __init__(self, monkeypatch): self.paths: list[str] = [] + self.authorization: list[str | None] = [] transport = httpx.MockTransport(self._answer) monkeypatch.setattr(rest_proxy_module.httpx, "AsyncClient", lambda **kw: _RealAsyncClient(transport=transport, **kw)) def _answer(self, request: httpx.Request) -> httpx.Response: self.paths.append(request.url.path) + self.authorization.append(request.headers.get("authorization")) async def body(): yield b'{"jsonrpc": "2.0", "id": 1, "result": {}}' @@ -124,12 +127,12 @@ async def body(): return httpx.Response(200, headers={"content-type": "application/json"}, content=body()) -async def _proxied(token: str | None, method: str, path: str, headers: dict[str, str]) -> int: +async def _proxied(token: str | None, method: str, path: str, headers: dict[str, str], **children) -> int: """The status of one request through the gateway's served app, its path as uvicorn hands it: percent-decoded, its - dot segments and doubled slashes kept.""" - gw = Gateway(host="127.0.0.1", port=0, server_name="unit", internal_mcp_servers=[], control_token=token, - rest_proxy_urls={"mcp-svc": "http://svc", "shop": "http://shop-backend"}, - website_urls={"shop": "http://shop-backend"}) + dot segments and doubled slashes kept; an MCP server and a website behind it unless ``children`` say otherwise.""" + gw = Gateway(host="127.0.0.1", port=0, server_name="unit", control_token=token, **{ + "internal_mcp_servers": [], "rest_proxy_urls": {"mcp-svc": "http://svc", "shop": "http://shop-backend"}, + "website_urls": {"shop": "http://shop-backend"}, **children}) url = urlsplit(path) scope = {"type": "http", "asgi": {"version": "3.0", "spec_version": "2.3"}, "http_version": "1.1", "method": method, "scheme": "http", "path": unquote(url.path), "raw_path": url.path.encode(), "query_string": b"", @@ -186,3 +189,24 @@ async def test_a_gateway_started_without_a_token_guards_nothing_behind_its_proxy _Backing(monkeypatch) assert await _proxied(None, method, path, {}) != 401 + +@pytest.mark.asyncio +@pytest.mark.parametrize("method, path", [("POST", "/svc/mcp-x/rpc"), ("GET", "/svc/mcp-x/rpc/"), ("POST", "/svc/mcp-x/agentenv")]) +async def test_an_mcp_server_whose_endpoint_is_elsewhere_is_guarded_at_its_endpoint(monkeypatch, method, path): + backing = _Backing(monkeypatch) + servers = {"internal_mcp_servers": [InternalMCPServer(name="x", mcp_url="http://x:8000/rpc")], + "rest_proxy_urls": {"mcp-x": "http://x:8000"}, "website_urls": {}} + assert await _proxied(TOKEN, method, path, {}, **servers) == 401 + assert await _proxied(TOKEN, method, path, {"Authorization": f"Bearer {TOKEN}"}, **servers) != 401 + assert backing.paths == [path.removeprefix("/svc/mcp-x")] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("path", ["/svc/desk/agentenv/ext/browser/cdp", "/svc/desk/mcp"]) +async def test_a_child_env_of_its_own_takes_its_own_token_through_the_proxy(monkeypatch, path): + backing = _Backing(monkeypatch) + computer = {"internal_mcp_servers": [InternalMCPServer(name="desk", mcp_url="http://desk:18765/mcp")], + "rest_proxy_urls": {"desk": "http://desk:18765"}, "website_urls": {}} + assert await _proxied(TOKEN, "POST", path, {"Authorization": "Bearer the-computers"}, **computer) != 401 + forwarded = [(at, sent) for at, sent in zip(backing.paths, backing.authorization) if at != WELL_KNOWN_PATH] + assert forwarded == [(path.removeprefix("/svc/desk"), "Bearer the-computers")] diff --git a/tst/unit/providers/env_providers/env_gateway_provider_test.py b/tst/unit/providers/env_providers/env_gateway_provider_test.py index 7ddc4a0d4..351239426 100644 --- a/tst/unit/providers/env_providers/env_gateway_provider_test.py +++ b/tst/unit/providers/env_providers/env_gateway_provider_test.py @@ -2646,14 +2646,15 @@ def _sites(handler): @pytest.mark.asyncio -async def test_a_deploy_waits_for_a_site_until_its_cards_ready_says_so_and_never_reads_its_health_path(): - asked = [] +async def test_a_deploy_waits_for_a_site_until_its_cards_ready_says_so_with_its_token_and_never_reads_its_health_path(): + asked, sent = [], [] def site(request: httpx.Request) -> httpx.Response: asked.append(request.url.path) if request.url.path == "/svc/mail/.well-known/agent-env.json": return httpx.Response(200, json=_READY_CARD) if request.url.path == "/svc/mail/agentenv/ext/website/ready": + sent.append(request.headers.get("authorization")) ready = asked.count(request.url.path) >= 2 return httpx.Response(200, json={"ready": ready, "problems": [] if ready else ["seeded"]}) return httpx.Response(500) @@ -2661,8 +2662,9 @@ def site(request: httpx.Request) -> httpx.Response: with _sites(site): await env_gateway_provider._wait_for_websites( "http://gw", [WebsiteConfig(backend_image="m", frontend_image=None, environment_name="mail", - health_path="/healthz")], poll_s=0) + health_path="/healthz")], headers={"Authorization": "Bearer s3cret"}, poll_s=0) assert asked.count("/svc/mail/agentenv/ext/website/ready") == 2 and "/svc/mail/healthz" not in asked + assert sent == ["Bearer s3cret"] * 2 @pytest.mark.asyncio From 91e1539899625b2369b342eb2b002d6df777b8b6 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 13 Nov 2026 09:46:54 -0800 Subject: [PATCH 3/7] fix(gateway): guard a live website's data plane too The provider routes a live website's backend under /svc// but leaves it out of WEBSITE_URLS, so token_paths took it for a child env of its own and guarded nothing on it. Every backing route is now guarded but a child registered under its own name, a computer, which takes its own token. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/env/gateway/gateway.py | 23 +++++++++++----------- tst/unit/env/gateway/control_token_test.py | 9 +++++++++ 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/src/agent_env/env/gateway/gateway.py b/src/agent_env/env/gateway/gateway.py index 55c3f458d..ed7e4b54d 100644 --- a/src/agent_env/env/gateway/gateway.py +++ b/src/agent_env/env/gateway/gateway.py @@ -1731,23 +1731,22 @@ def site_mcp_endpoints(servers: list[InternalMCPServer], website_urls: dict[str, def token_paths(servers: list[InternalMCPServer], website_urls: dict[str, str], rest_proxy_urls: dict[str, str]) -> dict[str, tuple[str, ...]]: - """The paths under /svc// that take the gateway's control token, by key, for each MCP server and website the - gateway fronts: its data plane, extensions included, and an MCP server's own MCP endpoint, at the path its - ``mcp_url`` gives (``/mcp`` when none is under the route's address), which skips the gateway's tool rules, - trajectory and triggers. A child env of its own, a computer, takes its own token, so none of its paths are here.""" + """The paths under /svc// that take the gateway's control token, by key: every backing server's data plane, + extensions included, and an MCP server's own MCP endpoint, at the path its ``mcp_url`` gives (``/mcp`` when none is + under the route's address), which skips the gateway's tool rules, trajectory and triggers. A child env registered + under its own name, a computer, takes its own token, so none of its paths are here.""" sites = site_mcp_endpoints(servers, website_urls, rest_proxy_urls) paths = {} for key, base in rest_proxy_urls.items(): if key in website_urls: paths[key] = (RPC_PATH, *((sites[key],) if key in sites else ())) - continue - name = route_environment_name(key, base, servers, website=False) - if name == key: - continue - endpoints = [resolved.route_path for server in servers if server.name == name - if (resolved := card_url(base, server.mcp_url)) is not None and resolved.own_server - and resolved.route_path is not None] - paths[key] = (RPC_PATH, *(endpoints or ["/mcp"])) + elif (name := route_environment_name(key, base, servers, website=False)) != key: + endpoints = [resolved.route_path for server in servers if server.name == name + if (resolved := card_url(base, server.mcp_url)) is not None and resolved.own_server + and resolved.route_path is not None] + paths[key] = (RPC_PATH, *(endpoints or ["/mcp"])) + elif not any(server.name == key for server in servers): + paths[key] = (RPC_PATH,) return paths diff --git a/tst/unit/env/gateway/control_token_test.py b/tst/unit/env/gateway/control_token_test.py index 00a9cc259..f3a7d9504 100644 --- a/tst/unit/env/gateway/control_token_test.py +++ b/tst/unit/env/gateway/control_token_test.py @@ -210,3 +210,12 @@ async def test_a_child_env_of_its_own_takes_its_own_token_through_the_proxy(monk assert await _proxied(TOKEN, "POST", path, {"Authorization": "Bearer the-computers"}, **computer) != 401 forwarded = [(at, sent) for at, sent in zip(backing.paths, backing.authorization) if at != WELL_KNOWN_PATH] assert forwarded == [(path.removeprefix("/svc/desk"), "Bearer the-computers")] + + +@pytest.mark.asyncio +async def test_a_live_websites_data_plane_is_guarded_though_it_has_no_website_route(monkeypatch): + backing = _Backing(monkeypatch) + live = {"rest_proxy_urls": {"github": "http://github-website-backend:8000"}, "website_urls": {}} + assert await _proxied(TOKEN, "POST", "/svc/github/agentenv", {}, **live) == 401 + assert await _proxied(TOKEN, "POST", "/svc/github/agentenv", {"Authorization": f"Bearer {TOKEN}"}, **live) != 401 + assert backing.paths == ["/agentenv"] From 494df8d472e438429da6d89271394d4bb93f14ff Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 13 Nov 2026 12:37:38 -0800 Subject: [PATCH 4/7] feat(gateway): log each call the control token refuses, with its method and path A refused call answered 401 and left nothing in the gateway's log, so a harness caller missing the token showed only as its own error. The gateway now logs a warning naming the method and path it refused; it never logs a header. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/env/gateway/gateway.py | 1 + tst/unit/env/gateway/control_token_test.py | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/agent_env/env/gateway/gateway.py b/src/agent_env/env/gateway/gateway.py index ed7e4b54d..50bb391f6 100644 --- a/src/agent_env/env/gateway/gateway.py +++ b/src/agent_env/env/gateway/gateway.py @@ -546,6 +546,7 @@ async def app(scope, receive, send): headers = scope.get("headers", ()) if scope["type"] == "http" and self._takes_control_token(scope["path"]) and not hmac.compare_digest( dict(headers).get(b"authorization", b""), self._control_bearer): + logger.warning("refused %s %s: it takes the env's control token", scope["method"], scope["path"]) response = self._json_response({"ok": False, "error": "this route takes the env's control token"}, 401) await response(scope, receive, send) return diff --git a/tst/unit/env/gateway/control_token_test.py b/tst/unit/env/gateway/control_token_test.py index f3a7d9504..c757920ae 100644 --- a/tst/unit/env/gateway/control_token_test.py +++ b/tst/unit/env/gateway/control_token_test.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio +import logging from urllib.parse import unquote, urlsplit import httpx @@ -161,10 +162,12 @@ async def send(message): @pytest.mark.parametrize("method, path", GUARDED) @pytest.mark.parametrize("headers", [{}, {"Authorization": "Bearer wrong"}], ids=["none", "wrong"]) async def test_the_data_plane_and_a_servers_own_endpoint_behind_the_proxy_refuse_a_call_without_the_token( - monkeypatch, method, path, headers): + monkeypatch, caplog, method, path, headers): backing = _Backing(monkeypatch) + caplog.set_level(logging.WARNING, logger="agent_env.env.gateway.gateway") assert await _proxied(TOKEN, method, path, headers) == 401 assert backing.paths == [] + assert [r.getMessage() for r in caplog.records] == [f"refused {method} {unquote(path)}: it takes the env's control token"] @pytest.mark.asyncio From edcf1fbe6828350a6fbbad1409988af77b4cb801 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 13 Nov 2026 13:18:48 -0800 Subject: [PATCH 5/7] fix(gateway): take the token by the path a backing server receives, and find each MCP endpoint in one place The guard judged the decoded path, while the proxy forwards the raw one and httpx removes its dot segments, so /svc/mcp-x/public%2Fx/../agentenv reached /agentenv without the token. It now also judges the path the backing server receives (sent_path, which reaches_child_mcp uses too). mcp_endpoints is the one map of where each route's MCP endpoint is: the proxy's role stamp and tool rules and the control token all read it, so a server whose mcp_url isn't /mcp is stamped and gated at its own endpoint. A record without the token for a URL leaves the call to a grant (data_plane_headers), and a script run_code runs on an env's host or in its container gets the env's control headers, as one inside a computer does. The tool-rule test through /svc sends the token. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/env/envs/_deployment.py | 10 ++- src/agent_env/env/envs/mcp_server.py | 5 +- src/agent_env/env/envs/website.py | 5 +- src/agent_env/env/gateway/gateway.py | 65 +++++++++++-------- src/agent_env/env/gateway/rest_proxy.py | 22 +++++-- .../task_step/task_steps/run_code.py | 14 ++-- tst/integration/env/gateway/gateway_test.py | 2 +- tst/unit/env/envs/test_env_card_loads.py | 13 ++++ tst/unit/env/gateway/control_token_test.py | 4 +- .../env/gateway/rest_proxy_stream_test.py | 14 ++++ tst/unit/task_step/test_run_code.py | 15 +++-- 11 files changed, 115 insertions(+), 54 deletions(-) diff --git a/src/agent_env/env/envs/_deployment.py b/src/agent_env/env/envs/_deployment.py index 1ae76c656..426dfa45c 100644 --- a/src/agent_env/env/envs/_deployment.py +++ b/src/agent_env/env/envs/_deployment.py @@ -141,11 +141,17 @@ async def load_by_signed_url(env: MCPServerEnv | WebsiteEnv, environment_artifac if urls[path] is None: raise RuntimeError(f"{deployed_by}, so its server fetches {file.object_url} itself, and {type(stores[path]).__name__} " "can't sign a URL for it; loading into such an env needs an object store that signs URLs (S3, Cloud Storage)") - headers = env._deployed.control_headers_for(base_url) + headers = data_plane_headers(env, base_url) await protocol_v1.reset_data(base_url, timeout=timeout, headers=headers) await protocol_v1.add_data(base_url, file_parts(environment_artifact, files, urls), timeout=timeout, headers=headers) +def data_plane_headers(env: MCPServerEnv | WebsiteEnv, url: str) -> dict[str, str] | None: + """The headers a call to ``url`` sends of ``env``'s record's control token; None, which leaves the call to a run's + or a multi env's grant, when the record gives it none: no record, no token, or an address outside the record's.""" + return (env._deployed.control_headers_for(url) or None) if env._deployed is not None else None + + async def reset_server_data(env: MCPServerEnv | WebsiteEnv, *, mcp: bool) -> bool: """Empty what loads put into ``env``, an MCP server or with ``mcp=False`` a website, as each load does before it adds: through ``data/reset``, or an older server's ``/api/reset``, or else its ``_reset`` tool, which with no seed @@ -163,7 +169,7 @@ async def reset_server_data(env: MCPServerEnv | WebsiteEnv, *, mcp: bool) -> boo if child is not None and child.serves(METHOD_RESET) is False: return False await protocol_v1.reset_data(base_url, timeout=DATA_PLANE_LOAD_TIMEOUT_S, - headers=env._deployed.control_headers_for(base_url) if env._deployed is not None else None) + headers=data_plane_headers(env, base_url)) elif builtin is None: return False else: diff --git a/src/agent_env/env/envs/mcp_server.py b/src/agent_env/env/envs/mcp_server.py index 891d94bef..44f5c02dd 100644 --- a/src/agent_env/env/envs/mcp_server.py +++ b/src/agent_env/env/envs/mcp_server.py @@ -21,7 +21,8 @@ from agent_env.env import legacy_protocol from agent_env.env.envs._deployment import ( as_builtin, builtin_provider_for, close_deployed, close_replaced, deploy_refusal, deploy_through_provider, host_staging_refusal, - file_parts, legacy_load_file, load_by_signed_url, plugin_provider_like_a_builtin, provider_or_class, reset_server_data, + data_plane_headers, file_parts, legacy_load_file, load_by_signed_url, plugin_provider_like_a_builtin, provider_or_class, + reset_server_data, staged_paths, ) from agent_env.env.gateway import GatewayMode @@ -186,7 +187,7 @@ async def load_environment_artifact(self, environment_artifact: EnvironmentArtif # reset_data previously took the client default (30s), tighter than the load it # precedes -- dropping and recreating a large service's schemas can plausibly # exceed that. Give it the same budget as the load it belongs to. - headers = self._deployed.control_headers_for(base_url) if self._deployed is not None else None + headers = data_plane_headers(self, base_url) await protocol_v1.reset_data(base_url, timeout=timeout, headers=headers) await protocol_v1.add_data(base_url, file_parts(environment_artifact, files, staged), timeout=timeout, headers=headers) diff --git a/src/agent_env/env/envs/website.py b/src/agent_env/env/envs/website.py index c0991513d..b7a01c7d2 100644 --- a/src/agent_env/env/envs/website.py +++ b/src/agent_env/env/envs/website.py @@ -44,7 +44,8 @@ from agent_env.env import legacy_protocol from agent_env.env.envs._deployment import ( as_builtin, builtin_provider_for, close_deployed, close_replaced, deploy_refusal, deploy_through_provider, host_staging_refusal, - file_parts, legacy_load_file, load_by_signed_url, plugin_provider_like_a_builtin, provider_or_class, reset_server_data, + data_plane_headers, file_parts, legacy_load_file, load_by_signed_url, plugin_provider_like_a_builtin, provider_or_class, + reset_server_data, staged_paths, ) from agent_env.env.gateway import GatewayMode @@ -321,7 +322,7 @@ async def load_environment_artifact(self, environment_artifact: EnvironmentArtif try: staged = {path: f"file://{await self._copy_artifact_into_container(file, paths[path])}" for path, file in files.items()} - headers = self._deployed.control_headers_for(base_url) if self._deployed is not None else None + headers = data_plane_headers(self, base_url) await protocol_v1.reset_data(base_url, headers=headers) await protocol_v1.add_data(base_url, file_parts(environment_artifact, files, staged), timeout=DATA_PLANE_LOAD_TIMEOUT_S, headers=headers) diff --git a/src/agent_env/env/gateway/gateway.py b/src/agent_env/env/gateway/gateway.py index 0cd78be75..1fe7c1727 100644 --- a/src/agent_env/env/gateway/gateway.py +++ b/src/agent_env/env/gateway/gateway.py @@ -87,6 +87,7 @@ fetch_child_card, forward, is_child_mcp_path, + sent_path, listed_url, lowered, operations, @@ -179,7 +180,7 @@ def __init__( self.rest_proxy_urls = rest_proxy_urls or {} self._site_names = {site: lowered(names) for site, names in (site_names or {}).items()} self._listed_by_name = frozenset(listed_by_name) - self._site_mcp_endpoints = site_mcp_endpoints(internal_mcp_servers, self.website_urls, self.rest_proxy_urls) + self._mcp_endpoints = mcp_endpoints(internal_mcp_servers, self.website_urls, self.rest_proxy_urls) self._token_paths = token_paths(internal_mcp_servers, self.website_urls, self.rest_proxy_urls) self._site_cards: dict[str, SiteCard] = {} self._withheld = frozenset(name.strip().lower().encode() for name in provider_headers if name.strip()) @@ -547,7 +548,7 @@ def _asgi_app(self): async def app(scope, receive, send): if self._control_bearer and scope["type"] in ("http", "websocket"): headers = scope.get("headers", ()) - if scope["type"] == "http" and self._takes_control_token(scope["path"]) and not hmac.compare_digest( + if scope["type"] == "http" and self._takes_control_token(scope) and not hmac.compare_digest( dict(headers).get(b"authorization", b""), self._control_bearer): logger.warning("refused %s %s: it takes the env's control token", scope["method"], scope["path"]) response = self._json_response({"ok": False, "error": "this route takes the env's control token"}, 401) @@ -572,15 +573,18 @@ async def app(scope, receive, send): return app - def _takes_control_token(self, path: str) -> bool: - """Whether a request to ``path`` takes the deploy's control token: a control route, the env's data plane, or - one of ``token_paths`` behind /svc. Judged as the proxy forwards a path, after dot-segment normalisation. A - site's /website route never reaches its data plane.""" + def _takes_control_token(self, scope: dict) -> bool: + """Whether a request takes the deploy's control token: a control route, the env's data plane, or one of + ``token_paths`` behind /svc, by the path as routed or by the one the backing server receives (``sent_path``), + after dot-segment normalisation. A site's /website route never reaches its data plane.""" + path = scope["path"] if path in CONTROL_ROUTES or is_child_mcp_path(path, RPC_PATH): return True prefix, _, rest = path.lstrip("/").partition("/") service, _, rest = rest.partition("/") - return prefix == "svc" and any(is_child_mcp_path(rest, guarded) for guarded in self._token_paths.get(service, ())) + guarded = self._token_paths.get(service, ()) if prefix == "svc" else () + reached = (rest, sent_path(scope.get("raw_path") or path.encode(), path, rest)) + return any(is_child_mcp_path(at, endpoint) for at in reached if at is not None for endpoint in guarded) @asynccontextmanager async def _session_lifespan(self, mcp: FastMCP): @@ -797,9 +801,9 @@ async def _proxy_rest_request(self, request: Request, url_map: dict[str, str] | if base_url is None: return self._json_response({"error": f"Unknown service: {service_name}"}, 404) role = self._request_role(request) - site_mcp = self._site_mcp_endpoints.get(service_name) if url_map is self.rest_proxy_urls else None - mcp_server = url_map is self.rest_proxy_urls and (service_name.startswith("mcp-") or site_mcp is not None) - mcp_endpoint = site_mcp or "mcp" + mcp_endpoint = self._mcp_endpoints.get(service_name) if url_map is self.rest_proxy_urls else None + mcp_server = mcp_endpoint is not None + mcp_endpoint = mcp_endpoint or "mcp" if url_map is self.rest_proxy_urls and (cdp := await self._browsers.cdp_answer(request, base_url, role)) is not None: return cdp if mcp_server and request.method == "POST" and reaches_child_mcp(request, mcp_endpoint): @@ -1754,25 +1758,34 @@ def site_mcp_endpoints(servers: list[InternalMCPServer], website_urls: dict[str, return endpoints +def mcp_endpoints(servers: list[InternalMCPServer], website_urls: dict[str, str], + rest_proxy_urls: dict[str, str]) -> dict[str, str]: + """The MCP endpoint, as a path under its /svc// route, of each backing route that is an MCP server's: a site + that is an internal MCP server too (``site_mcp_endpoints``), and each ``mcp-`` route at the path its server's + ``mcp_url`` gives, ``/mcp`` when none is under the route's address. Where the proxy stamps the role and applies the + tool rules, and where the control token is taken.""" + endpoints = site_mcp_endpoints(servers, website_urls, rest_proxy_urls) + for key, base in rest_proxy_urls.items(): + if key in endpoints or not key.startswith("mcp-"): + continue + name = route_environment_name(key, base, servers, website=False) + found = [resolved.route_path for server in servers if server.name == name + if (resolved := card_url(base, server.mcp_url)) is not None and resolved.own_server + and resolved.route_path is not None] + endpoints[key] = found[0] if found else "/mcp" + return endpoints + + def token_paths(servers: list[InternalMCPServer], website_urls: dict[str, str], rest_proxy_urls: dict[str, str]) -> dict[str, tuple[str, ...]]: """The paths under /svc// that take the gateway's control token, by key: every backing server's data plane, - extensions included, and an MCP server's own MCP endpoint, at the path its ``mcp_url`` gives (``/mcp`` when none is - under the route's address), which skips the gateway's tool rules, trajectory and triggers. A child env registered - under its own name, a computer, takes its own token, so none of its paths are here.""" - sites = site_mcp_endpoints(servers, website_urls, rest_proxy_urls) - paths = {} - for key, base in rest_proxy_urls.items(): - if key in website_urls: - paths[key] = (RPC_PATH, *((sites[key],) if key in sites else ())) - elif (name := route_environment_name(key, base, servers, website=False)) != key: - endpoints = [resolved.route_path for server in servers if server.name == name - if (resolved := card_url(base, server.mcp_url)) is not None and resolved.own_server - and resolved.route_path is not None] - paths[key] = (RPC_PATH, *(endpoints or ["/mcp"])) - elif not any(server.name == key for server in servers): - paths[key] = (RPC_PATH,) - return paths + extensions included, and an MCP server's own endpoint (``mcp_endpoints``), which skips the gateway's trajectory and + triggers. A child env registered under its own name, a computer, takes its own token, so none of its paths are here.""" + endpoints = mcp_endpoints(servers, website_urls, rest_proxy_urls) + return {key: (RPC_PATH, *((endpoints[key],) if key in endpoints else ())) + for key, base in rest_proxy_urls.items() + if key in website_urls or route_environment_name(key, base, servers, website=False) != key + or not any(server.name == key for server in servers)} class CardURL(NamedTuple): diff --git a/src/agent_env/env/gateway/rest_proxy.py b/src/agent_env/env/gateway/rest_proxy.py index 5a12faab3..1d67eda58 100644 --- a/src/agent_env/env/gateway/rest_proxy.py +++ b/src/agent_env/env/gateway/rest_proxy.py @@ -276,15 +276,23 @@ def is_child_mcp_path(decoded_rest: str, endpoint: str = "mcp") -> bool: def reaches_child_mcp(request: HTTPConnection, endpoint: str = "mcp") -> bool: """Whether a /svc request reaches the backing server's MCP endpoint (:func:`is_child_mcp_path`), by the path as - routed here or by the one the server receives: the raw path forwarded, its dot segments removed by httpx and then - decoded, which differ when an encoded slash sits beside a dot segment (``public%2Fx/../mcp`` reaches ``/mcp``).""" - if is_child_mcp_path(request.path_params.get("path", ""), endpoint): - return True + routed here or by the one the server receives (:func:`sent_path`).""" + rest = request.path_params.get("path", "") + return any(is_child_mcp_path(at, endpoint) for at in (rest, sent_path(request.scope["raw_path"], request.scope["path"], rest)) + if at is not None) + + +def sent_path(raw_path: bytes, decoded_path: str, decoded_rest: str) -> str | None: + """The path the backing server receives for a /svc request whose routed remainder is ``decoded_rest``: the raw + remainder forwarded, its dot segments removed by httpx and then decoded, which differs from ``decoded_rest`` when an + encoded slash sits beside a dot segment (``public%2Fx/../mcp`` reaches ``/mcp``); None for a request-target + forward() refuses.""" + if not decoded_rest: + return "" try: - sent = httpx.URL(upstream_url("http://child", request)).path + return httpx.URL("http://child/" + raw_remainder(raw_path, decoded_path, decoded_rest).decode("latin-1")).path except (ValueError, httpx.InvalidURL): - return False # forward() refuses a request-target it can't build a URL for - return is_child_mcp_path(sent, endpoint) + return None def stamp_jsonrpc_role(body: bytes, role: str) -> bytes: diff --git a/src/agent_env/task_step/task_steps/run_code.py b/src/agent_env/task_step/task_steps/run_code.py index 2652f4f01..59401cfdc 100644 --- a/src/agent_env/task_step/task_steps/run_code.py +++ b/src/agent_env/task_step/task_steps/run_code.py @@ -2,12 +2,12 @@ The script defines ``run(input) -> JSON``, where ``input = {"args", "results"}``. The result is stored at ``context.metadata["script_results"][result_id]``. -A script run for an env also gets ``input["env"] = {"id", "url", "mcp_url"}``: the env's +A script run for an env also gets ``input["env"] = {"id", "url", "mcp_url", "headers"}``: the env's address, where its card is at ``/.well-known/agent-env.json``, and its MCP endpoint, as the script -reaches them: on the env's host as that host does (``Sandbox.url_on_host``), since a host can't -always call its own public URL; inside a computer, the driver on loopback, with the ``headers`` that -carry the computer's control token to its framework-only routes. A script run in an agent's sandbox -never gets the token. +reaches them, and the ``headers`` that carry the env's control token to its framework-only routes and +its data plane: on the env's host as that host does (``Sandbox.url_on_host``), since a host can't +always call its own public URL; inside a computer, the driver on loopback. A script run in an agent's +sandbox never gets the token. ``script_artifact_id`` is either a single-file ``FileArtifact`` or a ``FileArtifactUniverse`` whose ``.py`` members are staged side by side so the entry @@ -538,12 +538,14 @@ def _is_safe_member(name: str) -> bool: def _env_on_host(deployed: DeployedSandboxEnv, sandbox: Any) -> dict: - """The env's id, address and MCP endpoint as a script on its host reaches them.""" + """The env's id, address and MCP endpoint as a script on its host reaches them, and the headers its control token + travels in.""" url, mcp_url = deployed.environment_url, deployed.mcp_url return { "id": deployed.env_id, "url": url and sandbox.url_on_host(url), "mcp_url": mcp_url and sandbox.url_on_host(mcp_url), + "headers": deployed.control_headers, } diff --git a/tst/integration/env/gateway/gateway_test.py b/tst/integration/env/gateway/gateway_test.py index ec7a8d64b..2c1c779b5 100644 --- a/tst/integration/env/gateway/gateway_test.py +++ b/tst/integration/env/gateway/gateway_test.py @@ -1953,7 +1953,7 @@ async def test_a_tool_disabled_for_a_role_is_refused_through_svc_as_on_the_gatew after_id = seen[-1]["id"] if seen else 0 answers = {} for role in ("carol@example.com", "dana@example.com"): - async with httpx.AsyncClient(headers={AGENT_ENV_ROLE_HEADER: role}, timeout=60) as as_role: + async with httpx.AsyncClient(headers={AGENT_ENV_ROLE_HEADER: role, **result.control_headers}, timeout=60) as as_role: for path in ("/mcp", "/svc/mcp-email/mcp"): async with streamable_http_client(f"{gw}{path}", http_client=as_role) as (read, write, _): async with ClientSession(read, write) as session: diff --git a/tst/unit/env/envs/test_env_card_loads.py b/tst/unit/env/envs/test_env_card_loads.py index 90a0961d6..37c385d18 100644 --- a/tst/unit/env/envs/test_env_card_loads.py +++ b/tst/unit/env/envs/test_env_card_loads.py @@ -18,8 +18,10 @@ import httpx import pytest from agentenv_protocol import DATA_OBJECTS_EXTENSION_URI +from agentenv_protocol import client as protocol_v1 from agent_env.env.env import DeployedEnv, DeployedGatewayEnv, DeployedSandboxEnv +from agent_env.env.envs import _deployment from agent_env.env.envs.mcp_server import MCPServerEnv from agent_env.env.envs.multi_env import MultiEnv from agent_env.env.envs.website import WebsiteEnv @@ -72,6 +74,17 @@ async def test_a_load_sends_its_records_control_token_with_no_run_to_grant_it(se *[(f"{_CARD}/svc/mcp-email/agentenv", "Bearer s3cret")] * 2, *[(f"{_CARD}/svc/shop/agentenv", "Bearer s3cret")] * 2] +def test_a_record_with_no_token_for_an_address_leaves_a_call_there_to_a_grant(): + env = _mcp(_record(None)) + env._deployed.control_token = "s3cret" + at_gateway, at_card = f"{_GW}/svc/mcp-email", f"{_CARD}/svc/mcp-email" + with protocol_v1.authorized({_GW: {"Authorization": "Bearer granted"}}): + sent = protocol_v1.request_headers(at_gateway, _deployment.data_plane_headers(env, at_gateway)) + assert sent == {"Authorization": "Bearer granted"} + assert _deployment.data_plane_headers(env, at_card) == {"Authorization": "Bearer s3cret"} + assert _deployment.data_plane_headers(_mcp(None), at_card) is None + + @pytest.mark.asyncio async def test_a_child_env_missing_from_the_card_takes_the_legacy_path(sent): # email serves no card, so the gateway leaves it out of the composed card. diff --git a/tst/unit/env/gateway/control_token_test.py b/tst/unit/env/gateway/control_token_test.py index c757920ae..a05ec086a 100644 --- a/tst/unit/env/gateway/control_token_test.py +++ b/tst/unit/env/gateway/control_token_test.py @@ -31,7 +31,9 @@ GUARDED = [("POST", "/agentenv"), ("POST", "/svc/mcp-svc/agentenv"), ("POST", "/svc/mcp-svc/agentenv/ext/grant"), ("POST", "/svc/mcp-svc/x/../agentenv"), ("POST", "/svc/mcp-svc/%61gentenv"), ("POST", "/svc/shop/agentenv"), ("POST", "/svc/mcp-svc/mcp"), ("GET", "/svc/mcp-svc/mcp/"), - ("DELETE", "/svc/mcp-svc/./mcp"), ("POST", "/svc/mcp-svc//mcp")] + ("DELETE", "/svc/mcp-svc/./mcp"), ("POST", "/svc/mcp-svc//mcp"), ("POST", "/svc/mcp-svc/public%2Fx/../agentenv"), + ("POST", "/svc/mcp-svc/public%2Fx/../mcp"), ("POST", "/svc/mcp-svc/public%2Fx/../agentenv/ext/set_acting_user"), + ("POST", "/svc/mcp-svc/AgentEnv"), ("POST", "/svc/mcp-svc/MCP")] PROXIED_OPEN = [("GET", "/svc/mcp-svc/.well-known/agent-env.json"), ("GET", "/svc/mcp-svc/openapi.yaml"), ("GET", "/svc/mcp-svc/export-state"), ("POST", "/svc/shop/api/orders"), ("POST", "/svc/shop/mcp"), ("GET", "/website/shop/index.html")] diff --git a/tst/unit/env/gateway/rest_proxy_stream_test.py b/tst/unit/env/gateway/rest_proxy_stream_test.py index 2ed6ac102..5df7f530e 100644 --- a/tst/unit/env/gateway/rest_proxy_stream_test.py +++ b/tst/unit/env/gateway/rest_proxy_stream_test.py @@ -530,6 +530,20 @@ async def _post_rpc(gw: Gateway, payload, role: str, path: str = "/svc/mcp-svc/m return status, json.loads(b"".join(chunks)) +@pytest.mark.asyncio +async def test_a_server_whose_mcp_endpoint_is_elsewhere_has_the_role_stamped_there_and_not_at_mcp(monkeypatch): + up = _Upstream([b"{}"]) + up.install(monkeypatch) + gw = Gateway(host="127.0.0.1", port=0, server_name="t", rest_proxy_urls={"mcp-svc": UPSTREAM}, + internal_mcp_servers=[InternalMCPServer(name="svc", mcp_url=f"{UPSTREAM}/rpc")]) + body = json.dumps({"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {}}).encode() + for path in ("/svc/mcp-svc/rpc", "/svc/mcp-svc/mcp"): + await _call(gw, path, method="POST", body=body, headers={ + "AgentEnv-Role": "alice@example.com", "content-type": "application/json", "content-length": str(len(body))}) + assert [json.loads(r.content)["params"].get("_meta") for r in up.requests] == [ + {AGENT_ENV_ROLE_META_KEY: "alice@example.com"}, None] + + @pytest.mark.asyncio async def test_a_tool_disabled_for_the_caller_is_refused_as_mcp_refuses_it_and_never_reaches_the_child(monkeypatch): up = _Upstream([b"{}"]) diff --git a/tst/unit/task_step/test_run_code.py b/tst/unit/task_step/test_run_code.py index b21f10b7f..5527367e1 100644 --- a/tst/unit/task_step/test_run_code.py +++ b/tst/unit/task_step/test_run_code.py @@ -296,19 +296,19 @@ async def exec_with_output(self, *args): @pytest.mark.parametrize("deployed,expected", [ (DeployedGatewayEnv(env_id="gw", env_version=1, sandbox_id="sb", gateway_url="https://abc.modal.host", environment_card_url=_CARD_URL, environment_card={"name": "gw"}), - {"id": "gw", "url": "http://127.0.0.1:18765", "mcp_url": "http://127.0.0.1:18765/mcp"}), + {"id": "gw", "url": "http://127.0.0.1:18765", "mcp_url": "http://127.0.0.1:18765/mcp", "headers": {}}), (DeployedSandboxEnv(env_id="srv", env_version=1, sandbox_id="sb", env_provider_type="server", environment_card_url=_CARD_URL, environment_card={"name": "srv"}), - {"id": "srv", "url": "http://127.0.0.1:18765", "mcp_url": "http://127.0.0.1:18765/mcp"}), + {"id": "srv", "url": "http://127.0.0.1:18765", "mcp_url": "http://127.0.0.1:18765/mcp", "headers": {}}), # a plugin's env, whose card puts its MCP endpoint elsewhere (DeployedSandboxEnv(env_id="plug", env_version=1, sandbox_id="sb", env_provider_type="a-plugins", environment_card_url=_CARD_URL, environment_card={"name": "plug", "additionalInterfaces": [{"transport": "mcp", "url": "/v2/mcp"}]}), - {"id": "plug", "url": "http://127.0.0.1:18765", "mcp_url": "http://127.0.0.1:18765/v2/mcp"}), + {"id": "plug", "url": "http://127.0.0.1:18765", "mcp_url": "http://127.0.0.1:18765/v2/mcp", "headers": {}}), # a record from before the card, which kept only its MCP URL (DeployedSandboxEnv(env_id="old", env_version=1, sandbox_id="sb", mcp_url="https://abc.modal.host/mcp"), - {"id": "old", "url": None, "mcp_url": "http://127.0.0.1:18765/mcp"}), + {"id": "old", "url": None, "mcp_url": "http://127.0.0.1:18765/mcp", "headers": {}}), ]) def test_an_envs_address_and_mcp_endpoint_come_from_its_card_whatever_deployed_it(deployed, expected): sandbox = _ContainerEnvSandbox({18765: "https://abc.modal.host"}) @@ -328,11 +328,11 @@ def _written_input(sandbox): @pytest.mark.asyncio -async def test_a_script_on_an_envs_host_gets_the_envs_address_and_mcp_endpoint_as_that_host_reaches_them(): +async def test_a_script_on_an_envs_host_gets_the_envs_address_and_mcp_endpoint_as_that_host_reaches_them_and_its_token(): ctx = _make_context(with_agent=False) ctx.deployed_envs[0] = DeployedGatewayEnv( env_id="env-x", env_version=1, sandbox_id="sb-env", gateway_url="https://abc.modal.host", - environment_card_url=_CARD_URL, environment_card={"name": "env-x"}, + environment_card_url=_CARD_URL, environment_card={"name": "env-x"}, control_token="s3cret", ) with _sandbox([(0, "", ""), (0, "2", ""), (0, "{}", "")], agent_mode=False) as sandbox: sandbox.tunnel_urls = {18765: "https://abc.modal.host"} @@ -340,6 +340,7 @@ async def test_a_script_on_an_envs_host_gets_the_envs_address_and_mcp_endpoint_a assert _written_input(sandbox)["env"] == { "id": "env-x", "url": "http://127.0.0.1:18765", "mcp_url": "http://127.0.0.1:18765/mcp", + "headers": {"Authorization": "Bearer s3cret"}, } @@ -362,7 +363,7 @@ async def test_a_script_for_an_env_in_a_container_sandbox_runs_in_that_container ctx = await _step(env_id="env-x").execute(ctx) assert ctx.metadata["script_results"]["run-code-1"] == { - "id": "env-x", "url": "http://127.0.0.1:18765", "mcp_url": "http://127.0.0.1:18765/mcp", + "id": "env-x", "url": "http://127.0.0.1:18765", "mcp_url": "http://127.0.0.1:18765/mcp", "headers": {}, } assert not any(command[0] in ("sudo", "docker") for command in sandbox.commands) From fc7bbdd085b4b473859347cd2119b0b447a96260 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 13 Nov 2026 13:25:20 -0800 Subject: [PATCH 6/7] fix(gateway): judge the path a backing server receives against its real base The path a /svc request reaches was rebuilt on a root placeholder, while the proxy appends the raw remainder to the backing URL, path included. Behind a base http://x/prefix, /svc/mcp-x/../prefix/mcp reached the child's /prefix/mcp past the control token and the tool rules. reaches() now resolves it against the real base and compares it with the endpoint under the base's path; the guard and reaches_child_mcp share it. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/env/gateway/gateway.py | 10 +++---- src/agent_env/env/gateway/rest_proxy.py | 32 +++++++++++----------- tst/unit/env/gateway/control_token_test.py | 10 +++++++ 3 files changed, 31 insertions(+), 21 deletions(-) diff --git a/src/agent_env/env/gateway/gateway.py b/src/agent_env/env/gateway/gateway.py index 1fe7c1727..9bba5539a 100644 --- a/src/agent_env/env/gateway/gateway.py +++ b/src/agent_env/env/gateway/gateway.py @@ -87,7 +87,7 @@ fetch_child_card, forward, is_child_mcp_path, - sent_path, + reaches, listed_url, lowered, operations, @@ -575,7 +575,7 @@ async def app(scope, receive, send): def _takes_control_token(self, scope: dict) -> bool: """Whether a request takes the deploy's control token: a control route, the env's data plane, or one of - ``token_paths`` behind /svc, by the path as routed or by the one the backing server receives (``sent_path``), + ``token_paths`` behind /svc, by the path as routed or by the one the backing server receives (``reaches``), after dot-segment normalisation. A site's /website route never reaches its data plane.""" path = scope["path"] if path in CONTROL_ROUTES or is_child_mcp_path(path, RPC_PATH): @@ -583,8 +583,8 @@ def _takes_control_token(self, scope: dict) -> bool: prefix, _, rest = path.lstrip("/").partition("/") service, _, rest = rest.partition("/") guarded = self._token_paths.get(service, ()) if prefix == "svc" else () - reached = (rest, sent_path(scope.get("raw_path") or path.encode(), path, rest)) - return any(is_child_mcp_path(at, endpoint) for at in reached if at is not None for endpoint in guarded) + raw, base = scope.get("raw_path") or path.encode(), self.rest_proxy_urls.get(service, "") + return any(reaches(base, raw, path, rest, endpoint) for endpoint in guarded) @asynccontextmanager async def _session_lifespan(self, mcp: FastMCP): @@ -806,7 +806,7 @@ async def _proxy_rest_request(self, request: Request, url_map: dict[str, str] | mcp_endpoint = mcp_endpoint or "mcp" if url_map is self.rest_proxy_urls and (cdp := await self._browsers.cdp_answer(request, base_url, role)) is not None: return cdp - if mcp_server and request.method == "POST" and reaches_child_mcp(request, mcp_endpoint): + if mcp_server and request.method == "POST" and reaches_child_mcp(request, base_url, mcp_endpoint): if (refusal := await self._refuse_disabled_calls(await request.body(), role)) is not None: return refusal if request.method in _SAFE_METHODS: diff --git a/src/agent_env/env/gateway/rest_proxy.py b/src/agent_env/env/gateway/rest_proxy.py index 1d67eda58..e4ef8af28 100644 --- a/src/agent_env/env/gateway/rest_proxy.py +++ b/src/agent_env/env/gateway/rest_proxy.py @@ -105,7 +105,7 @@ async def forward(request: Request, base_url: str, *, role: str, timeout: httpx. return await _done(_json_response({"error": str(e)}, 400), on_done) content = await request.body() # /svc and /website request bodies are small (JSON-RPC, forms) headers = upstream_headers(request, role, withheld=withheld, site_names=site_names) - if mcp_server and request.method == "POST" and reaches_child_mcp(request, mcp_endpoint): + if mcp_server and request.method == "POST" and reaches_child_mcp(request, base_url, mcp_endpoint): # The gateway is the trust boundary for identity: a child's proxied MCP endpoint stays reachable, but the # role in its JSON-RPC `_meta` is the gateway's to write, like the header above. content = stamp_jsonrpc_role(content, role) @@ -274,25 +274,25 @@ def is_child_mcp_path(decoded_rest: str, endpoint: str = "mcp") -> bool: return normalized == at or normalized.startswith(at + "/") -def reaches_child_mcp(request: HTTPConnection, endpoint: str = "mcp") -> bool: - """Whether a /svc request reaches the backing server's MCP endpoint (:func:`is_child_mcp_path`), by the path as - routed here or by the one the server receives (:func:`sent_path`).""" - rest = request.path_params.get("path", "") - return any(is_child_mcp_path(at, endpoint) for at in (rest, sent_path(request.scope["raw_path"], request.scope["path"], rest)) - if at is not None) +def reaches_child_mcp(request: HTTPConnection, base_url: str, endpoint: str = "mcp") -> bool: + """Whether a /svc request reaches the MCP endpoint of the backing server at ``base_url`` (:func:`reaches`).""" + return reaches(base_url, request.scope["raw_path"], request.scope["path"], request.path_params.get("path", ""), endpoint) -def sent_path(raw_path: bytes, decoded_path: str, decoded_rest: str) -> str | None: - """The path the backing server receives for a /svc request whose routed remainder is ``decoded_rest``: the raw - remainder forwarded, its dot segments removed by httpx and then decoded, which differs from ``decoded_rest`` when an - encoded slash sits beside a dot segment (``public%2Fx/../mcp`` reaches ``/mcp``); None for a request-target - forward() refuses.""" - if not decoded_rest: - return "" +def reaches(base_url: str, raw_path: bytes, decoded_path: str, decoded_rest: str, endpoint: str) -> bool: + """Whether a /svc request whose routed remainder is ``decoded_rest`` reaches ``endpoint`` of the backing server at + ``base_url`` (:func:`is_child_mcp_path`): by that remainder, or by the path the server receives, the base's path and + the raw remainder forwarded, its dot segments removed by httpx and then decoded. They differ when an encoded slash + sits beside a dot segment (``public%2Fx/../mcp`` reaches ``/mcp``) or a dot segment climbs into a base's path + (``../prefix/mcp`` reaches base ``/prefix``'s ``/mcp``). False for a request-target forward() refuses.""" + if is_child_mcp_path(decoded_rest, endpoint): + return True try: - return httpx.URL("http://child/" + raw_remainder(raw_path, decoded_path, decoded_rest).decode("latin-1")).path + raw_rest = raw_remainder(raw_path, decoded_path, decoded_rest) if decoded_rest else b"" + sent = httpx.URL(f"{base_url.rstrip('/')}/{raw_rest.decode('latin-1')}").path except (ValueError, httpx.InvalidURL): - return None + return False + return is_child_mcp_path(sent, f"{urlsplit(base_url).path.rstrip('/')}/{endpoint.lstrip('/')}") def stamp_jsonrpc_role(body: bytes, role: str) -> bytes: diff --git a/tst/unit/env/gateway/control_token_test.py b/tst/unit/env/gateway/control_token_test.py index a05ec086a..167518023 100644 --- a/tst/unit/env/gateway/control_token_test.py +++ b/tst/unit/env/gateway/control_token_test.py @@ -224,3 +224,13 @@ async def test_a_live_websites_data_plane_is_guarded_though_it_has_no_website_ro assert await _proxied(TOKEN, "POST", "/svc/github/agentenv", {}, **live) == 401 assert await _proxied(TOKEN, "POST", "/svc/github/agentenv", {"Authorization": f"Bearer {TOKEN}"}, **live) != 401 assert backing.paths == ["/agentenv"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("path", ["/svc/mcp-x/../prefix/mcp", "/svc/mcp-x/../prefix/agentenv", "/svc/mcp-x/mcp"]) +async def test_a_server_behind_a_base_path_is_guarded_however_a_path_climbs_into_it(monkeypatch, path): + backing = _Backing(monkeypatch) + prefixed = {"internal_mcp_servers": [InternalMCPServer(name="x", mcp_url="http://x:8000/prefix/mcp")], + "rest_proxy_urls": {"mcp-x": "http://x:8000/prefix"}, "website_urls": {}} + assert await _proxied(TOKEN, "POST", path, {}, **prefixed) == 401 + assert backing.paths == [] From 34b1c43c47e39f39113040a5cc4d63b69597595c Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 13 Nov 2026 13:41:47 -0800 Subject: [PATCH 7/7] fix(gateway): compare a reached path with its backing base decoded the same way reaches() decoded the path a backing server receives but kept the base's path encoded, so behind a base like http://x/%70refix a climb into it matched neither. The base's path is now read through httpx, decoded and normalised as the sent path is. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/env/gateway/rest_proxy.py | 3 ++- tst/unit/env/gateway/control_token_test.py | 10 ++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/agent_env/env/gateway/rest_proxy.py b/src/agent_env/env/gateway/rest_proxy.py index e4ef8af28..1fecba0c0 100644 --- a/src/agent_env/env/gateway/rest_proxy.py +++ b/src/agent_env/env/gateway/rest_proxy.py @@ -290,9 +290,10 @@ def reaches(base_url: str, raw_path: bytes, decoded_path: str, decoded_rest: str try: raw_rest = raw_remainder(raw_path, decoded_path, decoded_rest) if decoded_rest else b"" sent = httpx.URL(f"{base_url.rstrip('/')}/{raw_rest.decode('latin-1')}").path + base = httpx.URL(base_url).path.rstrip("/") except (ValueError, httpx.InvalidURL): return False - return is_child_mcp_path(sent, f"{urlsplit(base_url).path.rstrip('/')}/{endpoint.lstrip('/')}") + return is_child_mcp_path(sent, f"{base}/{endpoint.lstrip('/')}") def stamp_jsonrpc_role(body: bytes, role: str) -> bytes: diff --git a/tst/unit/env/gateway/control_token_test.py b/tst/unit/env/gateway/control_token_test.py index 167518023..dce2ab210 100644 --- a/tst/unit/env/gateway/control_token_test.py +++ b/tst/unit/env/gateway/control_token_test.py @@ -227,10 +227,12 @@ async def test_a_live_websites_data_plane_is_guarded_though_it_has_no_website_ro @pytest.mark.asyncio -@pytest.mark.parametrize("path", ["/svc/mcp-x/../prefix/mcp", "/svc/mcp-x/../prefix/agentenv", "/svc/mcp-x/mcp"]) -async def test_a_server_behind_a_base_path_is_guarded_however_a_path_climbs_into_it(monkeypatch, path): +@pytest.mark.parametrize("base", ["http://x:8000/prefix", "http://x:8000/%70refix"], ids=["plain", "encoded"]) +@pytest.mark.parametrize("path", ["/svc/mcp-x/../prefix/mcp", "/svc/mcp-x/../prefix/agentenv", "/svc/mcp-x/mcp", + "/svc/mcp-x/public%2Fx/../agentenv"]) +async def test_a_server_behind_a_base_path_is_guarded_however_a_path_climbs_into_it(monkeypatch, base, path): backing = _Backing(monkeypatch) - prefixed = {"internal_mcp_servers": [InternalMCPServer(name="x", mcp_url="http://x:8000/prefix/mcp")], - "rest_proxy_urls": {"mcp-x": "http://x:8000/prefix"}, "website_urls": {}} + prefixed = {"internal_mcp_servers": [InternalMCPServer(name="x", mcp_url=f"{base}/mcp")], + "rest_proxy_urls": {"mcp-x": base}, "website_urls": {}} assert await _proxied(TOKEN, "POST", path, {}, **prefixed) == 401 assert backing.paths == []