Skip to content

Commit aef48d2

Browse files
feat(computer): the driver's /svc/<app>/ relays WebSockets and streams every method (#478)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
1 parent a641047 commit aef48d2

11 files changed

Lines changed: 561 additions & 34 deletions

File tree

‎packages/agentenv-protocol/README.md‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -770,7 +770,16 @@ launches one at a time, runs a load and a reset with the app stopped and relaunc
770770
follows a failed load with a reset, and fails a call that runs past `data_timeout_s` (300 s); a sync
771771
hook runs in a thread, which the reset and relaunch after it wait for. The driver fronts its apps:
772772
its card lists each one's card as a child environment, rooted at `/svc/<app>/`, where it forwards
773-
requests to the app's server.
773+
requests to the app's server, taking the control token:
774+
775+
- every method, its body and the answer's streamed both ways, so a large answer isn't held whole;
776+
- headers passed on but the hop-by-hop ones, those a `Connection` header names and the control token's
777+
`Authorization`; `Host` as the caller sent it, and in place of any address the caller forwarded,
778+
the one it reached the driver at: `X-Forwarded-Host`, `X-Forwarded-Proto` and `X-Forwarded-Prefix`
779+
(`/svc/<app>`), as the gateway sets them;
780+
- WebSockets, relayed to the app's server both ways with the subprotocol it picks until either side
781+
closes, the other then closed with its code; refused before they open without the token, or with no
782+
such app or one not answering. The `computer` extra brings `websockets` for this.
774783

775784
`MailApp.declaration()` returns what the class declares, an `AppDeclaration`, which a `computer_app`
776785
env stores (`browser: true` for a website browser), and `python -m agentenv_protocol.computer._declare <folder> <file> <class>` prints it as

‎packages/agentenv-protocol/pyproject.toml‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,13 +41,14 @@ app = [
4141
"mcp>=1.25,<2",
4242
"uvicorn>=0.30",
4343
]
44-
# A computer driver: ComputerDriverEnvironment serves its tools over mcp's FastMCP, crops and encodes frames, and
45-
# records the display as H.264 with PyAV (computer-recording/v1).
44+
# A computer driver: ComputerDriverEnvironment serves its tools over mcp's FastMCP, crops and encodes frames, records
45+
# the display as H.264 with PyAV (computer-recording/v1), and relays WebSockets to its apps on /svc/<app>/.
4646
computer = [
4747
"av>=17",
4848
"mcp>=1.25,<2",
4949
"Pillow>=10.4",
5050
"uvicorn>=0.30",
51+
"websockets>=13",
5152
]
5253
dev = [
5354
"a2a-sdk[http-server]>=0.3.26,<0.4",
@@ -59,6 +60,7 @@ dev = [
5960
"fastmcp>=3.4,<4", # the standalone-fastmcp mount path
6061
"Pillow>=10.4", # the computer extra's frame crops and encodings
6162
"av>=17", # the computer extra's encoder, and the tests' decoder
63+
"websockets>=13", # the computer extra's /svc/<app>/ relay
6264
]
6365
[project.urls]
6466
Homepage = "https://www.agentenvframework.com"

‎packages/agentenv-protocol/src/agentenv_protocol/agent_env_environment.py‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1546,18 +1546,27 @@ def _error_response(error: AgentEnvError) -> Response:
15461546
return _json_response(error_body(**error.fields()), error.status)
15471547

15481548

1549+
def control_check() -> Callable[[Any], bool]:
1550+
"""Whether a call's headers carry the env's control token as their bearer, with ``CONTROL_TOKEN_ENV`` set; without
1551+
it, any do."""
1552+
token = os.environ.get(CONTROL_TOKEN_ENV)
1553+
if not token:
1554+
return lambda headers: True
1555+
expected = f"Bearer {token}".encode()
1556+
return lambda headers: hmac.compare_digest(headers.get("authorization", "").encode(), expected)
1557+
1558+
15491559
def control_guard() -> Callable[[Callable], Callable]:
15501560
"""What wraps a framework-only route, everything an env serves but its MCP endpoint and its card: with
15511561
``CONTROL_TOKEN_ENV`` set, a call that doesn't carry that token as its bearer is refused before the route runs;
15521562
without it, nothing."""
1553-
token = os.environ.get(CONTROL_TOKEN_ENV)
1554-
if not token:
1563+
if not os.environ.get(CONTROL_TOKEN_ENV):
15551564
return lambda handler: handler
1556-
expected = f"Bearer {token}".encode()
1565+
allowed = control_check()
15571566

15581567
def guard(handler: Callable) -> Callable:
15591568
async def guarded(request: Request) -> Response:
1560-
if not hmac.compare_digest(request.headers.get("authorization", "").encode(), expected):
1569+
if not allowed(request.headers):
15611570
return _error_response(AgentEnvError(
15621571
"unauthorized", "this route is agent-env's own: it takes the env's control token", status=401))
15631572
return await handler(request)

‎packages/agentenv-protocol/src/agentenv_protocol/computer/apps.py‎

Lines changed: 1 addition & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
paths it owns, and its ``computer-app/v1`` ``ready`` whether it's ready. Every app's server must answer before the
44
computer is ready; an app that starts with the computer must also be ready, and a ``demand`` app is checked once
55
something launches it. The driver lists each app as a child environment named for its folder, its card's paths under
6-
``/svc/<name>/``, and forwards each request there to the app's server."""
6+
``/svc/<name>/``, where it forwards each request to the app's server (``svc``)."""
77
from __future__ import annotations
88

99
import asyncio
@@ -91,15 +91,6 @@ async def owned_paths(folders: list[Path]) -> list[str]:
9191
for path in client.extension_params(card, COMPUTER_APP).get("paths") or []]
9292

9393

94-
async def forward(folder: Path, method: str, path: str, query: str, body: bytes,
95-
content_type: Optional[str]) -> httpx.Response:
96-
"""The app's server's answer to the request, sent on as it came; it takes as long as the app does."""
97-
async with httpx.AsyncClient(transport=httpx.AsyncHTTPTransport(uds=str(folder / SOCKET)), base_url="http://app",
98-
timeout=None) as http:
99-
return await http.request(method, f"/{path}" + (f"?{query}" if query else ""), content=body,
100-
headers={"content-type": content_type} if content_type else None)
101-
102-
10394
def endpoint(card: dict, method: str) -> str:
10495
"""The path of a ``computer-app/v1`` method, as the app's card advertises it."""
10596
return client.find_extension_method(card, COMPUTER_APP, method)["endpoint"]

‎packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py‎

Lines changed: 21 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -4,21 +4,22 @@
44
from __future__ import annotations
55

66
import asyncio
7-
from typing import Any, Callable, ClassVar, Literal
7+
from pathlib import Path
8+
from typing import Any, Callable, ClassVar, Literal, Optional
89

9-
import httpx
1010
from pydantic import BaseModel, ConfigDict
1111
from starlette.requests import Request
1212
from starlette.responses import Response
13+
from starlette.websockets import WebSocket
1314

1415
from ..agent_env_environment import (_CARD_CONFIG_ATTR, AgentEnvEnvironment, AgentEnvFastMCPApplication,
1516
_error_response, control_guard, environment_card)
1617
from ..tool_context import ToolContext
1718
from ..types import EnvironmentCard, EnvironmentExtension, EnvironmentTool
18-
from . import _linux, apps, card
19+
from . import _linux, apps, card, svc
1920
from .checks import check_features, check_names
2021
from .dataplane import DataPlane
21-
from .errors import DeliveryFailed, TargetUnavailable
22+
from .errors import TargetUnavailable
2223
from .extensions import Extensions
2324
from .operations import ReadyCheck, check_class, default_for
2425
from .platform import Platform, detect
@@ -138,21 +139,28 @@ async def children_environments(self) -> list[EnvironmentCard] | None:
138139
return await apps.children(self._computer.apps) if self._computer.apps else None
139140

140141
def _computer_front_apps(self, app: Any) -> Any:
141-
app.custom_route("/svc/{name}/{path:path}", methods=["GET", "POST"])(control_guard()(self._computer_forward))
142+
app.custom_route("/svc/{name}/{path:path}", methods=svc.METHODS)(control_guard()(self._computer_forward))
143+
svc.websocket_route(app, "/svc/{name}/{path:path}", self._computer_relay)
142144
return app
143145

146+
def _computer_app_folder(self, name: str) -> Optional[Path]:
147+
return next((folder for folder in self._computer.apps if folder.name == name), None)
148+
144149
async def _computer_forward(self, request: Request) -> Response:
145150
"""A request under ``/svc/<name>/``, sent on to the app named so: its own server answers it."""
146151
name = request.path_params["name"]
147-
folder = next((folder for folder in self._computer.apps if folder.name == name), None)
148-
if folder is None:
152+
if (folder := self._computer_app_folder(name)) is None:
149153
return _error_response(TargetUnavailable(f"this computer has no app named {name!r}"))
150-
try:
151-
answer = await apps.forward(folder, request.method, request.path_params["path"], request.url.query,
152-
await request.body(), request.headers.get("content-type"))
153-
except (httpx.HTTPError, OSError) as e:
154-
return _error_response(DeliveryFailed(f"app {name}'s server isn't answering: {type(e).__name__}: {e}"))
155-
return Response(answer.content, status_code=answer.status_code, media_type=answer.headers.get("content-type"))
154+
return await svc.forward(folder, name, request)
155+
156+
async def _computer_relay(self, websocket: WebSocket) -> None:
157+
"""A WebSocket under ``/svc/<name>/``, relayed both ways to the app named so; refused, before it opens, with no
158+
such app."""
159+
name = websocket.path_params["name"]
160+
if (folder := self._computer_app_folder(name)) is None:
161+
await websocket.close(code=1008)
162+
return
163+
await svc.relay(folder, name, websocket)
156164

157165
# The card: what the driver's operations support.
158166

Lines changed: 182 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,182 @@
1+
"""The driver's ``/svc/<name>/`` route to each app on its computer: a request sent on to the app's server with its body
2+
and its answer streamed, and a WebSocket relayed to it both ways. It needs the ``computer`` extra's ``websockets``, so
3+
only the driver imports it; an app's venv, with the ``app`` extra, imports ``apps`` and never this."""
4+
from __future__ import annotations
5+
6+
import asyncio
7+
import contextlib
8+
import re
9+
from collections.abc import AsyncIterator, Iterable
10+
from pathlib import Path
11+
from typing import Any, Callable
12+
from urllib.parse import quote, unquote
13+
14+
import httpx
15+
import websockets
16+
from starlette.background import BackgroundTask
17+
from starlette.requests import Request
18+
from starlette.responses import Response, StreamingResponse
19+
from starlette.routing import WebSocketRoute
20+
from starlette.websockets import WebSocket, WebSocketDisconnect
21+
from websockets.asyncio.client import ClientConnection, unix_connect
22+
23+
from ..agent_env_environment import _error_response, control_check
24+
from .app import SOCKET
25+
from .errors import DeliveryFailed
26+
27+
#: The methods the route forwards.
28+
METHODS = ["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"]
29+
#: The headers that belong to one connection, which a proxy doesn't pass on.
30+
HOP_HEADERS = frozenset({"connection", "keep-alive", "proxy-authenticate", "proxy-authorization", "proxy-connection",
31+
"te", "trailer", "transfer-encoding", "upgrade"})
32+
#: The address a request reached the driver at, which the driver sets for the app in place of any the caller sent.
33+
FORWARDED = ("x-forwarded-host", "x-forwarded-proto", "x-forwarded-prefix")
34+
#: A WebSocket close code that means the connection ended without one, which a close can't send on.
35+
UNSENDABLE_CLOSES = (1005, 1006, 1015)
36+
#: A Host header that names a host, and its port or not, and nothing else.
37+
HOST = re.compile(r"(?:[A-Za-z0-9.-]+|\[[0-9A-Fa-f:.]+\])(?::[0-9]{1,5})?")
38+
#: What a path's segments may hold unescaped, besides letters, digits and ``_.-~``.
39+
PATH_SAFE = "/:@!$&'()*+,;="
40+
41+
42+
def passed_on(headers: Iterable[tuple[str, str]], *dropped: str) -> list[tuple[str, str]]:
43+
"""``headers`` as a proxy passes them on: without the hop-by-hop ones, those a ``Connection`` header names, and
44+
``dropped``."""
45+
headers = list(headers)
46+
named = {token.strip().lower() for name, value in headers if name.lower() == "connection"
47+
for token in value.split(",")}
48+
skipped = HOP_HEADERS | named | set(dropped)
49+
return [(name, value) for name, value in headers if name.lower() not in skipped]
50+
51+
52+
def to_app(headers: Iterable[tuple[str, str]], host: str, scheme: str, prefix: str) -> list[tuple[str, str]]:
53+
"""A caller's headers as the app gets them: passed on without the control token's ``Authorization`` or any address
54+
the caller forwarded, and with the one it reached the driver at, its ``host``, ``scheme`` and the app's
55+
``prefix``."""
56+
return passed_on(headers, "authorization", *FORWARDED) + [
57+
("x-forwarded-host", host), ("x-forwarded-proto", scheme), ("x-forwarded-prefix", prefix)]
58+
59+
60+
def target(connection: Request | WebSocket) -> str:
61+
"""The path and query a request under ``/svc/<name>/`` asks its app for, as it came: still percent-encoded, so an
62+
escaped ``?``, ``#`` or ``/`` stays one. The raw path's segments past any the driver is mounted under must be
63+
``svc`` and the app's name, else the routed path is escaped again."""
64+
mounted = len([segment for segment in connection.scope.get("root_path", "").split("/") if segment])
65+
parts = (connection.scope.get("raw_path") or b"").decode("latin-1").split("/")[1 + mounted:]
66+
if len(parts) > 2 and [unquote(part) for part in parts[:2]] == ["svc", connection.path_params["name"]]:
67+
tail = "/".join(parts[2:])
68+
else:
69+
tail = quote(connection.path_params["path"], safe=PATH_SAFE)
70+
query = connection.scope.get("query_string", b"").decode("latin-1")
71+
return f"/{tail}" + (f"?{query}" if query else "")
72+
73+
74+
def _bytes(headers: Iterable[tuple[str, str]]) -> list[tuple[bytes, bytes]]:
75+
"""Headers decoded as Latin-1, as Starlette and the helpers here hold them, as the bytes they came as."""
76+
return [(name.encode("latin-1"), value.encode("latin-1")) for name, value in headers]
77+
78+
79+
def websocket_route(app: Any, path: str, endpoint: Callable) -> None:
80+
"""``endpoint`` served at ``path`` as a WebSocket route of ``app``, an mcp or a standalone fastmcp FastMCP, which
81+
have no API for one."""
82+
for routes in ("_custom_starlette_routes", "_additional_http_routes"):
83+
if isinstance(getattr(app, routes, None), list):
84+
getattr(app, routes).append(WebSocketRoute(path, endpoint))
85+
return
86+
raise TypeError(f"a driver's app takes WebSocket routes on an mcp or a fastmcp FastMCP, not a {type(app).__name__}")
87+
88+
89+
async def forward(folder: Path, name: str, request: Request) -> Response:
90+
"""``request`` sent on to the app ``name``'s server in ``folder`` with the caller's headers and none of the
91+
client's own, its body and its answer streamed; it takes as long as the app does."""
92+
headers = to_app(request.headers.items(), request.headers.get("host", ""), request.url.scheme, f"/svc/{name}")
93+
has_body = request.headers.get("content-length", "0") != "0" or "transfer-encoding" in request.headers
94+
http = httpx.AsyncClient(transport=httpx.AsyncHTTPTransport(uds=str(folder / SOCKET)), base_url="http://app",
95+
timeout=None)
96+
http.headers.clear()
97+
try:
98+
answer = await http.send(http.build_request(request.method, target(request), headers=_bytes(headers),
99+
content=request.stream() if has_body else None), stream=True)
100+
except (httpx.HTTPError, OSError) as e:
101+
await http.aclose()
102+
return _error_response(DeliveryFailed(f"app {name}'s server isn't answering: {type(e).__name__}: {e}"))
103+
except BaseException:
104+
await http.aclose()
105+
raise
106+
107+
async def close() -> None:
108+
await answer.aclose()
109+
await http.aclose()
110+
111+
async def streamed() -> AsyncIterator[bytes]:
112+
try:
113+
async for chunk in answer.aiter_raw():
114+
yield chunk
115+
finally:
116+
await close()
117+
118+
response = StreamingResponse(streamed(), status_code=answer.status_code, background=BackgroundTask(close))
119+
response.raw_headers = _bytes(passed_on((name.decode("latin-1"), value.decode("latin-1"))
120+
for name, value in answer.headers.raw))
121+
return response
122+
123+
124+
async def relay(folder: Path, name: str, websocket: WebSocket) -> None:
125+
"""``websocket`` relayed both ways to the app ``name``'s server in ``folder`` until either side closes; refused,
126+
before it opens, without the control token, with a Host that isn't a host and port, or with the app's server not
127+
answering."""
128+
host = websocket.headers.get("host", "")
129+
if not control_check()(websocket.headers) or (host and not HOST.fullmatch(host)):
130+
await websocket.close(code=1008)
131+
return
132+
headers = [(key, _utf8(value)) for key, value in to_app(
133+
websocket.headers.items(), host, "https" if websocket.url.scheme == "wss" else "http", f"/svc/{name}")
134+
if key.lower() != "host" and not key.lower().startswith("sec-websocket-")]
135+
offered = [p.strip() for p in websocket.headers.get("sec-websocket-protocol", "").split(",") if p.strip()]
136+
try:
137+
upstream = await unix_connect(str(folder / SOCKET), f"ws://{host or 'app'}{target(websocket)}",
138+
additional_headers=headers, subprotocols=offered or None, max_size=None,
139+
compression=None, ping_interval=None, user_agent_header=None)
140+
except (OSError, ValueError, asyncio.TimeoutError, websockets.WebSocketException):
141+
await websocket.close(code=1011)
142+
return
143+
async with upstream:
144+
await websocket.accept(subprotocol=upstream.subprotocol)
145+
await _both_ways(websocket, upstream)
146+
147+
148+
def _utf8(value: str) -> str:
149+
"""A header value decoded as Latin-1 as the string ``websockets`` sends as the same bytes, which it encodes as
150+
UTF-8; bytes that aren't UTF-8 become replacement characters."""
151+
return value.encode("latin-1").decode("utf-8", "replace")
152+
153+
154+
async def _both_ways(client: WebSocket, app: ClientConnection) -> None:
155+
"""Each message from either side sent to the other until one side closes, then the other closed with its code."""
156+
async def inbound() -> None:
157+
while True:
158+
message = await client.receive()
159+
if message["type"] == "websocket.disconnect":
160+
code = message.get("code") or 1000
161+
await app.close(1000 if code in UNSENDABLE_CLOSES else code)
162+
return
163+
await app.send(message["text"] if message.get("text") is not None else message["bytes"])
164+
165+
async def outbound() -> None:
166+
with contextlib.suppress(websockets.ConnectionClosed):
167+
async for message in app:
168+
if isinstance(message, str):
169+
await client.send_text(message)
170+
else:
171+
await client.send_bytes(message)
172+
code = app.close_code or 1000
173+
with contextlib.suppress(RuntimeError, WebSocketDisconnect):
174+
await client.close(1000 if code in UNSENDABLE_CLOSES else code)
175+
176+
tasks = [asyncio.ensure_future(inbound()), asyncio.ensure_future(outbound())]
177+
try:
178+
await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
179+
finally:
180+
for task in tasks:
181+
task.cancel()
182+
await asyncio.gather(*tasks, return_exceptions=True)

0 commit comments

Comments
 (0)