Skip to content

Commit bfcfe1c

Browse files
feat(providers): a live WebSocket check decides each provider's declaration; boat and CoreWeave pass (#507)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
1 parent e04012e commit bfcfe1c

5 files changed

Lines changed: 271 additions & 3 deletions

File tree

‎src/agent_env/providers/sandbox_providers/AGENTS.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ sandboxes off this machine, runs containers' images by name, has no private netw
8686
| `CONTAINER_ACCEPTS` | `create_container` runs images other than by name: `Accepts.LOADABLE` (tar.gz, registry name or build context), `Accepts.NAME_OR_CONTEXT` |
8787
| `SANDBOX_ACCEPTS` | an agent's sandbox runs different images than the default (a provider that implements `create_vm` defaults to `LOADABLE`) |
8888
| `PRIVATE_NETWORK` | `create_container(private_network=True)` puts containers on a network they share |
89-
| `WEBSOCKETS` | its sandboxes' exposed ports carry WebSockets both ways, as our own tests or its tunnel's documentation show; a computer with a website browser among its apps, whose CDP address agents reach by WebSocket, refuses a provider without it at preflight, and a chain skips it at deploy |
89+
| `WEBSOCKETS` | its sandboxes' exposed ports carry WebSockets both ways, as `tst/integration/providers/sandbox_providers/websockets_live_test.py` or its tunnel's documentation show; a computer with a website browser among its apps, whose CDP address agents reach by WebSocket, refuses a provider without it at preflight, and a chain skips it at deploy |
9090

9191
## Sandbox handle contract
9292

‎src/agent_env/providers/sandbox_providers/boat/provider.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ class BoatSandboxProvider(SandboxProvider):
6262

6363
# boat serves hosted ports at https://<sandbox subdomain>-<port>.on.boat.dev.
6464
EGRESS_HOSTS: ClassVar[tuple[str, ...]] = ("*.on.boat.dev",)
65+
WEBSOCKETS = True
6566
_CREATE_ATTEMPTS = 6
6667
_RETRY_BASE_DELAY = 2.0
6768
_RETRY_MAX_DELAY = 60.0

‎src/agent_env/providers/sandbox_providers/coreweave/provider.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -196,6 +196,7 @@ class CoreWeaveSandboxProvider(SandboxProvider):
196196
# The platform pulls a container's image by its name; only a VM loads a tarball or builds a context.
197197
CONTAINER_ACCEPTS = Accepts.NAME
198198
SANDBOX_ACCEPTS = Accepts.NAME
199+
WEBSOCKETS = True
199200
# An exposed port's URL is <port>-<sandbox id>.<runner label>.<domain>, and a CoreWeave wildcard covers exactly one
200201
# label, so no one entry covers every runner. Each parent seen on an exposed port is added here, and every
201202
# provider's allowlists include it, so a restricted agent on any provider reaches environments served there.
Lines changed: 266 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,266 @@
1+
"""Whether each remote sandbox provider's ``WEBSOCKETS`` declaration is right: whether its tunnel carries what a
2+
website browser's CDP address needs. Through the tunnel, a CDP command is answered over a WebSocket, a 10 MB frame comes
3+
back intact, and after five idle minutes a connection pinged every 20 s, as agentenv's driver and gateway ping, still
4+
answers. A connection beside it that is never pinged records the tunnel's own idle limit, and doesn't decide. The server
5+
behind the tunnel is a stdlib one this test starts in a sandbox of the provider, a VM where it makes them, so only
6+
the tunnel is under test.
7+
8+
A case runs when the resolved config builds its provider (the ``remote_sandbox`` capability) and skips otherwise. Each
9+
prints its findings as a ``websockets-check`` JSON line. No test reads or prints a key."""
10+
11+
from __future__ import annotations
12+
13+
import asyncio
14+
import base64
15+
import json
16+
import os
17+
import time
18+
19+
import httpx
20+
import pytest
21+
import websockets
22+
from websockets.asyncio.client import connect
23+
24+
from agent_env.providers.sandbox_providers.sandbox import VmSandbox
25+
from agent_env.providers.sandbox_providers.sandbox_provider import build_sandbox_provider
26+
from tst.util.capabilities import skip_without_remote_sandbox
27+
28+
pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow, pytest.mark.asyncio]
29+
30+
#: The remote providers, each of which declares whether it carries WebSockets.
31+
REMOTE = ("e2b", "tensorlake", "vercel", "coreweave", "daytona", "boat", "modal", "modal_vm", "boxd", "sail_vm", "tenki",
32+
"agent37")
33+
PORT = 8765
34+
FRAME_BYTES = 10 * 1024 * 1024
35+
IDLE_S = 300
36+
PING_S = 20.0
37+
GET_VERSION = {"id": 1, "method": "Browser.getVersion"}
38+
#: What the server answers ``Browser.getVersion`` with.
39+
PRODUCT = "agentenv-websockets-check/1"
40+
SANDBOX_CPU = 1
41+
SANDBOX_MEMORY_MB = 2048
42+
SANDBOX_LIFETIME_S = 1800
43+
SERVED_WITHIN_S = 300
44+
SERVED_POLL_S = 3
45+
HTTP_TIMEOUT_S = 10
46+
OPEN_TIMEOUT_S = 60
47+
ANSWER_TIMEOUT_S = 60
48+
FRAME_TIMEOUT_S = 120
49+
ERROR_CHARS = 300
50+
51+
#: A WebSocket server in the standard library alone: it answers ``Browser.getVersion`` as a browser would, echoes every
52+
#: other message, answers pings, and answers a plain GET with 200 for a tunnel's health check.
53+
SERVER = r'''
54+
import base64, hashlib, json, socketserver, struct, sys
55+
56+
GUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"
57+
58+
59+
def read_message(stream):
60+
"""A whole message, its fragments joined: (opcode, payload), or (None, b"") once the connection ends."""
61+
opcode, parts = None, []
62+
while True:
63+
head = stream.read(2)
64+
if len(head) < 2:
65+
return None, b""
66+
fin, code, masked, length = head[0] & 0x80, head[0] & 0x0F, head[1] & 0x80, head[1] & 0x7F
67+
if length == 126:
68+
length = struct.unpack(">H", stream.read(2))[0]
69+
elif length == 127:
70+
length = struct.unpack(">Q", stream.read(8))[0]
71+
mask = stream.read(4) if masked else b""
72+
payload = stream.read(length)
73+
if mask:
74+
payload = bytes(byte ^ mask[i % 4] for i, byte in enumerate(payload))
75+
if code >= 8: # a control frame, never fragmented, may come between fragments
76+
return code, payload
77+
opcode = code if code else opcode
78+
parts.append(payload)
79+
if fin:
80+
return opcode, b"".join(parts)
81+
82+
83+
def send(conn, opcode, payload):
84+
length = len(payload)
85+
if length < 126:
86+
head = struct.pack(">BB", 0x80 | opcode, length)
87+
elif length < 1 << 16:
88+
head = struct.pack(">BBH", 0x80 | opcode, 126, length)
89+
else:
90+
head = struct.pack(">BBQ", 0x80 | opcode, 127, length)
91+
conn.sendall(head + payload)
92+
93+
94+
class Handler(socketserver.BaseRequestHandler):
95+
def handle(self):
96+
conn, data = self.request, b""
97+
while b"\r\n\r\n" not in data:
98+
chunk = conn.recv(4096)
99+
if not chunk:
100+
return
101+
data += chunk
102+
lines = data.split(b"\r\n\r\n", 1)[0].decode("latin-1").split("\r\n")
103+
headers = {name.strip().lower(): value.strip() for name, _, value in (line.partition(":") for line in lines[1:])}
104+
key = headers.get("sec-websocket-key")
105+
if not key:
106+
conn.sendall(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok")
107+
return
108+
accept = base64.b64encode(hashlib.sha1((key + GUID).encode()).digest()).decode()
109+
conn.sendall(("HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n"
110+
"Sec-WebSocket-Accept: " + accept + "\r\n\r\n").encode())
111+
stream = conn.makefile("rb")
112+
while True:
113+
opcode, payload = read_message(stream)
114+
if opcode is None or opcode == 8:
115+
if opcode == 8:
116+
send(conn, 8, payload[:2])
117+
return
118+
if opcode == 9:
119+
send(conn, 10, payload)
120+
elif opcode == 1 and json.loads(payload).get("method") == "Browser.getVersion":
121+
answer = {"id": json.loads(payload)["id"], "result": {"product": sys.argv[2]}}
122+
send(conn, 1, json.dumps(answer).encode())
123+
elif opcode in (1, 2):
124+
send(conn, opcode, payload)
125+
126+
127+
class Server(socketserver.ThreadingTCPServer):
128+
allow_reuse_address = True
129+
daemon_threads = True
130+
131+
132+
Server(("0.0.0.0", int(sys.argv[1])), Handler).serve_forever()
133+
'''
134+
135+
PYTHON_IMAGE = "public.ecr.aws/docker/library/python:3.12-alpine"
136+
DIR = "/tmp/websockets-check"
137+
138+
139+
def _start_script() -> str:
140+
"""Writes the server and starts it detached, in Docker where the sandbox has it, else on its own Python."""
141+
encoded = base64.b64encode(SERVER.encode()).decode()
142+
return f"""set -e
143+
mkdir -p {DIR}
144+
echo {encoded} | base64 -d > {DIR}/server.py
145+
if command -v docker >/dev/null 2>&1; then
146+
docker run -d --name websockets-check -p {PORT}:{PORT} -v {DIR}:/check {PYTHON_IMAGE} python /check/server.py {PORT} {PRODUCT} >/dev/null
147+
else
148+
setsid nohup python3 {DIR}/server.py {PORT} {PRODUCT} > {DIR}/server.log 2>&1 < /dev/null &
149+
fi
150+
"""
151+
152+
153+
@pytest.mark.parametrize("name", [pytest.param(name, marks=skip_without_remote_sandbox(name)) for name in REMOTE])
154+
async def test_a_providers_websockets_declaration_matches_what_its_tunnel_carries(name):
155+
provider = build_sandbox_provider(name)
156+
found: dict = {"provider": name, "declared": provider.WEBSOCKETS}
157+
try:
158+
sandbox = await _sandbox(provider)
159+
try:
160+
await _serve(sandbox)
161+
url = sandbox.tunnel_urls[PORT]
162+
await _until_served(url)
163+
found.update(await _checked("ws" + url.removeprefix("http")))
164+
finally:
165+
await sandbox.terminate()
166+
finally:
167+
close = getattr(provider, "close", None)
168+
if close is not None:
169+
await close()
170+
found["carries"] = all(found.get(check) is True for check in ("cdp", "frame", "idle_pinged"))
171+
print("websockets-check", json.dumps(found))
172+
173+
assert found["carries"] == found["declared"], (
174+
f"{name} declares WEBSOCKETS = {found['declared']}, but its tunnel "
175+
f"{'carries' if found['carries'] else 'does not carry'} what a website browser's CDP address needs: {found}")
176+
177+
178+
async def _sandbox(provider):
179+
"""A VM, or on a provider that makes only containers, a container of a stock Python image."""
180+
try:
181+
return await provider.create_vm(cpu=SANDBOX_CPU, memory=SANDBOX_MEMORY_MB, exposed_ports=[PORT],
182+
timeout=SANDBOX_LIFETIME_S)
183+
except NotImplementedError:
184+
return await provider.create_container(image_name=PYTHON_IMAGE, port=PORT, env={}, cpu=SANDBOX_CPU,
185+
memory=SANDBOX_MEMORY_MB, timeout=SANDBOX_LIFETIME_S)
186+
187+
188+
async def _serve(sandbox) -> None:
189+
if isinstance(sandbox, VmSandbox):
190+
await sandbox.exec_script(_start_script())
191+
return
192+
exit_code, stdout, stderr = await sandbox.exec_with_output("sh", "-c", _start_script())
193+
assert exit_code == 0, f"starting the server failed (exit {exit_code}): {stdout[-500:]} {stderr[-500:]}"
194+
195+
196+
async def _until_served(url: str, timeout_s: float = SERVED_WITHIN_S) -> None:
197+
deadline = time.monotonic() + timeout_s
198+
async with httpx.AsyncClient(timeout=HTTP_TIMEOUT_S) as client:
199+
while time.monotonic() < deadline:
200+
try:
201+
if (await client.get(url)).status_code == 200:
202+
return
203+
except httpx.HTTPError:
204+
pass
205+
await asyncio.sleep(SERVED_POLL_S)
206+
raise TimeoutError(f"nothing answered at {url} within {timeout_s:.0f} s")
207+
208+
209+
async def _checked(ws_url: str) -> dict:
210+
"""What the tunnel at ``ws_url`` carries: a CDP answer, a 10 MB frame back intact, and, after ``IDLE_S`` idle, an
211+
answer on a pinged connection, and how long one never pinged lasted."""
212+
found: dict = {}
213+
try:
214+
async with connect(ws_url, max_size=None, open_timeout=OPEN_TIMEOUT_S, ping_interval=None) as conn:
215+
found["cdp"] = await _version(conn) == PRODUCT
216+
frame = os.urandom(FRAME_BYTES)
217+
await conn.send(frame)
218+
found["frame"] = await asyncio.wait_for(conn.recv(), FRAME_TIMEOUT_S) == frame
219+
except (OSError, asyncio.TimeoutError, websockets.WebSocketException) as e:
220+
found.setdefault("cdp", False)
221+
found.setdefault("frame", False)
222+
found["error"] = _error(e)
223+
return found
224+
try:
225+
pinged = await connect(ws_url, max_size=None, open_timeout=OPEN_TIMEOUT_S, ping_interval=PING_S,
226+
ping_timeout=PING_S)
227+
except (OSError, asyncio.TimeoutError, websockets.WebSocketException) as e:
228+
found["idle_pinged"] = False
229+
found["idle_pinged_error"] = _error(e)
230+
return found
231+
try:
232+
silent = await connect(ws_url, max_size=None, open_timeout=OPEN_TIMEOUT_S, ping_interval=None)
233+
except (OSError, asyncio.TimeoutError, websockets.WebSocketException) as e:
234+
silent = None
235+
found["idle_unpinged_error"] = _error(e)
236+
started = time.monotonic()
237+
238+
async def lasted(conn) -> int:
239+
await conn.wait_closed()
240+
return round(time.monotonic() - started)
241+
242+
silent_lasted = asyncio.ensure_future(lasted(silent)) if silent is not None else None
243+
try:
244+
await asyncio.sleep(IDLE_S)
245+
if silent_lasted is not None:
246+
found["idle_unpinged_s"] = silent_lasted.result() if silent_lasted.done() else None
247+
try:
248+
found["idle_pinged"] = await _version(pinged) == PRODUCT
249+
except (asyncio.TimeoutError, websockets.WebSocketException) as e:
250+
found["idle_pinged"] = False
251+
found["idle_pinged_error"] = _error(e)
252+
finally:
253+
if silent_lasted is not None:
254+
silent_lasted.cancel()
255+
await silent.close()
256+
await pinged.close()
257+
return found
258+
259+
260+
async def _version(conn) -> str:
261+
await conn.send(json.dumps(GET_VERSION))
262+
return json.loads(await asyncio.wait_for(conn.recv(), ANSWER_TIMEOUT_S))["result"]["product"]
263+
264+
265+
def _error(e: BaseException) -> str:
266+
return f"{type(e).__name__}: {e}"[:ERROR_CHARS]

‎tst/unit/providers/sandbox_providers/provider_capabilities_test.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -61,9 +61,9 @@
6161
(TensorlakeSandboxProvider, False, True, LOADABLE, NAME, False, False),
6262
(TenkiSandboxProvider, False, True, LOADABLE, NAME, False, True),
6363
(BoxdSandboxProvider, False, True, LOADABLE, NAME, False, True),
64-
(CoreWeaveSandboxProvider, False, True, NAME, NAME, False, False),
64+
(CoreWeaveSandboxProvider, False, True, NAME, NAME, False, True),
6565
(DaytonaSandboxProvider, False, True, LOADABLE, NAME, False, False),
66-
(BoatSandboxProvider, False, True, LOADABLE, NAME, False, False),
66+
(BoatSandboxProvider, False, True, LOADABLE, NAME, False, True),
6767
(Agent37SandboxProvider, False, True, LOADABLE, NAME, False, True),
6868
], ids=["local", "modal", "modal_vm", "e2b", "sail_vm", "tensorlake", "tenki", "boxd", "coreweave", "daytona", "boat",
6969
"agent37"])

0 commit comments

Comments
 (0)