|
| 1 | +"""Whether each remote sandbox provider's ``WEBSOCKETS`` declaration is right: whether its tunnel carries what a |
| 2 | +website browser's CDP address needs. Through the tunnel, a CDP command is answered over a WebSocket, a 10 MB frame comes |
| 3 | +back intact, and after five idle minutes a connection pinged every 20 s, as agentenv's driver and gateway ping, still |
| 4 | +answers. A connection beside it that is never pinged records the tunnel's own idle limit, and doesn't decide. The server |
| 5 | +behind the tunnel is a stdlib one this test starts in a sandbox of the provider, a VM where it makes them, so only |
| 6 | +the tunnel is under test. |
| 7 | +
|
| 8 | +A case runs when the resolved config builds its provider (the ``remote_sandbox`` capability) and skips otherwise. Each |
| 9 | +prints its findings as a ``websockets-check`` JSON line. No test reads or prints a key.""" |
| 10 | + |
| 11 | +from __future__ import annotations |
| 12 | + |
| 13 | +import asyncio |
| 14 | +import base64 |
| 15 | +import json |
| 16 | +import os |
| 17 | +import time |
| 18 | + |
| 19 | +import httpx |
| 20 | +import pytest |
| 21 | +import websockets |
| 22 | +from websockets.asyncio.client import connect |
| 23 | + |
| 24 | +from agent_env.providers.sandbox_providers.sandbox import VmSandbox |
| 25 | +from agent_env.providers.sandbox_providers.sandbox_provider import build_sandbox_provider |
| 26 | +from tst.util.capabilities import skip_without_remote_sandbox |
| 27 | + |
| 28 | +pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow, pytest.mark.asyncio] |
| 29 | + |
| 30 | +#: The remote providers, each of which declares whether it carries WebSockets. |
| 31 | +REMOTE = ("e2b", "tensorlake", "vercel", "coreweave", "daytona", "boat", "modal", "modal_vm", "boxd", "sail_vm", "tenki", |
| 32 | + "agent37") |
| 33 | +PORT = 8765 |
| 34 | +FRAME_BYTES = 10 * 1024 * 1024 |
| 35 | +IDLE_S = 300 |
| 36 | +PING_S = 20.0 |
| 37 | +GET_VERSION = {"id": 1, "method": "Browser.getVersion"} |
| 38 | +#: What the server answers ``Browser.getVersion`` with. |
| 39 | +PRODUCT = "agentenv-websockets-check/1" |
| 40 | +SANDBOX_CPU = 1 |
| 41 | +SANDBOX_MEMORY_MB = 2048 |
| 42 | +SANDBOX_LIFETIME_S = 1800 |
| 43 | +SERVED_WITHIN_S = 300 |
| 44 | +SERVED_POLL_S = 3 |
| 45 | +HTTP_TIMEOUT_S = 10 |
| 46 | +OPEN_TIMEOUT_S = 60 |
| 47 | +ANSWER_TIMEOUT_S = 60 |
| 48 | +FRAME_TIMEOUT_S = 120 |
| 49 | +ERROR_CHARS = 300 |
| 50 | + |
| 51 | +#: A WebSocket server in the standard library alone: it answers ``Browser.getVersion`` as a browser would, echoes every |
| 52 | +#: other message, answers pings, and answers a plain GET with 200 for a tunnel's health check. |
| 53 | +SERVER = r''' |
| 54 | +import base64, hashlib, json, socketserver, struct, sys |
| 55 | +
|
| 56 | +GUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11" |
| 57 | +
|
| 58 | +
|
| 59 | +def read_message(stream): |
| 60 | + """A whole message, its fragments joined: (opcode, payload), or (None, b"") once the connection ends.""" |
| 61 | + opcode, parts = None, [] |
| 62 | + while True: |
| 63 | + head = stream.read(2) |
| 64 | + if len(head) < 2: |
| 65 | + return None, b"" |
| 66 | + fin, code, masked, length = head[0] & 0x80, head[0] & 0x0F, head[1] & 0x80, head[1] & 0x7F |
| 67 | + if length == 126: |
| 68 | + length = struct.unpack(">H", stream.read(2))[0] |
| 69 | + elif length == 127: |
| 70 | + length = struct.unpack(">Q", stream.read(8))[0] |
| 71 | + mask = stream.read(4) if masked else b"" |
| 72 | + payload = stream.read(length) |
| 73 | + if mask: |
| 74 | + payload = bytes(byte ^ mask[i % 4] for i, byte in enumerate(payload)) |
| 75 | + if code >= 8: # a control frame, never fragmented, may come between fragments |
| 76 | + return code, payload |
| 77 | + opcode = code if code else opcode |
| 78 | + parts.append(payload) |
| 79 | + if fin: |
| 80 | + return opcode, b"".join(parts) |
| 81 | +
|
| 82 | +
|
| 83 | +def send(conn, opcode, payload): |
| 84 | + length = len(payload) |
| 85 | + if length < 126: |
| 86 | + head = struct.pack(">BB", 0x80 | opcode, length) |
| 87 | + elif length < 1 << 16: |
| 88 | + head = struct.pack(">BBH", 0x80 | opcode, 126, length) |
| 89 | + else: |
| 90 | + head = struct.pack(">BBQ", 0x80 | opcode, 127, length) |
| 91 | + conn.sendall(head + payload) |
| 92 | +
|
| 93 | +
|
| 94 | +class Handler(socketserver.BaseRequestHandler): |
| 95 | + def handle(self): |
| 96 | + conn, data = self.request, b"" |
| 97 | + while b"\r\n\r\n" not in data: |
| 98 | + chunk = conn.recv(4096) |
| 99 | + if not chunk: |
| 100 | + return |
| 101 | + data += chunk |
| 102 | + lines = data.split(b"\r\n\r\n", 1)[0].decode("latin-1").split("\r\n") |
| 103 | + headers = {name.strip().lower(): value.strip() for name, _, value in (line.partition(":") for line in lines[1:])} |
| 104 | + key = headers.get("sec-websocket-key") |
| 105 | + if not key: |
| 106 | + conn.sendall(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok") |
| 107 | + return |
| 108 | + accept = base64.b64encode(hashlib.sha1((key + GUID).encode()).digest()).decode() |
| 109 | + conn.sendall(("HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n" |
| 110 | + "Sec-WebSocket-Accept: " + accept + "\r\n\r\n").encode()) |
| 111 | + stream = conn.makefile("rb") |
| 112 | + while True: |
| 113 | + opcode, payload = read_message(stream) |
| 114 | + if opcode is None or opcode == 8: |
| 115 | + if opcode == 8: |
| 116 | + send(conn, 8, payload[:2]) |
| 117 | + return |
| 118 | + if opcode == 9: |
| 119 | + send(conn, 10, payload) |
| 120 | + elif opcode == 1 and json.loads(payload).get("method") == "Browser.getVersion": |
| 121 | + answer = {"id": json.loads(payload)["id"], "result": {"product": sys.argv[2]}} |
| 122 | + send(conn, 1, json.dumps(answer).encode()) |
| 123 | + elif opcode in (1, 2): |
| 124 | + send(conn, opcode, payload) |
| 125 | +
|
| 126 | +
|
| 127 | +class Server(socketserver.ThreadingTCPServer): |
| 128 | + allow_reuse_address = True |
| 129 | + daemon_threads = True |
| 130 | +
|
| 131 | +
|
| 132 | +Server(("0.0.0.0", int(sys.argv[1])), Handler).serve_forever() |
| 133 | +''' |
| 134 | + |
| 135 | +PYTHON_IMAGE = "public.ecr.aws/docker/library/python:3.12-alpine" |
| 136 | +DIR = "/tmp/websockets-check" |
| 137 | + |
| 138 | + |
| 139 | +def _start_script() -> str: |
| 140 | + """Writes the server and starts it detached, in Docker where the sandbox has it, else on its own Python.""" |
| 141 | + encoded = base64.b64encode(SERVER.encode()).decode() |
| 142 | + return f"""set -e |
| 143 | +mkdir -p {DIR} |
| 144 | +echo {encoded} | base64 -d > {DIR}/server.py |
| 145 | +if command -v docker >/dev/null 2>&1; then |
| 146 | + docker run -d --name websockets-check -p {PORT}:{PORT} -v {DIR}:/check {PYTHON_IMAGE} python /check/server.py {PORT} {PRODUCT} >/dev/null |
| 147 | +else |
| 148 | + setsid nohup python3 {DIR}/server.py {PORT} {PRODUCT} > {DIR}/server.log 2>&1 < /dev/null & |
| 149 | +fi |
| 150 | +""" |
| 151 | + |
| 152 | + |
| 153 | +@pytest.mark.parametrize("name", [pytest.param(name, marks=skip_without_remote_sandbox(name)) for name in REMOTE]) |
| 154 | +async def test_a_providers_websockets_declaration_matches_what_its_tunnel_carries(name): |
| 155 | + provider = build_sandbox_provider(name) |
| 156 | + found: dict = {"provider": name, "declared": provider.WEBSOCKETS} |
| 157 | + try: |
| 158 | + sandbox = await _sandbox(provider) |
| 159 | + try: |
| 160 | + await _serve(sandbox) |
| 161 | + url = sandbox.tunnel_urls[PORT] |
| 162 | + await _until_served(url) |
| 163 | + found.update(await _checked("ws" + url.removeprefix("http"))) |
| 164 | + finally: |
| 165 | + await sandbox.terminate() |
| 166 | + finally: |
| 167 | + close = getattr(provider, "close", None) |
| 168 | + if close is not None: |
| 169 | + await close() |
| 170 | + found["carries"] = all(found.get(check) is True for check in ("cdp", "frame", "idle_pinged")) |
| 171 | + print("websockets-check", json.dumps(found)) |
| 172 | + |
| 173 | + assert found["carries"] == found["declared"], ( |
| 174 | + f"{name} declares WEBSOCKETS = {found['declared']}, but its tunnel " |
| 175 | + f"{'carries' if found['carries'] else 'does not carry'} what a website browser's CDP address needs: {found}") |
| 176 | + |
| 177 | + |
| 178 | +async def _sandbox(provider): |
| 179 | + """A VM, or on a provider that makes only containers, a container of a stock Python image.""" |
| 180 | + try: |
| 181 | + return await provider.create_vm(cpu=SANDBOX_CPU, memory=SANDBOX_MEMORY_MB, exposed_ports=[PORT], |
| 182 | + timeout=SANDBOX_LIFETIME_S) |
| 183 | + except NotImplementedError: |
| 184 | + return await provider.create_container(image_name=PYTHON_IMAGE, port=PORT, env={}, cpu=SANDBOX_CPU, |
| 185 | + memory=SANDBOX_MEMORY_MB, timeout=SANDBOX_LIFETIME_S) |
| 186 | + |
| 187 | + |
| 188 | +async def _serve(sandbox) -> None: |
| 189 | + if isinstance(sandbox, VmSandbox): |
| 190 | + await sandbox.exec_script(_start_script()) |
| 191 | + return |
| 192 | + exit_code, stdout, stderr = await sandbox.exec_with_output("sh", "-c", _start_script()) |
| 193 | + assert exit_code == 0, f"starting the server failed (exit {exit_code}): {stdout[-500:]} {stderr[-500:]}" |
| 194 | + |
| 195 | + |
| 196 | +async def _until_served(url: str, timeout_s: float = SERVED_WITHIN_S) -> None: |
| 197 | + deadline = time.monotonic() + timeout_s |
| 198 | + async with httpx.AsyncClient(timeout=HTTP_TIMEOUT_S) as client: |
| 199 | + while time.monotonic() < deadline: |
| 200 | + try: |
| 201 | + if (await client.get(url)).status_code == 200: |
| 202 | + return |
| 203 | + except httpx.HTTPError: |
| 204 | + pass |
| 205 | + await asyncio.sleep(SERVED_POLL_S) |
| 206 | + raise TimeoutError(f"nothing answered at {url} within {timeout_s:.0f} s") |
| 207 | + |
| 208 | + |
| 209 | +async def _checked(ws_url: str) -> dict: |
| 210 | + """What the tunnel at ``ws_url`` carries: a CDP answer, a 10 MB frame back intact, and, after ``IDLE_S`` idle, an |
| 211 | + answer on a pinged connection, and how long one never pinged lasted.""" |
| 212 | + found: dict = {} |
| 213 | + try: |
| 214 | + async with connect(ws_url, max_size=None, open_timeout=OPEN_TIMEOUT_S, ping_interval=None) as conn: |
| 215 | + found["cdp"] = await _version(conn) == PRODUCT |
| 216 | + frame = os.urandom(FRAME_BYTES) |
| 217 | + await conn.send(frame) |
| 218 | + found["frame"] = await asyncio.wait_for(conn.recv(), FRAME_TIMEOUT_S) == frame |
| 219 | + except (OSError, asyncio.TimeoutError, websockets.WebSocketException) as e: |
| 220 | + found.setdefault("cdp", False) |
| 221 | + found.setdefault("frame", False) |
| 222 | + found["error"] = _error(e) |
| 223 | + return found |
| 224 | + try: |
| 225 | + pinged = await connect(ws_url, max_size=None, open_timeout=OPEN_TIMEOUT_S, ping_interval=PING_S, |
| 226 | + ping_timeout=PING_S) |
| 227 | + except (OSError, asyncio.TimeoutError, websockets.WebSocketException) as e: |
| 228 | + found["idle_pinged"] = False |
| 229 | + found["idle_pinged_error"] = _error(e) |
| 230 | + return found |
| 231 | + try: |
| 232 | + silent = await connect(ws_url, max_size=None, open_timeout=OPEN_TIMEOUT_S, ping_interval=None) |
| 233 | + except (OSError, asyncio.TimeoutError, websockets.WebSocketException) as e: |
| 234 | + silent = None |
| 235 | + found["idle_unpinged_error"] = _error(e) |
| 236 | + started = time.monotonic() |
| 237 | + |
| 238 | + async def lasted(conn) -> int: |
| 239 | + await conn.wait_closed() |
| 240 | + return round(time.monotonic() - started) |
| 241 | + |
| 242 | + silent_lasted = asyncio.ensure_future(lasted(silent)) if silent is not None else None |
| 243 | + try: |
| 244 | + await asyncio.sleep(IDLE_S) |
| 245 | + if silent_lasted is not None: |
| 246 | + found["idle_unpinged_s"] = silent_lasted.result() if silent_lasted.done() else None |
| 247 | + try: |
| 248 | + found["idle_pinged"] = await _version(pinged) == PRODUCT |
| 249 | + except (asyncio.TimeoutError, websockets.WebSocketException) as e: |
| 250 | + found["idle_pinged"] = False |
| 251 | + found["idle_pinged_error"] = _error(e) |
| 252 | + finally: |
| 253 | + if silent_lasted is not None: |
| 254 | + silent_lasted.cancel() |
| 255 | + await silent.close() |
| 256 | + await pinged.close() |
| 257 | + return found |
| 258 | + |
| 259 | + |
| 260 | +async def _version(conn) -> str: |
| 261 | + await conn.send(json.dumps(GET_VERSION)) |
| 262 | + return json.loads(await asyncio.wait_for(conn.recv(), ANSWER_TIMEOUT_S))["result"]["product"] |
| 263 | + |
| 264 | + |
| 265 | +def _error(e: BaseException) -> str: |
| 266 | + return f"{type(e).__name__}: {e}"[:ERROR_CHARS] |
0 commit comments