|
| 1 | +"""The CDP addresses a website browser hands agents: a token agentenv mints for each user, of which only a hash is kept |
| 2 | +where the computer's session can read it, discovery answered from the browser, and a relay that gives each WebSocket |
| 3 | +connection its own browser-level session over the browser's one debugging pipe.""" |
| 4 | +from __future__ import annotations |
| 5 | + |
| 6 | +import asyncio |
| 7 | +import contextlib |
| 8 | +import hashlib |
| 9 | +import json |
| 10 | +import logging |
| 11 | +import os |
| 12 | +import secrets |
| 13 | +from dataclasses import dataclass |
| 14 | +from datetime import datetime, timezone |
| 15 | +from pathlib import Path |
| 16 | +from typing import TYPE_CHECKING, Optional |
| 17 | + |
| 18 | +from starlette.requests import Request |
| 19 | +from starlette.websockets import WebSocket, WebSocketDisconnect |
| 20 | + |
| 21 | +if TYPE_CHECKING: |
| 22 | + from .website_browser import DevToolsPipe |
| 23 | + |
| 24 | +logger = logging.getLogger(__name__) |
| 25 | + |
| 26 | +#: How many random bytes a token holds. |
| 27 | +TOKEN_BYTES = 32 |
| 28 | +#: The route agents reach the relay under, through the driver's /svc/<app>/. |
| 29 | +ROUTE = "/cdp/" |
| 30 | +#: The close code and reason a connection gets when its browser goes away, which a client may connect again after. |
| 31 | +RELAUNCHED = (1012, "the browser relaunched: connect again") |
| 32 | +#: The close code and reason a connection gets when its client falls behind by more than ``OUTBOX_LIMIT_BYTES``. |
| 33 | +TOO_SLOW = (1013, "the client fell too far behind: connect again") |
| 34 | +#: How much a connection's client may have waiting to be sent to it before it is closed; a single answer bigger than |
| 35 | +#: this still goes when nothing else waits. |
| 36 | +OUTBOX_LIMIT_BYTES = 64 << 20 |
| 37 | +#: CDP's own answer to a command for a session the connection doesn't have. |
| 38 | +NO_SESSION = {"code": -32001, "message": "Session with given id not found."} |
| 39 | + |
| 40 | + |
| 41 | +@dataclass |
| 42 | +class Address: |
| 43 | + """A user's CDP address: ``http_url``, the base clients discover from, and ``ws_url``, the browser's WebSocket.""" |
| 44 | + |
| 45 | + user: str |
| 46 | + http_url: str |
| 47 | + ws_url: str |
| 48 | + |
| 49 | + |
| 50 | +def reached(request: Request) -> str: |
| 51 | + """The address a request reached the browser at, scheme, host and path prefix: what the driver forwarded, else the |
| 52 | + request's own.""" |
| 53 | + proto = request.headers.get("x-forwarded-proto") or request.url.scheme |
| 54 | + host = request.headers.get("x-forwarded-host") or request.headers.get("host") or request.url.netloc |
| 55 | + return f"{proto}://{host}{request.headers.get('x-forwarded-prefix', '')}".rstrip("/") |
| 56 | + |
| 57 | + |
| 58 | +def websocket_form(url: str) -> str: |
| 59 | + return "wss://" + url.removeprefix("https://") if url.startswith("https://") else \ |
| 60 | + "ws://" + url.removeprefix("http://") |
| 61 | + |
| 62 | + |
| 63 | +def _hash(token: str) -> str: |
| 64 | + return hashlib.sha256(token.encode()).hexdigest() |
| 65 | + |
| 66 | + |
| 67 | +def _address(user: str, base: str, token: str) -> Address: |
| 68 | + root = f"{base}{ROUTE}{token}" |
| 69 | + return Address(user, f"{root}/", f"{websocket_form(root)}/devtools/browser") |
| 70 | + |
| 71 | + |
| 72 | +class Tokens: |
| 73 | + """The tokens minted, one per user: in memory, so asking twice gives the same address, and in ``path`` only by |
| 74 | + hash, with the user and the base address, so an address still works after the server restarts while the |
| 75 | + computer's session, which can read the file, can't use one.""" |
| 76 | + |
| 77 | + def __init__(self, path: Path) -> None: |
| 78 | + self.path = path |
| 79 | + self._minted: dict[str, Address] = {} |
| 80 | + try: |
| 81 | + stored = json.loads(path.read_text()) |
| 82 | + self._known = {digest: (entry["user"], entry["base"]) for digest, entry in stored.items()} |
| 83 | + except (OSError, ValueError, TypeError, KeyError): |
| 84 | + self._known = {} |
| 85 | + |
| 86 | + def mint(self, user: str, base: str) -> Address: |
| 87 | + """``user``'s address under ``base``: the one minted before, else a new one.""" |
| 88 | + if user not in self._minted: |
| 89 | + token = secrets.token_urlsafe(TOKEN_BYTES) |
| 90 | + self._minted[user] = _address(user, base, token) |
| 91 | + self._known[_hash(token)] = (user, base) |
| 92 | + self._save() |
| 93 | + return self._minted[user] |
| 94 | + |
| 95 | + def address(self, token: str) -> Optional[Address]: |
| 96 | + """The address ``token`` was minted for, or None for one that wasn't.""" |
| 97 | + known = self._known.get(_hash(token)) |
| 98 | + return _address(*known, token) if known is not None else None |
| 99 | + |
| 100 | + def _save(self) -> None: |
| 101 | + self.path.parent.mkdir(parents=True, exist_ok=True) |
| 102 | + part = self.path.with_name(f".{self.path.name}.part") |
| 103 | + part.write_text(json.dumps({digest: {"user": user, "base": base} |
| 104 | + for digest, (user, base) in self._known.items()})) |
| 105 | + os.replace(part, self.path) |
| 106 | + |
| 107 | + |
| 108 | +class _Connection: |
| 109 | + """One client's WebSocket, its sessions over the pipe (its browser-level one and those it attached), its unanswered |
| 110 | + commands, and what waits to be sent to it.""" |
| 111 | + |
| 112 | + def __init__(self, websocket: WebSocket, user: str, session: str) -> None: |
| 113 | + self.websocket, self.user, self.session = websocket, user, session |
| 114 | + self.sessions = {session} |
| 115 | + self.commands: set[asyncio.Future] = set() |
| 116 | + self.outbox: asyncio.Queue[str | tuple[int, str]] = asyncio.Queue() |
| 117 | + self.waiting_bytes = 0 |
| 118 | + self.closing = False |
| 119 | + |
| 120 | + |
| 121 | +class Relay: |
| 122 | + """Each WebSocket connection given its own browser-level session over ``devtools``, its commands sent on it and the |
| 123 | + answers and events of its sessions sent back, so several clients share one browser; and a record of who connected |
| 124 | + when.""" |
| 125 | + |
| 126 | + def __init__(self, devtools: DevToolsPipe) -> None: |
| 127 | + self.devtools = devtools |
| 128 | + self.connections: list[dict] = [] |
| 129 | + self._by_session: dict[str, _Connection] = {} |
| 130 | + self._open: set[_Connection] = set() |
| 131 | + devtools.listen(self._event) |
| 132 | + |
| 133 | + async def serve(self, websocket: WebSocket, user: str) -> None: |
| 134 | + """``websocket``, a client of ``user``'s, relayed over a browser-level session of its own until either ends.""" |
| 135 | + try: |
| 136 | + session = (await self.devtools.call("Target.attachToBrowserTarget"))["sessionId"] |
| 137 | + except Exception as e: |
| 138 | + await websocket.close(code=1013, reason=f"the browser isn't answering: {type(e).__name__}") |
| 139 | + return |
| 140 | + connection = _Connection(websocket, user, session) |
| 141 | + self._by_session[session] = connection |
| 142 | + self._open.add(connection) |
| 143 | + record = {"user": user, "opened_at": _now()} |
| 144 | + self.connections.append(record) |
| 145 | + await websocket.accept() |
| 146 | + sending = asyncio.ensure_future(self._send(connection)) |
| 147 | + try: |
| 148 | + while True: |
| 149 | + message = await websocket.receive() |
| 150 | + if message["type"] == "websocket.disconnect": |
| 151 | + break |
| 152 | + text = message.get("text") if message.get("text") is not None else message["bytes"].decode() |
| 153 | + command = asyncio.ensure_future(self._command(connection, json.loads(text))) |
| 154 | + connection.commands.add(command) |
| 155 | + command.add_done_callback(connection.commands.discard) |
| 156 | + except (WebSocketDisconnect, ValueError): |
| 157 | + pass |
| 158 | + finally: |
| 159 | + record["closed_at"] = _now() |
| 160 | + connection.closing = True |
| 161 | + self._end(connection) |
| 162 | + for task in [*connection.commands, sending]: |
| 163 | + task.cancel() |
| 164 | + await asyncio.gather(*connection.commands, sending, return_exceptions=True) |
| 165 | + with contextlib.suppress(Exception): |
| 166 | + await self.devtools.call("Target.detachFromTarget", {"sessionId": session}) |
| 167 | + |
| 168 | + def ended(self) -> None: |
| 169 | + """The browser has gone: every connection is closed with a reason a client may connect again after.""" |
| 170 | + for connection in list(self._open): |
| 171 | + self._close(connection, RELAUNCHED) |
| 172 | + |
| 173 | + async def _command(self, connection: _Connection, message: dict) -> None: |
| 174 | + session = message.get("sessionId") or connection.session |
| 175 | + if session not in connection.sessions: |
| 176 | + answer = {"error": NO_SESSION} |
| 177 | + else: |
| 178 | + try: |
| 179 | + answer = await self.devtools.request(message.get("method", ""), message.get("params") or {}, |
| 180 | + session_id=session) |
| 181 | + except ConnectionError as e: |
| 182 | + answer = {"error": {"code": -32000, "message": str(e)}} |
| 183 | + reply = {"id": message.get("id"), **{key: answer[key] for key in ("result", "error") if key in answer}} |
| 184 | + if message.get("sessionId"): |
| 185 | + reply["sessionId"] = message["sessionId"] |
| 186 | + self._put(connection, reply) |
| 187 | + |
| 188 | + def _event(self, message: dict) -> None: |
| 189 | + connection = self._by_session.get(message.get("sessionId")) |
| 190 | + if connection is None: |
| 191 | + return |
| 192 | + params = message.get("params") or {} |
| 193 | + if message.get("method") == "Target.attachedToTarget" and params.get("sessionId"): |
| 194 | + connection.sessions.add(params["sessionId"]) |
| 195 | + self._by_session[params["sessionId"]] = connection |
| 196 | + elif message.get("method") == "Target.detachedFromTarget" and params.get("sessionId"): |
| 197 | + connection.sessions.discard(params["sessionId"]) |
| 198 | + self._by_session.pop(params["sessionId"], None) |
| 199 | + if message["sessionId"] == connection.session: |
| 200 | + message = {key: value for key, value in message.items() if key != "sessionId"} |
| 201 | + self._put(connection, message) |
| 202 | + |
| 203 | + def _put(self, connection: _Connection, message: dict) -> None: |
| 204 | + """``message`` waiting to be sent to ``connection``'s client, which is closed instead, with what waited dropped, |
| 205 | + once it has fallen too far behind.""" |
| 206 | + if connection.closing: |
| 207 | + return |
| 208 | + text = json.dumps(message) |
| 209 | + if connection.waiting_bytes and connection.waiting_bytes + len(text) > OUTBOX_LIMIT_BYTES: |
| 210 | + self._close(connection, TOO_SLOW) |
| 211 | + return |
| 212 | + connection.waiting_bytes += len(text) |
| 213 | + connection.outbox.put_nowait(text) |
| 214 | + |
| 215 | + def _close(self, connection: _Connection, close: tuple[int, str]) -> None: |
| 216 | + """``connection`` closed with ``close`` once what's being sent to it has gone, and nothing else sent.""" |
| 217 | + connection.closing = True |
| 218 | + self._end(connection) |
| 219 | + while not connection.outbox.empty(): |
| 220 | + connection.outbox.get_nowait() |
| 221 | + connection.waiting_bytes = 0 |
| 222 | + connection.outbox.put_nowait(close) |
| 223 | + |
| 224 | + def _end(self, connection: _Connection) -> None: |
| 225 | + self._open.discard(connection) |
| 226 | + for session in connection.sessions: |
| 227 | + self._by_session.pop(session, None) |
| 228 | + |
| 229 | + async def _send(self, connection: _Connection) -> None: |
| 230 | + with contextlib.suppress(WebSocketDisconnect, RuntimeError): |
| 231 | + while True: |
| 232 | + message = await connection.outbox.get() |
| 233 | + if isinstance(message, tuple): |
| 234 | + await connection.websocket.close(code=message[0], reason=message[1]) |
| 235 | + return |
| 236 | + connection.waiting_bytes -= len(message) |
| 237 | + await connection.websocket.send_text(message) |
| 238 | + |
| 239 | + |
| 240 | +def _now() -> str: |
| 241 | + return datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z") |
0 commit comments