diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index 38841a1aa..401c5ea65 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -544,6 +544,53 @@ class GitHub(WebsiteEnvironment): - **The run:** a run that deploys a live site is marked live. The deploy record says `live`, and the run's metadata lists its live sites with `reproducible: false`, because the real site may change between runs. +A third-party site is an app someone else wrote, such as a web store, that serves its own pages, wrapped so agentenv +deploys it like any site. `ForwardingSite` runs in the app's container beside it: it answers the card, the data plane +and `website/v1` itself, and forwards every other request to the app. + +```python +from agentenv_protocol.website import ForwardingSite, session_state, website + + +@website(domains=["store.localhost"], builtin_routes=False, clock="real", ready_timeout_s=600) +class Store(ForwardingSite): + upstream = "http://127.0.0.1:80" # the app, in the same container + ready_path = "/admin" # ready once the app answers this below 500 + sign_in_headers = ("X-Auto-Login",) # only the site's own calls send these; logged with values removed + + @session_state + async def signed_in(self, user: str | None) -> dict: + answer = await self.ask_app("/admin", headers={"X-Auto-Login": "..."}) + ... # return the cookies the app answered with +``` + +- **Forwarding:** a request goes to the app as it came: its method, the site's own path with its encoding intact (an + escaped `/` stays escaped, and a forwarded prefix is taken off), its query, the `Host` the browser named and its + body, streamed, with no header or cookie of the forwarder's own. The hop-by-hop headers and `sign_in_headers` stay + behind. The answer comes back as the app gave it, streamed and still encoded, a redirect unfollowed, but a cookie + the app ties to one of the site's names (`Domain`) comes back the host's alone, as agentenv sets a sign-in's + cookies, so a sign-in replaces the app's session cookie rather than setting a second one beside it. An app not + answering is a 502 (`upstream_unavailable`). It forwards HTTP alone, and refuses a WebSocket with close code 1011. +- **`builtin_routes=False`** serves none of agentenv's own site routes, so `/api/session` reaches the app too, and the + card names no `session_path` or `session_cookie`. Any `WebsiteEnvironment` may set it. +- **`clock="real"`** declares a site that keeps real time, as an app that reads the system clock does: its card serves + no `clock/v1`, its `website/v1` params say `clock: "real"` (`WEBSITE_CLOCK_PARAM`), its answers carry no + `X-Universe-Now`, and `sync_env_clock` passes it over. The conformance check skips its two clock checks. + `@website` refuses `clock_patch` with it, since there's no universe time to put in a page. +- **Its data plane:** the app keeps its own data, so the site serves `data/get` alone, which exports nothing. It takes + no load and can't reset: agentenv's reset passes over a card serving no `data/reset`, and the conformance check + passes a card serving `data/get` alone once `data/get` answers. +- **Its session:** `@session_state` signs the browser in with whatever cookies the app sets, for example from one + server-side request carrying a sign-in header, which `sign_in_headers` then keeps the browser from sending. The + card names them too (`WEBSITE_SIGN_IN_HEADERS_PARAM`), so should a browser's request carry one, the browser's + network log, and so `network.har`, keeps it with its value taken out. +- **Its own calls:** `ask_app(path, method=..., headers=..., content=...)` asks the app as the site does, under its + first name as `Host`, through the forwarder's client, which keeps no cookie and follows no redirect. Its session + and ready checks use it, as `ForwardingSite`'s own ready check does. +- **Its names:** an app with a fixed base URL answers any other `Host` with a redirect to its own name. A browser + reaches it by that name alone, the site proxy naming it, and the conformance check's `any_host` passes `/` redirecting + to one of the card's `domains` as it does `/` serving HTML. + ## Computer drivers A computer driver is what runs inside a computer an agent uses through screenshots and input. Write diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/svc.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/svc.py index 1581e1acb..b68df7689 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/svc.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/svc.py @@ -21,14 +21,12 @@ from websockets.asyncio.client import ClientConnection from ..agent_env_environment import _error_response +from ..passthrough import passed_on from .errors import DeliveryFailed from .served import TOKEN_HEADER, served #: The methods the route forwards. METHODS = ["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"] -#: The headers that belong to one connection, which a proxy doesn't pass on. -HOP_HEADERS = frozenset({"connection", "keep-alive", "proxy-authenticate", "proxy-authorization", "proxy-connection", - "te", "trailer", "transfer-encoding", "upgrade"}) #: The address a request reached the driver at, which the driver sets for the app in place of any the caller sent. FORWARDED = ("x-forwarded-host", "x-forwarded-proto", "x-forwarded-prefix") #: A WebSocket close code that means the connection ended without one, which a close can't send on. @@ -45,16 +43,6 @@ FORWARDER_LOGGER = "httpx" -def passed_on(headers: Iterable[tuple[str, str]], *dropped: str) -> list[tuple[str, str]]: - """``headers`` as a proxy passes them on: without the hop-by-hop ones, those a ``Connection`` header names, and - ``dropped``.""" - headers = list(headers) - named = {token.strip().lower() for name, value in headers if name.lower() == "connection" - for token in value.split(",")} - skipped = HOP_HEADERS | named | set(dropped) - return [(name, value) for name, value in headers if name.lower() not in skipped] - - def to_app(headers: Iterable[tuple[str, str]], host: str, scheme: str, prefix: str) -> list[tuple[str, str]]: """A caller's headers as the app gets them: passed on without the control token's ``Authorization`` or any address the caller forwarded, nor any app token, which the driver sets itself where the app takes one, and with the diff --git a/packages/agentenv-protocol/src/agentenv_protocol/passthrough.py b/packages/agentenv-protocol/src/agentenv_protocol/passthrough.py new file mode 100644 index 000000000..5162a5d8e --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/passthrough.py @@ -0,0 +1,19 @@ +"""What a proxy passes on of the headers it relays, shared by the driver's ``/svc//`` route and a website that +forwards to an app beside it.""" +from __future__ import annotations + +from collections.abc import Iterable + +#: The headers that belong to one connection, which a proxy doesn't pass on. +HOP_HEADERS = frozenset({"connection", "keep-alive", "proxy-authenticate", "proxy-authorization", "proxy-connection", + "te", "trailer", "transfer-encoding", "upgrade"}) + + +def passed_on(headers: Iterable[tuple[str, str]], *dropped: str) -> list[tuple[str, str]]: + """``headers`` as a proxy passes them on: without the hop-by-hop ones, those a ``Connection`` header names, and + ``dropped``, each matched without regard to case.""" + headers = list(headers) + named = {token.strip().lower() for name, value in headers if name.lower() == "connection" + for token in value.split(",")} + skipped = HOP_HEADERS | named | {name.lower() for name in dropped} + return [(name, value) for name, value in headers if name.lower() not in skipped] diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/__init__.py b/packages/agentenv-protocol/src/agentenv_protocol/website/__init__.py index de53f2176..28a7ed153 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website/__init__.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/__init__.py @@ -1,8 +1,8 @@ """The website SDK: a site's server, meeting ``urn:agentenv:website/v1``. Subclass WebsiteEnvironment, declare the site with ``@website(domains=[...], static="dist/")``, mark its API with ``@api.get/post/put/patch/delete(path)``, and -``serve()`` it; it needs only the plain package, and uvicorn to serve. The class is optional: a server in any language -that meets the contract works, which ``python -m agentenv_protocol.website.conformance `` checks. Whether a site -is ready is its ``WebsiteReadiness`` answer to ``ready``, which ``website_readiness`` reads of one site and +``serve()`` it, or wrap an app that serves its own pages with ``ForwardingSite``; it needs only the plain package, and +uvicorn to serve. The class is optional: a server in any language that meets the contract works, which ``python -m +agentenv_protocol.website.conformance `` checks. Whether a site is ready is its ``WebsiteReadiness`` answer to ``ready``, which ``website_readiness`` reads of one site and ``env_readiness`` of every site of an env. What a task left a site with is its data plane's ``data/get``.""" from __future__ import annotations @@ -19,15 +19,20 @@ website, ) from .errors import ApiError, BadRequest, Conflict, Forbidden, NotFound, Unauthorized +from .forwarding import ForwardingSite from .readiness import ( DEFAULT_HEALTH_PATH, READY_TIMEOUT_S, + REAL_CLOCK, SITE_READY_TIMEOUT_S, + UNIVERSE_CLOCK, WEBSITE_ALLOW_HOSTS_PARAM, WEBSITE_BROWSE_URL_PARAM, + WEBSITE_CLOCK_PARAM, WEBSITE_EXTENSION_URI, WEBSITE_LIVE_PARAM, WEBSITE_READY_OP, + WEBSITE_SIGN_IN_HEADERS_PARAM, WebsiteReadiness, env_readiness, website_readiness, @@ -38,9 +43,10 @@ __all__ = [ "WebsiteEnvironment", "WebsiteDeclaration", "website", "api", "ApiContext", "ApiRoute", "ready_check", "session_state", "host_problem", "ApiError", "BadRequest", "Unauthorized", "Forbidden", "NotFound", "Conflict", - "UniverseClock", "ClockUnavailable", "SessionSigner", "reserved_path", + "UniverseClock", "ClockUnavailable", "SessionSigner", "reserved_path", "ForwardingSite", "WEBSITE_EXTENSION_URI", "CLOCK_EXTENSION_URI", "UNIVERSE_NOW_HEADER", "DEFAULT_HEALTH_PATH", "SESSION_PATH", "SESSION_COOKIE", "SESSION_SECRET_ENV", "ROLE_KEY_ENV", "ROLE_KEY_HEADER", - "WEBSITE_READY_OP", "WEBSITE_BROWSE_URL_PARAM", "WEBSITE_LIVE_PARAM", "WEBSITE_ALLOW_HOSTS_PARAM", "WEBSITE_SETTINGS_ENV", "READY_TIMEOUT_S", "SITE_READY_TIMEOUT_S", "WebsiteReadiness", + "WEBSITE_READY_OP", "WEBSITE_BROWSE_URL_PARAM", "WEBSITE_LIVE_PARAM", "WEBSITE_ALLOW_HOSTS_PARAM", "WEBSITE_CLOCK_PARAM", + "WEBSITE_SIGN_IN_HEADERS_PARAM", "UNIVERSE_CLOCK", "REAL_CLOCK", "WEBSITE_SETTINGS_ENV", "READY_TIMEOUT_S", "SITE_READY_TIMEOUT_S", "WebsiteReadiness", "website_readiness", "env_readiness", ] diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/conformance.py b/packages/agentenv-protocol/src/agentenv_protocol/website/conformance.py index a22850aa2..cb7ed030b 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website/conformance.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/conformance.py @@ -3,16 +3,19 @@ ``--strict``, when a recommended one warned), 2 when the server couldn't be reached. Everything goes over HTTP. The required checks are the contract, what agentenv, the gateway and a browser rely on of a website env's server: a -card declaring the site's domains, readiness and routes; ``/`` serving HTML under a made-up host; the site ready, by -the card's ``website/v1`` ``ready``, whose answer must be a ``WebsiteReadiness`` (by the declared health path, -deprecated, for a card without it); the data plane's and card's paths not the site's pages; and the data plane the -card advertises, whose round trip at the card's ``url`` (data/get, data/reset, data/get, which no longer returns the -exported data, data/add of what the first data/get returned, data/get again) returns what it was given; and, for a -card declaring an ``mcp`` interface, a site that is an MCP server too, that endpoint answering ``initialize`` and -listing, every page of ``tools/list`` read, no tool its card doesn't (streamable HTTP, ``MCP_PROTOCOL_VERSION``); a -card's tool it doesn't list is warned, since a hidden-by-default tool is listed to no caller until one enables it. -A failed one is ``failed``. The recommended checks are what an SDK site does and the gateway uses when a server offers it: -``X-Universe-Now`` on every answer the other checks got, ``clock/v1``'s ``sync_time`` against a clock the checker serves, the forwarded headers +card declaring the site's domains, readiness and routes; ``/`` serving HTML under a made-up host, or redirecting to +one of the site's own names; the site ready, by the card's ``website/v1`` ``ready``, whose answer must be a +``WebsiteReadiness`` (by the declared health path, deprecated, for a card without it); the data plane's and card's +paths not the site's pages; and the data plane the card advertises, whose round trip at the card's ``url`` (data/get, +data/reset, data/get, which no longer returns the exported data, data/add of what the first data/get returned, +data/get again) returns what it was given, or, for a card serving ``data/get`` alone, a site whose app keeps its own +data and so takes no load and can't reset, which answers it; and, for a card declaring an ``mcp`` interface, a site +that is an MCP server too, that endpoint answering ``initialize`` and listing, every page of ``tools/list`` read, no +tool its card doesn't (streamable HTTP, ``MCP_PROTOCOL_VERSION``); a card's tool it doesn't list is warned, since a +hidden-by-default tool is listed to no caller until one enables it. A failed one is ``failed``. The recommended checks +are what an SDK site does and the gateway uses when a server offers it: ``X-Universe-Now`` on every answer the other +checks got, ``clock/v1``'s ``sync_time`` against a clock the checker serves (both skipped for a site whose +``website/v1`` params declare ``clock: "real"``, a third-party app that keeps real time), the forwarded headers wherever an answer shows them (on ``/`` and on a GET route with its trailing slash toggled, which a server may redirect), and the cookies those answers set: none tied to a domain, the session path's the one the card names, and none whose lifetime is an absolute ``Expires`` with no ``Max-Age``. A failed one is ``warned``. @@ -36,6 +39,7 @@ import uuid from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from typing import Callable, Iterator, Literal +from urllib.parse import urlsplit import httpx from pydantic import ValidationError @@ -43,9 +47,12 @@ from ..client import find_extension, find_extension_method from ..types import MCP_TRANSPORT, METHOD_ADD, METHOD_GET, METHOD_RESET, RPC_PATH, WELL_KNOWN_PATH from .clock import CLOCK_EXTENSION_URI, UNIVERSE_NOW_HEADER, iso, parse_time -from .readiness import DEFAULT_HEALTH_PATH, WEBSITE_EXTENSION_URI, WEBSITE_READY_OP, WebsiteReadiness +from .readiness import (DEFAULT_HEALTH_PATH, REAL_CLOCK, WEBSITE_CLOCK_PARAM, WEBSITE_EXTENSION_URI, WEBSITE_READY_OP, + WebsiteReadiness, names_host) Status = Literal["passed", "failed", "skipped", "warned"] +#: The checks of a site's clock, which a card declaring a real clock skips. +CLOCK_CHECKS = ("universe_now", "clock") CLOCK_START = datetime.datetime(2031, 5, 6, 7, 8, 9, tzinfo=datetime.timezone.utc) CLOCK_SLACK_S = 120 MCP_PROTOCOL_VERSION = "2025-06-18" @@ -114,14 +121,21 @@ def run(self) -> list[Result]: site = (find_extension(card, WEBSITE_EXTENSION_URI) or {}).get("params") or {} health = site.get("health") if isinstance(site.get("health"), str) else DEFAULT_HEALTH_PATH # (name, required, check): a recommended check's failure is warned. - steps = (("card", True, lambda: self._card(response, card, site)), ("any_host", True, self._any_host), + steps = (("card", True, lambda: self._card(response, card, site)), ("any_host", True, lambda: self._any_host(site)), ("health", True, lambda: self._health(card, health)), ("universe_now", False, self._universe_now), ("reserved_paths", True, self._reserved_paths), ("data_plane", True, lambda: self._data_plane(card)), ("mcp", True, lambda: self._mcp(card)), ("clock", False, lambda: self._clock(card)), ("forwarded", False, lambda: self._forwarded(site)), ("cookies", False, lambda: self._cookies(site))) - outcomes = {name: _outcome(step) for name, _required, step in steps if name != "universe_now"} - outcomes["universe_now"] = _outcome(self._universe_now) + real_clock = site.get(WEBSITE_CLOCK_PARAM) == REAL_CLOCK + + def outcome(name: str, step: Callable[[], tuple[Status, str]]) -> tuple[Status, str]: + if real_clock and name in CLOCK_CHECKS: + return "skipped", "the card declares a real clock, which nothing moves" + return _outcome(step) + + outcomes = {name: outcome(name, step) for name, _required, step in steps if name != "universe_now"} + outcomes["universe_now"] = outcome("universe_now", self._universe_now) results = [] for name, required, _step in steps: status, detail = outcomes[name] @@ -145,13 +159,20 @@ def _card(self, response: httpx.Response, card: dict, site: dict) -> tuple[Statu problems.append("its params' routes aren't a list of {method, path, name}") return ("failed" if problems else "passed"), "; ".join(problems) - def _any_host(self) -> tuple[Status, str]: + def _any_host(self, site: dict) -> tuple[Status, str]: + """``/`` under a made-up Host serves HTML, or redirects to one of the site's own names: an app with a fixed + base URL, which a browser reaches only by its name.""" host = f"conformance-{uuid.uuid4().hex[:8]}.invalid" - response = self.get("/", headers={"Host": host}) + response = self.get("/", headers={"Host": host}, follow_redirects=False) kind = response.headers.get("content-type", "") if response.status_code == 200 and kind.startswith("text/html"): return "passed", f"/ served HTML for Host {host}" - return "failed", f"/ for Host {host} answered {response.status_code} {kind or 'without a content type'}" + to = (urlsplit(response.headers.get("location", "")).hostname or "") if response.is_redirect else "" + domains = site.get("domains") if isinstance(site.get("domains"), list) else [] + if to and any(isinstance(domain, str) and names_host(domain.lower(), to) for domain in domains): + return "passed", f"/ for Host {host} redirected to the site's own name {to}" + return "failed", (f"/ for Host {host} answered {response.status_code} {kind or 'without a content type'}" + + (f", redirecting to {to}, not one of the site's names" if to else "")) def _health(self, card: dict, health: str) -> tuple[Status, str]: ready = find_extension_method(card, WEBSITE_EXTENSION_URI, WEBSITE_READY_OP) @@ -208,11 +229,15 @@ def _data_plane(self, card: dict) -> tuple[Status, str]: operations = (card.get("capabilities") or {}).get("operations") operations = operations if isinstance(operations, list) else [] missing = [m for m in (METHOD_GET, METHOD_RESET, METHOD_ADD) if m not in operations] - if missing: + get_alone = set(missing) == {METHOD_RESET, METHOD_ADD} + if missing and not get_alone: return "failed", f"the card's operations lack {', '.join(missing)}" url = card.get("url") self.rpc_url = self._url(url if isinstance(url, str) and url else RPC_PATH) exported = self._rpc(METHOD_GET).get("parts") or [] + if get_alone: + return "passed", (f"the card serves {METHOD_GET} alone, which answered with {len(exported)} part(s): a site " + "whose app keeps its own data, which takes no load and can't reset") self._rpc(METHOD_RESET) added_back = False try: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/environment.py b/packages/agentenv-protocol/src/agentenv_protocol/website/environment.py index 15e08e11c..2c8cf0f2c 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website/environment.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/environment.py @@ -53,10 +53,11 @@ create_fastmcp_app, extension, ) +from ..types import EnvironmentExtension from .clock import CLOCK_EXTENSION_URI, UNIVERSE_NOW_HEADER, ClockUnavailable, UniverseClock, clock_patch_script, iso from .errors import ApiError, NotFound -from .readiness import (SITE_READY_TIMEOUT_S, WEBSITE_ALLOW_HOSTS_PARAM, WEBSITE_EXTENSION_URI, WEBSITE_LIVE_PARAM, - WEBSITE_READY_OP, WebsiteReadiness) +from .readiness import (REAL_CLOCK, SITE_READY_TIMEOUT_S, UNIVERSE_CLOCK, WEBSITE_ALLOW_HOSTS_PARAM, WEBSITE_CLOCK_PARAM, + WEBSITE_EXTENSION_URI, WEBSITE_LIVE_PARAM, WEBSITE_READY_OP, WebsiteReadiness) from .routes import (ApiContext, ApiRoute, endpoint, error_response, forwarded_prefix, openapi, openapi_path, request_role, reserved_path, routes_of) from .session import SessionSigner, session_cookie_name @@ -75,8 +76,9 @@ class WebsiteDeclaration(NamedTuple): """What ``@website`` declares: the site's domains, its static folder (relative to the class's file), how long it - may take to be ready after its server starts, whether its HTML pages read the universe's time, and, for a live - site, that it is one and the other hosts its pages reach.""" + may take to be ready after its server starts, whether its HTML pages read the universe's time, for a live site + that it is one and the other hosts its pages reach, whether it serves agentenv's built-in site routes, and its + clock: the universe's, which a deploy can move, or real time, for an app that can't follow another.""" domains: tuple[str, ...] = () static: Optional[str] = None @@ -84,13 +86,19 @@ class WebsiteDeclaration(NamedTuple): clock_patch: bool = False live: bool = False allow_hosts: tuple[str, ...] = () + builtin_routes: bool = True + clock: str = UNIVERSE_CLOCK def website(_cls: type | None = None, *, domains: list[str] | tuple[str, ...] = (), static: str | None = None, ready_timeout_s: float = SITE_READY_TIMEOUT_S, clock_patch: bool = False, live: bool = False, - allow_hosts: list[str] | tuple[str, ...] = ()) -> Any: + allow_hosts: list[str] | tuple[str, ...] = (), builtin_routes: bool = True, + clock: str = UNIVERSE_CLOCK) -> Any: """Declare a WebsiteEnvironment's site; ``live=True`` a real one, at its ``domains``, whose pages reach the - ``allow_hosts`` too.""" + ``allow_hosts`` too. ``builtin_routes=False`` serves none of agentenv's own site routes (``/api/session``), so a + mounted app answers every path but the card's and the data plane's, as a third-party app wrapped by its site does; + ``clock="real"`` declares a site that keeps real time, whose card serves no ``clock/v1`` and whose answers carry no + ``X-Universe-Now``.""" if _cls is not None: raise TypeError("use @website(...) with parentheses") for field, hosts in (("domains", domains), ("allow_hosts", allow_hosts)): @@ -106,6 +114,12 @@ def website(_cls: type | None = None, *, domains: list[str] | tuple[str, ...] = raise ValueError("a live site serves no pages, the real site does, so it takes no static folder") if allow_hosts and not live: raise ValueError("allow_hosts are the other hosts a live site's pages reach; declare live=True") + if not isinstance(builtin_routes, bool): + raise ValueError(f"builtin_routes is True or False, not {builtin_routes!r}") + if clock not in (UNIVERSE_CLOCK, REAL_CLOCK): + raise ValueError(f"clock is {UNIVERSE_CLOCK!r} or {REAL_CLOCK!r}, not {clock!r}") + if clock_patch and clock == REAL_CLOCK: + raise ValueError("clock_patch puts the universe's time in a site's pages; a site with a real clock has none to put") if isinstance(ready_timeout_s, bool) or not isinstance(ready_timeout_s, (int, float)) or not ( 0 < ready_timeout_s < math.inf): raise ValueError(f"ready_timeout_s is a number of seconds above 0, not {ready_timeout_s!r}") @@ -114,7 +128,7 @@ def deco(cls: type) -> type: if not (isinstance(cls, type) and issubclass(cls, WebsiteEnvironment)): raise TypeError(f"@website declares a WebsiteEnvironment subclass, not {cls!r}") setattr(cls, _WEBSITE_ATTR, WebsiteDeclaration(tuple(domains), static, float(ready_timeout_s), clock_patch, live, - tuple(allow_hosts))) + tuple(allow_hosts), builtin_routes, clock)) return cls return deco @@ -223,15 +237,20 @@ def create_app(self) -> Starlette: else: asgi = Starlette(middleware=middleware) AgentEnvStarletteApplication(self._build_card(), self).add_routes_to_app(asgi) - if SESSION_PATH not in taken and not declaration.live: + if SESSION_PATH not in taken and not declaration.live and declaration.builtin_routes: asgi.router.routes.append(Route(SESSION_PATH, self._session, methods=["GET", "DELETE"])) asgi.router.routes.extend(Route(r.path, _unreserved(endpoint(getattr(self, r.fn.__name__), r, self.context)), methods=[r.method]) for r in api_routes) - asgi.router.default = _Fallback(root, type(self).app) + asgi.router.default = _Fallback(root, self.fallback_app()) self.asgi = asgi self._mounted = True return asgi + def fallback_app(self) -> Any: + """The app that answers what the site's routes and static folder don't: ``app``, unless a subclass builds its + own, as one that forwards to an app beside it does.""" + return type(self).app + def serves_mcp(self) -> bool: """Whether the site is an MCP server too: it has ``@tool`` or ``@resource`` methods, or forwards tools.""" return any(True for _ in _marked_members(self, _TOOL_ATTR)) or any( @@ -252,14 +271,23 @@ def serve(self) -> None: def extension_params(self, uri: str) -> dict: if uri == WEBSITE_EXTENSION_URI: declaration = self.declaration() + clock = {WEBSITE_CLOCK_PARAM: REAL_CLOCK} if declaration.clock == REAL_CLOCK else {} if declaration.live: return {"domains": list(declaration.domains), WEBSITE_LIVE_PARAM: True, - WEBSITE_ALLOW_HOSTS_PARAM: list(declaration.allow_hosts)} - return {"domains": list(declaration.domains), - "routes": [r.advertised() for r in self.routes()], "session_cookie": self.session_cookie, - "session_path": SESSION_PATH} + WEBSITE_ALLOW_HOSTS_PARAM: list(declaration.allow_hosts), **clock} + session = {"session_cookie": self.session_cookie, "session_path": SESSION_PATH} \ + if declaration.builtin_routes else {} + return {"domains": list(declaration.domains), "routes": [r.advertised() for r in self.routes()], + **session, **clock} return super().extension_params(uri) + def served_extensions(self, extensions: list[EnvironmentExtension]) -> list[EnvironmentExtension]: + """A site with a real clock serves no ``clock/v1``: nothing could move its time.""" + served = super().served_extensions(extensions) + if self.declaration().clock == REAL_CLOCK: + return [extension for extension in served if extension.uri != CLOCK_EXTENSION_URI] + return served + @extension(CLOCK_EXTENSION_URI, description="Sync this site to the gateway's virtual clock.") async def sync_time(self, env_get_time_url: str, timezone: Optional[str] = None) -> dict: """Follow the gateway's virtual time at ``env_get_time_url`` from now on, in ``timezone`` when the task names @@ -367,8 +395,8 @@ async def __call__(self, scope: dict, receive: Any, send: Any) -> None: if scope["type"] in ("http", "websocket"): headers = dict(scope["headers"]) host = _first(headers.get(b"x-forwarded-host")) + scope = dict(scope, **{SITE_RAW_PATH: scope.get("raw_path")}) proto = _first(headers.get(b"x-forwarded-proto")).lower() - scope = dict(scope) if host: scope["headers"] = [(k, v) for k, v in scope["headers"] if k != b"host"] + [(b"host", host.encode())] if proto in ("http", "https"): @@ -382,6 +410,11 @@ async def __call__(self, scope: dict, receive: Any, send: Any) -> None: await self.app(scope, receive, send) +#: Where the scope keeps the request's path as it reached the server, its encoding intact, before ``_Forwarded`` put a +#: forwarded prefix before it: the site's own path, as a forwarding site sends it on. +SITE_RAW_PATH = "agentenv.site_raw_path" + + def _first(value: bytes | None) -> str: return value.decode("latin-1").split(",")[0].strip() if value else "" @@ -399,13 +432,13 @@ def __init__(self, app: Any, env: WebsiteEnvironment) -> None: self.app, self.env = app, env async def __call__(self, scope: dict, receive: Any, send: Any) -> None: - if scope["type"] != "http": + if scope["type"] != "http" or self.env.declaration().clock == REAL_CLOCK: return await self.app(scope, receive, send) clock = self.env._clock() try: now: Any = await clock.read() except ClockUnavailable as e: - if not reserved_path(_route_path(scope)): + if not reserved_path(route_path(scope)): return await error_response(ApiError(str(e), code="clock_unavailable", status=503))(scope, receive, send) now = None token = clock.answering_at(now) @@ -453,7 +486,7 @@ def _with_script(html: bytes, script: bytes) -> bytes: return html[:at] + script + html[at:] -def _route_path(scope: dict) -> str: +def route_path(scope: dict) -> str: """The site's own path of a request, as routing reads it: its path without the ``root_path`` a forwarded prefix put before it (``_Forwarded``).""" root, path = scope.get("root_path", ""), scope["path"] @@ -464,7 +497,7 @@ def _unreserved(handler: Any) -> Any: """``handler``, an ``@api`` route's endpoint, answering 404 at the data plane's and card's paths, so a broad route (``/{rest:path}``) never serves a page there; the routes agentenv registers at those paths answer before it.""" async def answer(request: Request) -> Response: - if reserved_path(_route_path(request.scope)): + if reserved_path(route_path(request.scope)): return error_response(NotFound("not found")) return await handler(request) return answer @@ -483,7 +516,7 @@ async def __call__(self, scope: dict, receive: Any, send: Any) -> None: if self.app is not None: await self.app(scope, receive, send) return - path, reading = _route_path(scope), scope["method"] in ("GET", "HEAD") + path, reading = route_path(scope), scope["method"] in ("GET", "HEAD") if reserved_path(path): return await error_response(NotFound("not found"))(scope, receive, send) if reading and (file := self._file(path)) is not None: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/forwarding.py b/packages/agentenv-protocol/src/agentenv_protocol/website/forwarding.py new file mode 100644 index 000000000..c017ef58e --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/forwarding.py @@ -0,0 +1,185 @@ +"""A site whose pages an app beside it serves: ``ForwardingSite``, a ``WebsiteEnvironment`` that answers its card, its +data plane and ``website/v1`` itself and sends every other request on to ``upstream``, an app in the same container +such as a third-party web app it wraps. The request goes on as it came: its method, the site's own path with its +encoding intact and its query, its headers but the hop-by-hop ones and the site's ``sign_in_headers``, the ``Host`` +the browser named and its body, streamed. The answer comes back as the app gave it, streamed and still encoded, but +for a cookie the app ties to one of the site's own names (``Domain``), which comes back the host's alone, as agentenv +sets its own, so the cookie a sign-in sets is the app's cookie, not one beside it. Such a site usually declares +``@website(builtin_routes=False, clock="real")``, so every path but agentenv's own reaches the app and nothing +expects the app to follow the universe's clock; it signs a browser in through a ``@session_state`` of its own, which +asks the app with ``ask_app`` as its ready checks do. Its app keeps its own data, so its data plane serves +``data/get`` alone, which exports nothing: it takes no load and can't reset, which agentenv reads off its card, as it +reads any site's that serves no ``data/reset``. Its card names its ``sign_in_headers``, whose values the browser's +network log takes out. + +It forwards HTTP alone: a WebSocket is refused (close code 1011).""" +from __future__ import annotations + +from collections.abc import AsyncIterator, Iterable +from functools import cached_property +from http.cookiejar import DefaultCookiePolicy +from typing import Any, ClassVar, Optional +from urllib.parse import quote + +import httpx +from starlette.background import BackgroundTask +from starlette.requests import Request +from starlette.responses import StreamingResponse + +from ..agent_env_environment import get_data +from ..passthrough import passed_on +from .environment import SITE_RAW_PATH, WebsiteEnvironment, ready_check, route_path +from .errors import ApiError +from .readiness import WEBSITE_EXTENSION_URI, WEBSITE_SIGN_IN_HEADERS_PARAM +from .routes import error_response + +#: How long the forwarder waits to connect to the app; once connected, it waits as long as the app takes to answer. +CONNECT_TIMEOUT_S = 10.0 +#: How long the ready check waits for the app's answer, inside the ready check's own bound. +READY_ANSWER_S = 8.0 +#: How long ``ask_app`` waits for the app's answer unless told otherwise. +ASK_TIMEOUT_S = 60.0 +#: What a path's segments may hold unescaped, besides letters, digits and ``_.-~``. +PATH_SAFE = "/:@!$&'()*+,;=" +#: The close code a WebSocket gets: the forwarder relays HTTP alone. +WEBSOCKET_REFUSED = 1011 +#: The forwarder's client keeps no cookie, so each request carries the cookies its own browser sent and no others. +KEEPS_NO_COOKIES = DefaultCookiePolicy(allowed_domains=[]) + + +class ForwardingSite(WebsiteEnvironment): + """A site whose pages ``upstream`` serves: set ``upstream`` to the app's address in the container, ``ready_path`` + to a path it answers once it's up (the site is ready when that answers below 500), and ``sign_in_headers`` to + the request headers only the site's own calls may send the app, such as the sign-in header its ``@session_state`` + uses: the forwarder drops them from every browser's request, and the browser's network log takes their values out. + A site's own calls go through ``ask_app``.""" + + upstream: ClassVar[str] = "http://127.0.0.1:80" + ready_path: ClassVar[str] = "/" + sign_in_headers: ClassVar[tuple[str, ...]] = () + + def fallback_app(self) -> Any: + return self._forwarder + + def extension_params(self, uri: str) -> dict: + params = super().extension_params(uri) + if uri == WEBSITE_EXTENSION_URI and self.sign_in_headers: + return {**params, WEBSITE_SIGN_IN_HEADERS_PARAM: list(self.sign_in_headers)} + return params + + async def ask_app(self, path: str, *, method: str = "GET", headers: Optional[dict[str, str]] = None, + content: Optional[bytes] = None, timeout: float = ASK_TIMEOUT_S) -> httpx.Response: + """The app's answer to ``method`` ``path`` asked as the site: under its first name as ``Host``, with + ``headers``, such as one of its ``sign_in_headers``, which only the site's own calls send, and + ``content``; through the forwarder's client, so with no cookie but those in ``headers`` and no redirect + followed.""" + names = self._own_names() + return await self._forwarder.client.request( + method, self._forwarder.url(path), content=content, timeout=timeout, + headers={**({"host": names[0]} if names else {}), **(headers or {})}) + + @cached_property + def _forwarder(self) -> Forwarder: + return Forwarder(self.upstream, self.sign_in_headers, self._own_names()) + + def _own_names(self) -> list[str]: + """The site's exact names, the first the one its own calls ask the app under.""" + return [name for name in self.declaration().domains if not name.startswith("*.")] + + @get_data + async def exported(self) -> list: + """Nothing: the app keeps its own data, which a fresh deploy starts afresh.""" + return [] + + @ready_check + async def upstream_answers(self) -> bool: + """The app answers ``ready_path``, for the site's first name, below 500.""" + answer = await self.ask_app(self.ready_path, timeout=READY_ANSWER_S) + if answer.status_code >= 500: + raise RuntimeError(f"{self.ready_path} answered {answer.status_code}") + return True + + +class Forwarder: + """The ASGI app that sends each HTTP request on to ``upstream`` and streams its answer back, a cookie tied to one + of ``names`` the host's alone.""" + + def __init__(self, upstream: str, dropped: Iterable[str] = (), names: Iterable[str] = ()) -> None: + self.upstream = upstream.rstrip("/") + self.dropped = tuple(dropped) + self.names = frozenset(name.lower() for name in names) + self._client: Optional[httpx.AsyncClient] = None + + @property + def client(self) -> httpx.AsyncClient: + """The client requests go on with: straight to the app, whatever proxy the environment names, following no + redirect, so the browser gets each one, and with no headers or cookies of its own, so the app gets only the + request's: a cookie one browser was answered with never goes on with another's request.""" + if self._client is None: + self._client = httpx.AsyncClient(timeout=httpx.Timeout(None, connect=CONNECT_TIMEOUT_S), trust_env=False, + follow_redirects=False) + self._client.headers.clear() + self._client.cookies.jar.set_policy(KEEPS_NO_COOKIES) + return self._client + + def url(self, path: str) -> str: + """Where ``path`` is on the app: a path as a request names it, ``/`` first, or one given without it.""" + return self.upstream + (path if path.startswith("/") else f"/{path}") + + async def __call__(self, scope: dict, receive: Any, send: Any) -> None: + if scope["type"] == "websocket": + await send({"type": "websocket.close", "code": WEBSOCKET_REFUSED, "reason": "the site forwards HTTP alone"}) + return + if scope["type"] != "http": + return + request = Request(scope, receive) + headers = passed_on(_text(scope["headers"]), *self.dropped) + has_body = request.headers.get("content-length", "0") != "0" or "transfer-encoding" in request.headers + try: + answer = await self.client.send(self.client.build_request( + scope["method"], self.url(target(scope)), headers=_bytes(headers), + content=request.stream() if has_body else None), stream=True) + except httpx.HTTPError as e: + await error_response(ApiError(f"the app at {self.upstream} isn't answering: {type(e).__name__}: {e}", + code="upstream_unavailable", status=502))(scope, receive, send) + return + + async def streamed() -> AsyncIterator[bytes]: + try: + async for chunk in answer.aiter_raw(): + yield chunk + finally: + await answer.aclose() + + response = StreamingResponse(streamed(), status_code=answer.status_code, background=BackgroundTask(answer.aclose)) + response.raw_headers = _bytes((name, host_only(value, self.names) if name.lower() == "set-cookie" else value) + for name, value in passed_on(_text(answer.headers.raw))) + await response(scope, receive, send) + + +def target(scope: dict) -> str: + """The path and query a request asks the app for: the site's own path as it reached the server, still + percent-encoded, so an escaped ``/`` stays one, before any forwarded prefix was put before it.""" + raw = scope.get(SITE_RAW_PATH) + path = raw.decode("latin-1") if raw else quote(route_path(scope), safe=PATH_SAFE) + query = scope.get("query_string", b"").decode("latin-1") + return path + (f"?{query}" if query else "") + + +def host_only(set_cookie: str, names: frozenset[str]) -> str: + """``set_cookie`` without a ``Domain`` attribute naming one of ``names``, so the cookie is the host's alone.""" + name_value, *attributes = set_cookie.split(";") + return ";".join([name_value, *(attribute for attribute in attributes if not _names(attribute, names))]) + + +def _names(attribute: str, names: frozenset[str]) -> bool: + key, _, value = attribute.partition("=") + return key.strip().lower() == "domain" and value.strip().lstrip(".").lower() in names + + +def _text(headers: Iterable[tuple[bytes, bytes]]) -> list[tuple[str, str]]: + return [(name.decode("latin-1"), value.decode("latin-1")) for name, value in headers] + + +def _bytes(headers: Iterable[tuple[str, str]]) -> list[tuple[bytes, bytes]]: + return [(name.encode("latin-1"), value.encode("latin-1")) for name, value in headers] diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/readiness.py b/packages/agentenv-protocol/src/agentenv_protocol/website/readiness.py index fa2327f20..2a8ae0a97 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website/readiness.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/readiness.py @@ -23,11 +23,27 @@ #: A live site's ``website/v1`` params: that it is one, and the other hosts its pages reach. WEBSITE_LIVE_PARAM = "live" WEBSITE_ALLOW_HOSTS_PARAM = "allow_hosts" +#: A site's ``website/v1`` param naming its clock when it keeps real time, which the conformance check reads. +WEBSITE_CLOCK_PARAM = "clock" +#: A site's ``website/v1`` param naming the request headers only its own calls send, a sign-in header among them, whose +#: values the browser's network log takes out. +WEBSITE_SIGN_IN_HEADERS_PARAM = "sign_in_headers" +#: A site's clocks: the universe's, which ``clock/v1`` can tie to the gateway's, and real time, which a third-party app +#: keeps and nothing moves. +UNIVERSE_CLOCK, REAL_CLOCK = "universe", "real" READY_TIMEOUT_S = 10.0 SITE_READY_TIMEOUT_S = 300.0 DEFAULT_HEALTH_PATH = "/api/health" +def names_host(domain: str, host: str) -> bool: + """Whether ``host`` is a name the site's ``domain`` names: itself, or for ``*.example.com`` one label more.""" + if domain.startswith("*."): + head, dot, rest = host.partition(".") + return bool(head) and bool(dot) and rest == domain[2:] + return host == domain + + class WebsiteReadiness(BaseModel): """What a site says of its readiness, ``website/v1``'s ``ready``: whether it is ``ready``, the ``problems`` that keep it from being so, each a line a person reads, such as a ``@ready_check`` method's and what it returned, and, diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/environment.py b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/environment.py index ffcf32496..94087ea5f 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website_browser/environment.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website_browser/environment.py @@ -31,6 +31,7 @@ from ..tool_context import DEFAULT_ROLE, ROLE_HEADER, normalize_role from ..transfers import WriteNamespaceGrant from ..types import METHOD_ADD, RPC_PATH, EnvironmentExtension, IntakeDeclaration, IntakeFormat, intake_extension +from ..website.readiness import names_host from . import data from .contexts import CONTEXTS, TASK, Contexts, label_problem from .control import AGENT, DOWNLOADS, EXIT, LIVE_START_TAB_TIMEOUT_S, LOADED, STOP, UPLOADS, BrowserControl, _now @@ -898,15 +899,7 @@ def _read_sites() -> list[SiteNames]: def _site_of(url: str, sites: list[SiteNames]) -> Optional[str]: """The name of the env's site among ``sites`` that serves ``url``'s host, None for any other.""" host = (urlsplit(url).hostname or "").rstrip(".") - return next((site.name for site in sites if any(_matches(domain.lower(), host) for domain in site.domains)), None) - - -def _matches(pattern: str, host: str) -> bool: - """Whether ``host`` is the name ``pattern`` names: itself, or for ``*.example.com`` one label more.""" - if pattern.startswith("*."): - head, dot, rest = host.partition(".") - return bool(head) and bool(dot) and rest == pattern[2:] - return host == pattern + return next((site.name for site in sites if any(names_host(domain.lower(), host) for domain in site.domains)), None) def _refuse_failed(tabs: list[dict]) -> None: diff --git a/packages/agentenv-protocol/tests/test_website_conformance.py b/packages/agentenv-protocol/tests/test_website_conformance.py index 6d2ce401f..ec8f2e2fd 100644 --- a/packages/agentenv-protocol/tests/test_website_conformance.py +++ b/packages/agentenv-protocol/tests/test_website_conformance.py @@ -78,10 +78,12 @@ def _handwritten(*, leak_host: bool = False, domain_cookie: bool = False, dated_ leak_redirect: bool = False, stamp_rpc: bool = True, reset_keeps: bool = False, get_fails_after_reset: bool = False, rpc_reply: tuple[int, dict] | None = None, ready_answer: object = None, ready_status: int = 200, rpc_path: str | None = None, - ready_endpoint: str = "/ready", card_url: str | None = None) -> Starlette: + ready_endpoint: str = "/ready", card_url: str | None = None, + foreign_redirect: str | None = None) -> Starlette: """A server written without the SDK: the contract by hand, optionally broken. Its state is its ``state.data``; with ``ready_answer`` its card advertises ``website/v1``'s ``ready``, which answers it with ``ready_status``. With ``rpc_path`` its card's - ``url`` names its data plane there, not at ``/agentenv``; ``card_url`` puts what it says in the card's ``url`` alone.""" + ``url`` names its data plane there, not at ``/agentenv``; ``card_url`` puts what it says in the card's ``url`` alone. + With ``foreign_redirect``, ``/`` under an ``.invalid`` Host redirects there, as an app with a fixed base URL does.""" state = {"items": ["a"], "url": None, "at": None, "reset": False} def now() -> datetime.datetime: @@ -92,6 +94,8 @@ def stamp(response: Response) -> Response: return response async def page(request: Request) -> Response: + if foreign_redirect and request.headers["host"].endswith(".invalid"): + return stamp(Response(status_code=302, headers={"Location": foreign_redirect})) host = request.headers["host"] if leak_host else request.headers.get("x-forwarded-host", request.headers["host"]) response = stamp(HTMLResponse(f'x')) response.set_cookie("s", "1", domain="example.com" if domain_cookie else None, expires=3600 if dated_cookie else None) @@ -176,6 +180,19 @@ def test_a_handwritten_server_meeting_the_contract_passes() -> None: assert statuses(results) == {**dict.fromkeys(CHECKS, "passed"), "mcp": "skipped"}, results +@pytest.mark.parametrize(("location", "status", "detail"), [ + ("http://h.example/", "passed", "redirected to the site's own name h.example"), + ("https://h.example:8443/admin", "passed", "redirected to the site's own name h.example"), + ("http://elsewhere.example/", "failed", "answered 302 without a content type, redirecting to elsewhere.example, " + "not one of the site's names"), + ("/", "failed", "answered 302 without a content type"), +]) +def test_any_host_passes_a_redirect_to_the_sites_own_name_alone(location: str, status: str, detail: str) -> None: + results = {r.name: r for r in conformance.check("http://testserver", + client=TestClient(_handwritten(foreign_redirect=location)))} + assert (results["any_host"].status, results["any_host"].detail.endswith(detail)) == (status, True), results["any_host"] + + @pytest.mark.parametrize(("answer", "status", "detail"), [ ({"ready": True, "problems": [], "since": "boot"}, "passed", "website/v1 ready says it is ready"), ({"ready": False, "problems": ["seeded", "indexed"]}, "failed", "website/v1 ready says it is not ready: seeded, indexed"), diff --git a/packages/agentenv-protocol/tests/test_website_forwarding.py b/packages/agentenv-protocol/tests/test_website_forwarding.py new file mode 100644 index 000000000..5df1c233f --- /dev/null +++ b/packages/agentenv-protocol/tests/test_website_forwarding.py @@ -0,0 +1,394 @@ +"""A site that wraps an app beside it: ForwardingSite sends every request agentenv's own routes don't answer to the app +as it came, its path's encoding, its query, its Host and its body, without the hop-by-hop headers or the ones the site +drops, and streams the app's answer back as the app gave it, still encoded, its redirects not followed; the card, the +data plane and website/v1 stay the site's; it is ready once the app answers its ready path; and the switches a +third-party app needs: no built-in site routes, and a real clock, which serves no clock/v1, stamps no X-Universe-Now +and has the conformance check skip its clock checks.""" + +from __future__ import annotations + +import asyncio +import gzip +import threading +from collections.abc import Iterator +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +import pytest +from starlette.testclient import TestClient +from starlette.websockets import WebSocketDisconnect + +from agentenv_protocol.client import find_extension_method +from agentenv_protocol.website import ( + CLOCK_EXTENSION_URI, + UNIVERSE_NOW_HEADER, + WEBSITE_EXTENSION_URI, + WEBSITE_SIGN_IN_HEADERS_PARAM, + ForwardingSite, + WebsiteEnvironment, + conformance, + session_state, + website, +) +from agentenv_protocol.website.forwarding import WEBSOCKET_REFUSED, Forwarder + +HOST = "shop.localhost" +PAGE = b"Shophome" +ZIPPED = gzip.compress(b'{"items": [1, 2, 3]}') + + +class App: + """The app a site wraps, on loopback: each request it got, by method, raw path, headers and body, and its answers + by path.""" + + def __init__(self) -> None: + self.seen: list[dict] = [] + self.status = 200 + app = self + + class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def _answer(self) -> None: + length = int(self.headers.get("Content-Length") or 0) + if self.headers.get("Transfer-Encoding", "").lower() == "chunked": + body = b"" + while (size := int(self.rfile.readline().split(b";")[0], 16)): + body += self.rfile.read(size) + self.rfile.readline() + self.rfile.readline() + else: + body = self.rfile.read(length) if length else b"" + app.seen.append({"method": self.command, "path": self.path, + "headers": [(k.lower(), v) for k, v in self.headers.items()], "body": body}) + path = self.path.partition("?")[0] + if path == "/redirect": + self._send(302, b"", [("Location", f"http://{HOST}/elsewhere")]) + elif path == "/zipped": + self._send(200, ZIPPED, [("Content-Type", "application/json"), ("Content-Encoding", "gzip")]) + elif path == "/cookies": + self._send(200, b"ok", [("Content-Type", "text/plain"), ("Set-Cookie", "a=1; Path=/"), + ("Set-Cookie", "b=2; Path=/admin; HttpOnly"), + ("Set-Cookie", f"c=3; Domain={HOST}; Path=/"), + ("Set-Cookie", f"d=4; domain=.{HOST.upper()}"), + ("Set-Cookie", "e=5; Domain=other.example; Path=/")]) + elif path == "/streamed": + self.send_response(200) + self.send_header("Content-Type", "text/plain") + self.send_header("Transfer-Encoding", "chunked") + self.end_headers() + for part in (b"one,", b"two,", b"three"): + self.wfile.write(b"%x\r\n%s\r\n" % (len(part), part)) + self.wfile.write(b"0\r\n\r\n") + else: + self._send(app.status, PAGE, [("Content-Type", "text/html; charset=utf-8"), ("X-App", "magento")]) + + def _send(self, status: int, body: bytes, headers: list[tuple[str, str]]) -> None: + self.send_response(status) + for name, value in headers: + self.send_header(name, value) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + if self.command != "HEAD": + self.wfile.write(body) + + do_GET = do_POST = do_PUT = do_PATCH = do_DELETE = do_HEAD = do_OPTIONS = _answer + + def log_message(self, *args) -> None: + pass + + self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.url = f"http://127.0.0.1:{self.server.server_address[1]}" + threading.Thread(target=self.server.serve_forever, daemon=True).start() + + def close(self) -> None: + self.server.shutdown() + self.server.server_close() + + def last(self) -> dict: + return self.seen[-1] + + +@pytest.fixture +def app() -> Iterator[App]: + served = App() + yield served + served.close() + + +def _site(upstream: str, **declared) -> type[ForwardingSite]: + @website(domains=[HOST], builtin_routes=False, clock="real", **declared) + class Shop(ForwardingSite): + sign_in_headers = ("X-Auto-Login",) + ready_path = "/admin" + + @session_state + def signed_in(self, user): + return {"cookies": [{"name": "admin", "value": "s1", "path": "/admin", "max_age": 3600}], + "local_storage": []} + + Shop.upstream = upstream + return Shop + + +@pytest.fixture +def client(app: App) -> Iterator[TestClient]: + with TestClient(_site(app.url)().create_app(), headers={"X-Forwarded-Host": HOST}) as forwarded: + yield forwarded + + +def test_a_request_goes_to_the_app_as_it_came_without_hop_by_hop_or_sign_in_headers(app, client): + answer = client.get("/admin/a%2Fb/c?x=1&y=%20z", headers={ + "X-Custom": "kept", "Cookie": "admin=s1", "Connection": "keep-alive, X-Per-Hop", "X-Per-Hop": "gone", + "Keep-Alive": "timeout=5", "x-auto-login": "admin:admin1234"}) + + seen = app.last() + headers = dict(seen["headers"]) + assert (answer.status_code, answer.content, answer.headers["x-app"]) == (200, PAGE, "magento") + assert (seen["method"], seen["path"]) == ("GET", "/admin/a%2Fb/c?x=1&y=%20z") + assert (headers["host"], headers["x-custom"], headers["cookie"]) == (HOST, "kept", "admin=s1") + assert not {"x-per-hop", "keep-alive", "x-auto-login"} & set(headers), headers + + +async def _forwarded(forwarder: Forwarder, path: str, headers: list[tuple[bytes, bytes]]) -> list[dict]: + """The ASGI messages ``forwarder`` sends for a GET of ``path`` with ``headers`` alone, as a browser's request.""" + sent: list[dict] = [] + requested, answered = asyncio.Event(), asyncio.Event() + + async def receive() -> dict: + if not requested.is_set(): + requested.set() + return {"type": "http.request", "body": b"", "more_body": False} + await answered.wait() + return {"type": "http.disconnect"} + + async def send(message: dict) -> None: + sent.append(message) + if message["type"] == "http.response.body" and not message.get("more_body"): + answered.set() + + await forwarder({"type": "http", "method": "GET", "path": path, "raw_path": path.encode(), "root_path": "", + "query_string": b"", "headers": headers}, receive, send) + return sent + + +def _each_forwarded(forwarder: Forwarder, *requests: tuple[str, list[tuple[bytes, bytes]]]) -> list[list[dict]]: + """What ``forwarder`` sends for each of ``requests`` in turn, on one event loop, its client closed after.""" + async def forward() -> list[list[dict]]: + try: + return [await _forwarded(forwarder, path, headers) for path, headers in requests] + finally: + await forwarder.client.aclose() + + return asyncio.run(forward()) + + +def test_the_app_gets_only_the_headers_the_request_had_and_none_of_the_forwarders_own(app): + """A request with no Accept-Encoding, User-Agent or Accept reaches the app with none, so the app never encodes + an answer for a client that didn't ask.""" + [sent] = _each_forwarded(Forwarder(app.url), ("/admin", [(b"host", HOST.encode())])) + + assert sent[0]["status"] == 200 + assert app.last()["headers"] == [("host", HOST)], app.last()["headers"] + + +def test_a_cookie_one_browser_was_answered_with_never_goes_on_with_anothers_request(app): + """The app sets cookies on one browser's answer, host-only ones among them; another browser's request, with no + cookie of its own, reaches the app with none, and one with its own cookie with that alone.""" + host = [(b"host", HOST.encode())] + answered, bare, own = _each_forwarded(Forwarder(app.url, names=[HOST]), ("/cookies", host), ("/admin", host), + ("/admin", [*host, (b"cookie", b"admin=other")])) + bare_seen = app.seen[-2]["headers"] + + set_cookies = [value for name, value in answered[0]["headers"] if name.lower() == b"set-cookie"] + assert b"a=1; Path=/" in set_cookies and (bare[0]["status"], own[0]["status"]) == (200, 200) + assert bare_seen == [("host", HOST)], bare_seen + assert app.last()["headers"] == [("host", HOST), ("cookie", "admin=other")], app.last()["headers"] + + +def test_ask_app_asks_as_the_site_under_its_name_with_a_header_only_its_own_calls_send(app): + """The site's own call reaches the app under the site's name, with the headers it gives, a dropped one among them, + and its method and body; a cookie the app answered one call with goes on with no later one.""" + site = _site(app.url)() + + async def ask() -> tuple: + try: + return (await site.ask_app("/cookies", headers={"X-Auto-Login": "admin:test"}), + await site.ask_app("admin/form", method="POST", content=b"user=amy")) + finally: + await site.fallback_app().client.aclose() + + signed, posted = asyncio.run(ask()) + first, second = (dict(seen["headers"]) for seen in app.seen[-2:]) + + assert (signed.status_code, first["host"], first["x-auto-login"]) == (200, HOST, "admin:test") + assert signed.headers.get_list("set-cookie")[0] == "a=1; Path=/" + assert (posted.status_code, app.seen[-1]["method"], app.seen[-1]["path"], app.seen[-1]["body"]) == ( + 200, "POST", "/admin/form", b"user=amy") + assert "cookie" not in second and second["host"] == HOST, second + + +def test_a_body_goes_to_the_app_whole_sent_at_once_or_chunked(app, client): + client.post("/admin/form", content=b"user=amy&next=%2F", headers={"Content-Type": "application/x-www-form-urlencoded"}) + sent = app.last() + client.put("/admin/upload", content=iter([b"part one, ", b"part two"])) + chunked = app.last() + + assert (sent["method"], sent["body"]) == ("POST", b"user=amy&next=%2F") + assert dict(sent["headers"])["content-type"] == "application/x-www-form-urlencoded" + assert (chunked["method"], chunked["body"]) == ("PUT", b"part one, part two") + + +def test_the_answer_comes_back_as_the_app_gave_it_but_a_cookie_tied_to_the_sites_name_is_the_hosts_alone(app, client): + """Redirects unfollowed, bodies still encoded, chunks streamed, cookies as the app set them, but for one whose + Domain names the site, which is the host's alone, as agentenv sets a sign-in's cookies.""" + moved = client.get("/redirect", follow_redirects=False) + cookies = client.get("/cookies") + with client.stream("GET", "/zipped") as zipped: + raw = b"".join(zipped.iter_raw()) + streamed = client.get("/streamed") + + assert (moved.status_code, moved.headers["location"]) == (302, f"http://{HOST}/elsewhere") + assert cookies.headers.get_list("set-cookie") == ["a=1; Path=/", "b=2; Path=/admin; HttpOnly", "c=3; Path=/", + "d=4", "e=5; Domain=other.example; Path=/"] + assert (zipped.headers["content-encoding"], raw) == ("gzip", ZIPPED) + assert streamed.content == b"one,two,three" + + +def test_every_method_reaches_the_app_head_and_options_included(app, client): + head = client.head("/admin") + options = client.options("/admin") + deleted = client.delete("/admin/item/7") + + assert [seen["method"] for seen in app.seen] == ["HEAD", "OPTIONS", "DELETE"] + assert (head.status_code, head.content, options.status_code, deleted.status_code) == (200, b"", 200, 200) + + +def test_the_card_names_the_sites_sign_in_headers_and_a_site_with_none_names_none(client): + card = client.get("/.well-known/agent-env.json").json() + params = next(ext for ext in card["capabilities"]["extensions"] if ext["uri"] == WEBSITE_EXTENSION_URI)["params"] + + @website(domains=[HOST], builtin_routes=False, clock="real") + class Bare(ForwardingSite): + pass + + bare = Bare().extension_params(WEBSITE_EXTENSION_URI) + + assert params[WEBSITE_SIGN_IN_HEADERS_PARAM] == ["X-Auto-Login"] + assert WEBSITE_SIGN_IN_HEADERS_PARAM not in bare + + +def test_the_card_and_a_data_plane_of_data_get_alone_stay_the_sites_and_api_session_reaches_the_app(app, client): + card = client.get("/.well-known/agent-env.json").json() + exported = client.post("/agentenv", json={"jsonrpc": "2.0", "id": 1, "method": "data/get", "params": {}}).json() + reset = client.post("/agentenv", json={"jsonrpc": "2.0", "id": 2, "method": "data/reset", "params": {}}).json() + signed_in = client.get("/api/session") + + site = next(ext for ext in card["capabilities"]["extensions"] if ext["uri"] == WEBSITE_EXTENSION_URI) + assert card["capabilities"]["operations"] == ["data/get"] and exported["result"]["parts"] == [] + assert reset["error"]["code"] == -32601 + assert [seen["path"] for seen in app.seen] == ["/api/session"] and signed_in.content == PAGE + assert "session_path" not in site["params"] and "session_cookie" not in site["params"] + + +def test_its_session_state_signs_a_browser_in_with_the_cookies_it_chose(client): + card = client.get("/.well-known/agent-env.json").json() + op = find_extension_method(card, WEBSITE_EXTENSION_URI, "session") + answer = client.request(op.get("method", "POST"), op["endpoint"], json={"user": None}).json() + + assert answer == {"cookies": [{"name": "admin", "value": "s1", "path": "/admin", "max_age": 3600}], + "local_storage": []} + + +def test_an_app_not_answering_is_a_502_and_the_site_not_ready_until_its_ready_path_answers(app): + gone = App() + url = gone.url + gone.close() + with TestClient(_site(url)().create_app(), headers={"X-Forwarded-Host": HOST}) as down: + answer = down.get("/admin") + ready = down.get(_ready_endpoint(down)).json() + + app.status = 503 + with TestClient(_site(app.url)().create_app()) as failing: + failing_ready = failing.get(_ready_endpoint(failing)).json() + app.status = 200 + with TestClient(_site(app.url)().create_app()) as up: + up_ready = up.get(_ready_endpoint(up)).json() + + assert (answer.status_code, answer.json()["code"]) == (502, "upstream_unavailable") and url in answer.json()["error"] + assert ready["ready"] is False and "upstream_answers" in ready["problems"][0], ready + assert failing_ready["ready"] is False and "answered 503" in failing_ready["problems"][0], failing_ready + assert up_ready["ready"] is True + assert (app.last()["path"], dict(app.last()["headers"])["host"]) == ("/admin", HOST) + + +def _ready_endpoint(client: TestClient) -> str: + return find_extension_method(client.get("/.well-known/agent-env.json").json(), WEBSITE_EXTENSION_URI, + "ready")["endpoint"] + + +def test_a_forwarded_prefix_leaves_the_app_the_sites_own_path(app, client): + client.get("/admin/x%2Fy", headers={"X-Forwarded-Prefix": "/website/shop"}) + + assert app.last()["path"] == "/admin/x%2Fy" + + +def test_a_websocket_is_refused(client): + with pytest.raises(WebSocketDisconnect) as refused: + with client.websocket_connect("/socket"): + pass + + assert refused.value.code == WEBSOCKET_REFUSED + + +def test_a_real_clock_serves_no_clock_v1_stamps_no_universe_time_and_declares_itself(app, client): + card = client.get("/.well-known/agent-env.json").json() + page = client.get("/admin") + + uris = [ext["uri"] for ext in card["capabilities"]["extensions"]] + site = next(ext for ext in card["capabilities"]["extensions"] if ext["uri"] == WEBSITE_EXTENSION_URI) + assert CLOCK_EXTENSION_URI not in uris and site["params"]["clock"] == "real" + assert UNIVERSE_NOW_HEADER not in page.headers + + +def test_the_conformance_check_skips_the_clock_checks_of_a_site_with_a_real_clock_and_passes_it(app, client): + results = {result.name: result for result in conformance.check("http://testserver", client=client)} + + assert (results["clock"].status, results["universe_now"].status) == ("skipped", "skipped") + assert "real clock" in results["clock"].detail + assert all(result.status in ("passed", "skipped") for result in results.values()), { + name: (result.status, result.detail) for name, result in results.items()} + + +def test_a_plain_site_keeps_its_built_in_routes_and_the_universe_clock_by_default(): + @website(domains=["notes.test"]) + class Notes(WebsiteEnvironment): + pass + + with TestClient(Notes().create_app()) as client: + card = client.get("/.well-known/agent-env.json").json() + session = client.get("/api/session") + page = client.get("/missing") + + uris = [ext["uri"] for ext in card["capabilities"]["extensions"]] + site = next(ext for ext in card["capabilities"]["extensions"] if ext["uri"] == WEBSITE_EXTENSION_URI) + assert CLOCK_EXTENSION_URI in uris and "clock" not in site["params"] and site["params"]["session_path"] + assert session.status_code == 200 and UNIVERSE_NOW_HEADER in page.headers + + +def test_without_built_in_routes_api_session_is_the_apps_path(): + @website(domains=["notes.test"], builtin_routes=False) + class Bare(WebsiteEnvironment): + pass + + with TestClient(Bare().create_app()) as client: + assert client.get("/api/session").status_code == 404 + + +@pytest.mark.parametrize("declared, problem", [ + ({"clock": "virtual"}, "clock is 'universe' or 'real'"), + ({"clock": "real", "clock_patch": True}, "a site with a real clock"), + ({"builtin_routes": "no"}, "builtin_routes is True or False"), +], ids=["unknown-clock", "patch-on-real", "routes-not-bool"]) +def test_website_refuses_a_clock_or_routes_switch_it_cant_honor(declared, problem): + with pytest.raises(ValueError, match=problem): + website(domains=["notes.test"], **declared) diff --git a/src/agent_env/env/envs/computer/utils/params.py b/src/agent_env/env/envs/computer/utils/params.py index eb6442546..3bed3c3b7 100644 --- a/src/agent_env/env/envs/computer/utils/params.py +++ b/src/agent_env/env/envs/computer/utils/params.py @@ -22,14 +22,16 @@ class Site(BaseModel): - """A website the computer reaches by its names: its route on the env gateway, as this process reaches it, and the - names its card's website/v1 declares.""" + """A website the computer reaches by its names: its route on the env gateway, as this process reaches it, the + names its card's website/v1 declares, and the request headers it names as its sign-in headers, whose values the + browser's network log takes out.""" model_config = ConfigDict(extra="forbid") name: str url: str domains: list[str] = Field(default_factory=list) + sign_in_headers: list[str] = Field(default_factory=list) class ComputerParams(BaseModel): diff --git a/src/agent_env/env/envs/computer/utils/site_proxy.py b/src/agent_env/env/envs/computer/utils/site_proxy.py index d8bc69190..6c37054ee 100644 --- a/src/agent_env/env/envs/computer/utils/site_proxy.py +++ b/src/agent_env/env/envs/computer/utils/site_proxy.py @@ -94,6 +94,11 @@ r"^\s*(text/|application/(json|x-www-form-urlencoded|xml|javascript|ecmascript)\b|[^;]*\+(json|xml)\b)", re.I) #: What the network log keeps of a header a site names as its sign-in header. REMOVED = "" +#: A site's fields in the config ``install`` reads that the config it serves from keeps as given, which the session +#: user reads too; its certificate and key ``install`` writes to files, and its upstream headers to one root alone +#: reads (``PRIVATE_SITE_FIELDS``). Every field a site's config has is one or the other. +SERVED_SITE_FIELDS = ("name", "upstream", "domains", "sign_in_headers") +PRIVATE_SITE_FIELDS = ("cert_pem", "key_pem", "upstream_headers") #: How a network log line begins, its number, its kind and its entry's id, so a proxy started again finds its last #: number and the entries left open without reading every body. NUMBERED = re.compile(rb'^\{"seq":(\d+),"at":"[^"]*","(open|end)":\{"id":"([^"]*)"') @@ -326,9 +331,8 @@ def load(cls, path: str) -> Config: headers = json.loads(Path(data["upstream_headers"]).read_text()) except PermissionError: headers = {} - sites = [Site(name=s["name"], upstream=s["upstream"], domains=list(s["domains"]), cert=s.get("cert"), - key=s.get("key"), upstream_headers=dict(headers.get(s["name"]) or {}), - sign_in_headers=list(s.get("sign_in_headers") or [])) for s in data["sites"]] + sites = [Site(**{field: s[field] for field in SERVED_SITE_FIELDS if field in s}, cert=s.get("cert"), + key=s.get("key"), upstream_headers=dict(headers.get(s["name"]) or {})) for s in data["sites"]] return cls(user=data.get("user"), sites=sites, serve_user=data.get("serve_user"), listen=data.get("listen", LOOPBACK), https_port=data.get("https_port", 443), http_port=data.get("http_port", 80), @@ -1403,7 +1407,7 @@ def install(config_path: str, paths: Paths = Paths()) -> dict: Path(ca).write_text(given["ca_pem"]) sites = [] for site in given["sites"]: - served = {"name": site["name"], "upstream": site["upstream"], "domains": site["domains"]} + served = {field: site[field] for field in SERVED_SITE_FIELDS if field in site} if site.get("cert_pem"): cert, key = folder / f"{site['name']}.crt", folder / f"{site['name']}.key" cert.write_text(site["cert_pem"]) diff --git a/src/agent_env/env/envs/computer/utils/sites.py b/src/agent_env/env/envs/computer/utils/sites.py index 931d902d8..8ac1d1389 100644 --- a/src/agent_env/env/envs/computer/utils/sites.py +++ b/src/agent_env/env/envs/computer/utils/sites.py @@ -35,7 +35,7 @@ import httpx from agentenv_protocol import client as protocol_client from agentenv_protocol.website import WEBSITE_EXTENSION_URI as WEBSITE -from agentenv_protocol.website import WEBSITE_LIVE_PARAM +from agentenv_protocol.website import WEBSITE_LIVE_PARAM, WEBSITE_SIGN_IN_HEADERS_PARAM from agentenv_protocol.website_browser import NETWORK_LOG_ENV, SITE_PROXY_ENV from cryptography import x509 from cryptography.hazmat.primitives import hashes, serialization @@ -313,12 +313,15 @@ def browser_environment(display: str) -> dict[str, str]: def proxy_config(table: SiteTable, issued: Issued, *, user: Optional[str], os_type: str, forward_port: bool, - upstream_headers: Optional[dict[str, dict[str, str]]] = None, direct: Iterable[str] = ()) -> dict: + upstream_headers: Optional[dict[str, dict[str, str]]] = None, direct: Iterable[str] = (), + sign_in_headers: Optional[dict[str, list[str]]] = None) -> dict: """What ``site_proxy.py install`` reads: the session user, the CA, the trust files on an ``os_type`` computer and the variables naming them, which its profile script sets as the computer's boot environment does, each route with - its certificate and key, and the headers its gateway's sandbox provider asks requests to it to carry - (``upstream_headers``, by site), which the proxy sends upstream alone, and with ``forward_port``, for a browser - alone, the forward port, the live sites' hosts it takes straight to them (``direct``) and its network log.""" + its certificate and key, the headers its gateway's sandbox provider asks requests to it to carry + (``upstream_headers``, by site), which the proxy sends upstream alone, and those its site names as its sign-in + headers (``sign_in_headers``, by site), whose values its network log takes out, and with ``forward_port``, for a + browser alone, the forward port, the live sites' hosts it takes straight to them (``direct``) and its network + log.""" sites = [] for route in table.routes: site = {"name": route.site, "upstream": route.url, "domains": list(route.domains)} @@ -326,6 +329,8 @@ def proxy_config(table: SiteTable, issued: Issued, *, user: Optional[str], os_ty site["cert_pem"], site["key_pem"] = issued.leaves[route.site] if (upstream_headers or {}).get(route.site): site["upstream_headers"] = dict(upstream_headers[route.site]) + if (sign_in_headers or {}).get(route.site): + site["sign_in_headers"] = list(sign_in_headers[route.site]) sites.append(site) return {"user": user, "ca_pem": issued.ca_pem, "sites": sites, "log": str(LOG), "trust_files": trust_files(os_type), "trust_environment": dict(TRUST_ENVIRONMENTS[os_type]), @@ -456,9 +461,10 @@ async def route(sandbox: Sandbox, sites: list[Site], *, user: Optional[str], net channel = RootChannel(sandbox) await channel.private_folder(FOLDER) await channel.write(PROXY, _PROXY_SOURCE.read_text()) - await channel.write_secret(CONFIG, json.dumps(proxy_config(table, issued, user=user, os_type=os_type, - forward_port=display == ALONE_DISPLAY, - upstream_headers=upstream_headers, direct=direct))) + await channel.write_secret(CONFIG, json.dumps(proxy_config( + table, issued, user=user, os_type=os_type, forward_port=display == ALONE_DISPLAY, + upstream_headers=upstream_headers, direct=direct, + sign_in_headers={site.name: site.sign_in_headers for site in sites if site.sign_in_headers}))) await channel.run(["chmod", "0600", str(CONFIG)]) output = await channel.run(["python3", str(PROXY), "install", str(CONFIG)]) report = json.loads(output.strip().splitlines()[-1]) @@ -624,7 +630,11 @@ def _stored_domains(record: DeployedEnv) -> dict[str, list[str]]: def _found(name: str, url: str, params: dict, stored: Optional[list[str]] = None) -> dict: - return {"name": name, "url": url, "domains": list(stored or params.get("domains") or [])} + """A site found, with its card's sign-in headers when it names any.""" + sign_in = params.get(WEBSITE_SIGN_IN_HEADERS_PARAM) + named = [header for header in sign_in if isinstance(header, str) and header] if isinstance(sign_in, list) else [] + return {"name": name, "url": url, "domains": list(stored or params.get("domains") or []), + **({"sign_in_headers": named} if named else {})} def _joined(address: str, url: str) -> str: diff --git a/src/agent_env/task_step/task_steps/browser/sign_in.py b/src/agent_env/task_step/task_steps/browser/sign_in.py index efe020002..5b5177fe3 100644 --- a/src/agent_env/task_step/task_steps/browser/sign_in.py +++ b/src/agent_env/task_step/task_steps/browser/sign_in.py @@ -150,8 +150,9 @@ def _own_names(website: WebsiteEnv, child: protocol_v1.ChildEnv) -> list[str]: def scoped(cookies: list[dict], names: list[str]) -> list[dict]: """Each of ``cookies``, which ``session`` leaves unplaced, set for each of the site's ``names``: an exact name by - its URL, so the cookie is the name's alone, over HTTPS, a ``.localhost`` one over HTTP; a ``*.`` name as a domain - cookie on the name it ends in.""" + its URL, so the cookie is the name's alone, over HTTPS, a ``.localhost`` one over HTTP, with its path given too, + since a URL alone puts a cookie on ``/admin`` at ``/``, its default path, beside the site's own; a ``*.`` name as + a domain cookie on the name it ends in.""" placed = [] for cookie in cookies: unplaced = {key: value for key, value in cookie.items() if key not in ("url", "domain")} @@ -160,8 +161,8 @@ def scoped(cookies: list[dict], names: list[str]) -> list[dict]: placed.append({**unplaced, "domain": f".{name[2:]}", "path": unplaced.get("path") or "/"}) else: scheme = "http" if name == "localhost" or name.endswith(".localhost") else "https" - placed.append({**{key: value for key, value in unplaced.items() if key != "path"}, - "url": f"{scheme}://{name}{unplaced.get('path') or '/'}"}) + path = unplaced.get("path") or "/" + placed.append({**unplaced, "url": f"{scheme}://{name}{path}", "path": path}) return placed diff --git a/tst/data/forwarding_website/Dockerfile b/tst/data/forwarding_website/Dockerfile new file mode 100644 index 000000000..7080524ee --- /dev/null +++ b/tst/data/forwarding_website/Dockerfile @@ -0,0 +1,12 @@ +FROM mirror.gcr.io/library/python:3.12-slim@sha256:423ed6ab25b1921a477529254bfeeabf5855151dc2c3141699a1bfc852199fbf +RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/* +WORKDIR /app +RUN pip install --no-cache-dir starlette uvicorn pydantic httpx +COPY agentenv_protocol /app/agentenv_protocol +COPY server.py /app/server.py +ENV PYTHONPATH=/app +ENV PORT=8000 +EXPOSE 8000 +RUN useradd --create-home --uid 10001 appuser +USER appuser +CMD ["python", "server.py"] diff --git a/tst/data/forwarding_website/server.py b/tst/data/forwarding_website/server.py new file mode 100644 index 000000000..5b297b85d --- /dev/null +++ b/tst/data/forwarding_website/server.py @@ -0,0 +1,78 @@ +"""A third-party app wrapped as a website env on ForwardingSite, named shop.localhost, for integration-testing a site +whose pages an app beside it serves: the app, on loopback, signs in only a request carrying X-Auto-Login, which the +site's own session request sends and the forwarder drops from every other. Like an app with a fixed base URL, it ties +its session cookie to its name (Domain), giving an anonymous visit one too, and sets it again on every answer. Each of +its /admin pages has as its title, as JSON, who the request's cookie signs in, the Host and path the app was asked and +whether it saw X-Auto-Login.""" +import json +import secrets +import threading +from http.cookies import SimpleCookie +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +from agentenv_protocol.website import ForwardingSite, session_state, website + +APP = ("127.0.0.1", 8080) +NAME = "shop.localhost" +AUTO_LOGIN = "X-Auto-Login" +SESSIONS: dict[str, str | None] = {} + + +def session_cookie(sid: str) -> str: + return f"sid={sid}; Domain={NAME}; Path=/admin; Max-Age=3600; HttpOnly" + + +def first_sid(header: str) -> str | None: + """The first sid a Cookie header sends, as PHP reads it: a browser holding two sends the longer path's first.""" + return next((value for name, _, value in (pair.strip().partition("=") for pair in header.split(";")) + if name == "sid"), None) + + +class App(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_GET(self): + if self.headers.get(AUTO_LOGIN): + sid = secrets.token_hex(8) + SESSIONS[sid] = self.headers[AUTO_LOGIN] + self._send(302, b"", [("Location", "/admin/home"), ("Set-Cookie", session_cookie(sid))]) + return + sent = first_sid(self.headers.get("Cookie", "")) + sid = sent if sent in SESSIONS else secrets.token_hex(8) + SESSIONS.setdefault(sid, None) + seen = {"user": SESSIONS[sid], "host": self.headers.get("Host"), "path": self.path, + "auto_login": AUTO_LOGIN.lower() in (name.lower() for name in self.headers)} + page = f"{json.dumps(seen)}

admin

".encode() + self._send(200, page, [("Content-Type", "text/html; charset=utf-8"), ("Cache-Control", "no-store"), + ("Set-Cookie", session_cookie(sid))]) + + def _send(self, status, body, headers): + self.send_response(status) + for name, value in headers: + self.send_header(name, value) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + pass + + +@website(domains=[NAME], builtin_routes=False, clock="real") +class Shop(ForwardingSite): + upstream = f"http://{APP[0]}:{APP[1]}" + ready_path = "/admin/home" + sign_in_headers = (AUTO_LOGIN,) + + @session_state + async def signed_in(self, user): + answer = await self.ask_app("/admin/home", headers={AUTO_LOGIN: user or "admin"}) + sid = SimpleCookie(answer.headers["set-cookie"])["sid"] + return {"cookies": [{"name": "sid", "value": sid.value, "path": "/admin", "max_age": 3600, "httpOnly": True}], + "local_storage": []} + + +if __name__ == "__main__": + server = ThreadingHTTPServer(APP, App) + threading.Thread(target=server.serve_forever, daemon=True).start() + Shop().serve() diff --git a/tst/integration/computer/browser_computer_test.py b/tst/integration/computer/browser_computer_test.py index 4ad96acb0..7561ac43c 100644 --- a/tst/integration/computer/browser_computer_test.py +++ b/tst/integration/computer/browser_computer_test.py @@ -2063,6 +2063,70 @@ async def test_under_a_chromium_default_a_site_gets_chromium_unnamed_which_brows assert context.metadata["browser_steps"]["in"]["names"] == ["shop.test"] +FORWARDED_PAGE = "http://shop.localhost/admin/home" +FORWARDED_CHROMIUM = {**MULTI_CHROMIUM, + "env_params": {"form": "container", "app_settings": {"chromium": {"start_urls": [FORWARDED_PAGE]}}}} + + +@pytest.fixture(scope="module") +def forwarding_shop(browser_computer): + """A site named shop.localhost whose pages an app beside its server serves, signed in to by a header only the + site's own request sends, under a local id, as a multi env naming chromium's local id must be.""" + with namespace_routing(): + _seed_bootstrap_envs() + uid = uuid.uuid4().hex[:8] + return WebsiteEnv.put(id=f"@local/~/forwarding-{uid}/shop", environment_name="shop", domains=["shop.localhost"], + server_docker_image_artifact=_image("forwarding_website", f"forwarding-shop-{uid}")) + + +async def _forwarded(browser: DeployedEnv, user: str | None) -> dict: + """What the forwarding site's app saw of the browser's tab on its page, from the tab's title, once that says + ``user`` is signed in, or what it last said once ``WAIT_S`` passes.""" + seen, deadline = {}, time.monotonic() + WAIT_S + while time.monotonic() < deadline: + for tab in (await _browser_op(browser, "tabs/list", {}))["tabs"]: + if tab["url"] == FORWARDED_PAGE and tab.get("title", "").startswith("{"): + seen = json.loads(tab["title"]) + if seen and seen["user"] == user: + return seen + await asyncio.sleep(POLL_S) + return seen + + +@pytest.mark.asyncio +async def test_a_site_wrapping_an_app_signs_chromium_in_by_a_header_only_it_sends_and_the_app_serves_its_pages( + browser_computer, forwarding_shop): + """M8.35: a ForwardingSite's app serves a multi env's chromium its pages at the site's name, over HTTP for a + .localhost name, seeing that name as its Host and the page's own path; browser_sign_in signs the browser in through + the site's @session_state, the one request that sends the app its sign-in header, and the start tab loads again + signed in; the browser's requests never carry the header, and network.har has the app's pages.""" + with namespace_routing(): + office = _office(forwarding_shop, [Env.get(browser_computer.id)]) + sign_in = BrowserSignInTaskStep(id="in", version=None, env_id=office.id, site=forwarding_shop.id, user="amy") + context = TaskStepContext() + try: + [deployed] = (await DeployEnvTaskStep(id="office", version=None, env_id=office.id, sandbox_type="local", + ttl_seconds=900, child_envs={"chromium": FORWARDED_CHROMIUM} + ).execute(context)).deployed_envs + browser = DeployedEnv.from_dict(deployed.child_envs["chromium"]) + before = await _forwarded(browser, None) + await sign_in.execute(context) + after = await _forwarded(browser, "amy") + entries = await _har_entries(browser) + finally: + report = await teardown_run(context) + + assert not report.failed + [start_tab] = deployed.child_envs["chromium"]["metadata"]["computer"]["start_tabs"]["chromium"] + assert (start_tab["url"], start_tab["site"], start_tab["outcome"]) == (FORWARDED_PAGE, "shop", "loaded"), start_tab + page = {"host": "shop.localhost", "path": "/admin/home", "auto_login": False} + assert (before, after) == ({"user": None, **page}, {"user": "amy", **page}), (before, after) + loads = [entry for entry in entries if entry["request"]["url"] == FORWARDED_PAGE] + assert len(loads) >= 2 and {entry["response"]["status"] for entry in loads} == {200}, loads + assert not [entry for entry in entries for header in entry["request"]["headers"] + if header["name"].lower() == "x-auto-login"] + + @pytest.mark.asyncio async def test_a_multi_env_naming_no_browser_deploys_none(browser_computer, shop): with namespace_routing(): diff --git a/tst/unit/env/envs/test_computer_env.py b/tst/unit/env/envs/test_computer_env.py index e3fdf992b..efd63ced5 100644 --- a/tst/unit/env/envs/test_computer_env.py +++ b/tst/unit/env/envs/test_computer_env.py @@ -1142,7 +1142,7 @@ async def test_a_computer_with_site_routing_reaches_the_websites_the_run_deploye ((sandbox, sites, options),) = routes assert sandbox is boots.sandboxes[0] assert [site.model_dump() for site in sites] == [{"name": "notes", "url": "http://localhost:8080/website/notes/", - "domains": ["notes.test"]}] + "domains": ["notes.test"], "sign_in_headers": []}] assert options == {"user": "user", "network": None, "label": "desktop", "os_type": "linux", "display": "x11", "live_hosts": ()} assert boots.boots[0]["environment"][HOLD_BOOT_LAUNCH_ENV] == "1" diff --git a/tst/unit/env/envs/test_computer_site_proxy.py b/tst/unit/env/envs/test_computer_site_proxy.py index a6818f9e3..48b886ffb 100644 --- a/tst/unit/env/envs/test_computer_site_proxy.py +++ b/tst/unit/env/envs/test_computer_site_proxy.py @@ -7,6 +7,7 @@ import asyncio import base64 import contextlib +import dataclasses import email.message import io import json @@ -914,6 +915,47 @@ def test_install_writes_the_sites_files_names_and_trust_then_starts_the_proxy_on f"export {name}={value}\n" for name, value in sites.TRUST_ENVIRONMENTS["linux"].items()) +def test_every_field_of_a_sites_config_is_served_as_given_or_kept_private_and_the_proxy_loads_each(): + """A site's config entry, with every field proxy_config writes, holds only fields install serves as given or keeps + out of what the session user reads; and the proxy's Site takes each served field, and the files and headers made + of the private ones, so a field added to one and not the others fails here, not silently.""" + table = sites.site_table([SiteParams(name="notes", url="http://gw/website/notes/", domains=["notes.test"])]) + given = sites.proxy_config(table, sites.issue(table.routes, label="t"), user=None, os_type="linux", + forward_port=True, upstream_headers={"notes": {"X-Gateway": "g"}}, + sign_in_headers={"notes": ["X-Auto-Login"]}) + (entry,) = given["sites"] + loaded = {f.name for f in dataclasses.fields(site_proxy.Site) if f.init} + + assert set(entry) == set(site_proxy.SERVED_SITE_FIELDS) | set(site_proxy.PRIVATE_SITE_FIELDS), sorted(entry) + assert loaded == set(site_proxy.SERVED_SITE_FIELDS) | {"cert", "key", "upstream_headers"}, sorted(loaded) + + +def test_install_keeps_a_sites_sign_in_headers_for_the_proxy_it_starts_whose_network_log_takes_their_values_out( + tmp_path, monkeypatch): + root, folder = tmp_path / "root", tmp_path / "sites" + (root / "etc/ssl/certs").mkdir(parents=True) + (root / "etc/ssl/certs/ca-certificates.crt").write_text("PUBLIC ROOTS\n") + folder.mkdir() + folder.chmod(0o700) + table = sites.site_table([SiteParams(name="shop", url="http://gw/website/shop/", domains=["shop.localhost"]), + SiteParams(name="notes", url="http://gw/website/notes/", domains=["notes.test"])]) + given = sites.proxy_config(table, sites.issue(table.routes, label="t"), user=None, os_type="linux", + forward_port=True, sign_in_headers={"shop": ["X-Auto-Login"]}) + (folder / "config.json").write_text(json.dumps(given)) + started = [] + monkeypatch.setattr(site_proxy, "start", lambda path, log: started.append(Config.load(path))) + monkeypatch.setattr(site_proxy, "checks_as", lambda user, path: {"sites": {}, "runtimes": {}}) + monkeypatch.setattr(site_proxy.shutil, "which", lambda tool: None) + + site_proxy.install(str(folder / "config.json"), site_proxy.Paths(root)) + + (config,) = started + served = {site.name: site for site in config.sites} + assert (served["shop"].sign_in_headers, served["notes"].sign_in_headers) == (["X-Auto-Login"], []) + assert site_proxy.logged_headers([("x-auto-login", "admin:test")], served["shop"].sign_in_headers) == [ + ["x-auto-login", site_proxy.REMOVED]] + + def test_install_keeps_the_upstream_headers_in_a_file_root_alone_reads_and_serve_json_names_it(tmp_path, monkeypatch): root, folder = tmp_path / "root", tmp_path / "sites" (root / "etc/ssl/certs").mkdir(parents=True) diff --git a/tst/unit/env/envs/test_computer_sites.py b/tst/unit/env/envs/test_computer_sites.py index 85937cfe1..932e6db10 100644 --- a/tst/unit/env/envs/test_computer_sites.py +++ b/tst/unit/env/envs/test_computer_sites.py @@ -390,6 +390,41 @@ async def test_a_site_naming_the_gateway_its_routes_are_on_is_refused_before_any assert sandbox.scripts == [] and sandbox.written == {} +@pytest.mark.asyncio +async def test_a_sites_sign_in_headers_go_from_its_card_to_its_proxy_whose_network_log_takes_their_values_out( + monkeypatch): + from agentenv_protocol import client as protocol_v1 + + from agent_env.env.env import DeployedGatewayEnv + from agent_env.env.envs.computer.utils import site_proxy + + cards = {"https://gw.example/svc/shop": {"capabilities": {"extensions": [{"uri": "urn:agentenv:website/v1", "params": { + "domains": ["shop.localhost"], "sign_in_headers": ["X-Auto-Login", 7, ""]}}]}}, + "https://gw.example/svc/notes": {"capabilities": {"extensions": [ + {"uri": "urn:agentenv:website/v1", "params": {"domains": ["notes.test"]}}]}}} + + async def get_card(url, timeout=10, verify=True): + return cards[url] + + monkeypatch.setattr(protocol_v1, "get_card", get_card) + gateway = DeployedGatewayEnv(env_id="sites", env_version=1, sandbox_id="s", gateway_url="https://gw.example", + website_frontend_urls={name: f"https://gw.example/website/{name}/" + for name in ("shop", "notes")}) + shop, notes = [Site(**found) for found in await sites.deployed_sites([gateway])] + sandbox = FakeVm({**_REPORT, "names": ["shop.localhost", "notes.test"], "checks": { + "shop.localhost": {"url": "http://shop.localhost/", "status": 200}, **_REPORT["checks"]}}) + + await sites.route(sandbox, [shop, notes], user="user", network=None, label="desktop", os_type="linux", + display="none") + + entries = {entry["name"]: entry for entry in json.loads(sandbox.written["/etc/agentenv/sites/config.json"])["sites"]} + assert (shop.sign_in_headers, notes.sign_in_headers) == (["X-Auto-Login"], []) + assert entries["shop"]["sign_in_headers"] == ["X-Auto-Login"] and "sign_in_headers" not in entries["notes"] + assert site_proxy.logged_headers([("x-auto-login", "admin:test"), ("Accept", "*/*")], + entries["shop"]["sign_in_headers"]) == [["x-auto-login", site_proxy.REMOVED], + ["Accept", "*/*"]] + + @pytest.mark.asyncio async def test_the_sites_a_run_deployed_are_read_from_each_gateways_records_and_their_cards(monkeypatch): from agentenv_protocol import client as protocol_v1 diff --git a/tst/unit/task_step/test_browser_sign_in.py b/tst/unit/task_step/test_browser_sign_in.py index 2c585d865..2ae27e168 100644 --- a/tst/unit/task_step/test_browser_sign_in.py +++ b/tst/unit/task_step/test_browser_sign_in.py @@ -102,9 +102,8 @@ async def test_a_browser_in_a_multi_envs_computer_is_signed_in_with_the_sites_se assert (json.loads(session.content), session.headers["authorization"]) == ({"user": "alex"}, "Bearer gateway-token") assert (str(add.url), add.headers["authorization"]) == (f"{_DESKTOP}/svc/chromium/agentenv", "Bearer desktop-token") body = json.loads(add.content) - unplaced = {key: value for key, value in _SESSION.items() if key != "path"} assert body["method"] == "data/add" and body["params"]["parts"] == [{"kind": "data", "data": { - "cookies": [{**unplaced, "url": "https://shop.test/"}, {**unplaced, "path": "/", "domain": ".shop.test"}], + "cookies": [{**_SESSION, "path": "/", "url": "https://shop.test/"}, {**_SESSION, "path": "/", "domain": ".shop.test"}], "local_storage": []}}] assert context.metadata["browser_steps"]["in"] == { "env_id": "office", "browser": None, "site": "shop-site", "user": "alex", "cookies": ["agentenv_session_shop"], @@ -193,12 +192,14 @@ async def test_a_site_that_cant_sign_in_fails_the_step_before_the_browser_is_tou assert served == [] -def test_each_cookie_is_placed_on_each_name_the_site_is_reached_by(): +def test_each_cookie_is_placed_on_each_name_the_site_is_reached_by_at_its_own_path(): cookie = {"name": "s", "value": "v", "path": "/app", "max_age": 60, "domain": "ignored"} assert scoped([cookie], ["shop.test", "*.shop.test", "notes.localhost"]) == [ - {"name": "s", "value": "v", "max_age": 60, "url": "https://shop.test/app"}, + {"name": "s", "value": "v", "path": "/app", "max_age": 60, "url": "https://shop.test/app"}, {"name": "s", "value": "v", "path": "/app", "max_age": 60, "domain": ".shop.test"}, - {"name": "s", "value": "v", "max_age": 60, "url": "http://notes.localhost/app"}] + {"name": "s", "value": "v", "path": "/app", "max_age": 60, "url": "http://notes.localhost/app"}] + assert scoped([{"name": "s", "value": "v"}], ["shop.test"]) == [ + {"name": "s", "value": "v", "url": "https://shop.test/", "path": "/"}] def test_the_step_round_trips_and_checks_its_fields(monkeypatch):