diff --git a/AGENTS.md b/AGENTS.md index aa980e2e8..a8b4d6a4d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -117,9 +117,9 @@ and deserialized through a registry. | Package | Contents | |---|---| -| `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset`, `close` and `load_file_artifact_universe`; teardown closes each env a run deployed before terminating its sandboxes by id. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`, one site: a server image, `backend_image` in its env.toml, serving the pages and API, checked at its `health` path; a separate frontend image is deprecated, warned of at put and optional in a bundle, and a stored `backend_docker_image_artifact` reads as the server image), `GatewayEnv` (`gateway_server`), `ComputerDriverEnv` (`computer_driver`, `env/envs/computer/driver_env.py`: a computer driver's files and what its class declares for each platform it supports, read at put in a process of its own, `agentenv_protocol.computer._declare`, where the driver's own dependencies needn't be installed, with the `port` it serves on and its install by OS, `[install.]` (`env/envs/computer/utils/install.py`): files, apt `packages` with extra `sources`, a `python` requirements file for a venv, `downloads` by URL and sha256, `run` scripts, the services the base's init runs, and `ports`; it deploys only as a computer env's driver), `ComputerEnv` (`computer`, in `env/envs/computer/`: a base computer and a driver, matched at put by gate 1 before anything is stored, then composed by `composer.py` into an image for each form and arch on a builder of the put's sandbox provider, by default the env's (`utils/builder.py`; `--sandbox` on `env build`, `bundle register` and `run`): this machine's Docker for `local`, else a VM of the provider, which fetches the put's stored files by signed URL, takes the rest inline and pushes to the image store itself, so it builds only with stores a remote VM reaches; each arch goes to the first link that runs it natively, else one that emulates it (a VM registers QEMU; `--no-emulate` leaves it `skipped` instead, for a later put), and its pair records the provider that built it (`builder`); a run or register that reuses a computer env composes what it has left (`pending`), a run only the forms its sandbox's deploys boot (`booted_forms`: on each link, the first form it boots natively, by the deploy's own rule, `boot.first_native`; a compose of a disk adds its container), and a deploy composes first what it boots that no compose started (`ComputerEnv.compose_unstarted`); tagged `agentenv-computers:` in the image store by a key of the image's Dockerfile, build context and lock so any env with the same inputs reuses it; every layer's packages, requirements and downloads are resolved at put in a container of the base for each arch (`utils/packages.py`, `utils/resolve_in_base.py`), stored in the object store by sha256 under `agentenv-packages/`, and listed in a lock stored by a digest of its inputs, which any env with the same inputs reuses, and builds install only the lock's files, every install step `RUN --network=none`; each pair's status goes into its metadata, `composed`, the arches built are written as its computer, a `computer` artifact over one index, and `agent-env env build` resumes what a put left; gate 3 (`utils/probing.py`) then boots each native build on the put's sandbox provider and probes it at `full`, making it `ready` or `failed`, and a build no provider runs natively stays `unverified`. Its deploy, through the `computer` env provider (`providers/env_providers/env_computer_provider.py`), boots its first `ready` build on a sandbox provider that runs its arch natively (`native_platforms()`), with no gateway in front and the driver's card as the env card, then checks its `display`; `deploy_env` `env_params` take a `form`, `computer_settings` (deploy-phase values of the base's settings; a secret one, here or on the env, only as a bare `secret:`/`env:` reference) and `driver_settings` (the driver's `ComputerSettings`, such as its default coordinate space); a `network` that allows nothing runs it on local Docker's internal network behind a relay (`providers/sandbox_providers/local_docker/relay/`); `reset` and `close` have the driver end its recordings first (`computer-recording/v1` `end_all`, each uploading what's left and its manifest under the grant it started with), and `reset` then boots a fresh container in its place (`reset_container`), keeping the computer's address but not an agent's MCP session to the old driver; its `load_file_artifact_universe` writes files under the session user's home through the driver's `data/add`, `Downloads` by default, each file's bytes through a read grant when the store's grants reach it and its network allows everything, else inline (`env/envs/computer/utils/inside.py`)), `ComputerAppEnv` (`computer_app`, `env/envs/computer/app_env.py`: an app on a computer, a `ComputerAppEnvironment` class its env.toml's `module` names, read at put by the same `_declare`, which declares each OS with its own decorator (`@linux(...)`, or `@install(os_type, spec)` for any OS type) holding its install, its launch and, optionally, a skill and settings and build settings that add to the class's there, and install steps and ready checks for every OS or those their `os=` names; everything that reads its settings, launch or steps asks for the computer's OS type; deployed alone it is a one-app computer env, a `ComputerEnv` of the computer and driver its env.toml names, the composer installing it under `/opt/agentenv/apps//` with its services as supervisord programs beside its own server (`agentenv_protocol.computer.serve_app`, in a venv with the protocol's `app` extra, on `.agentenv/server.sock` in its folder), which adds its launcher, launches a `boot` app once per boot and says through `computer-app/v1` whether it's ready, as the driver's `computer-ready` asks, its card named after it, and `env_params` `app_settings` handing it values of its `Settings` in `AGENTENV_APP_SETTINGS`, a secret one only as a bare reference, over its env.toml's `[settings]` and a computer env's `settings` beside it in `apps`, each by top-level field (`params.app_settings`; checked at put, a required one left to the deploy; the deploy record keeps the merged values as `app_settings`, secrets withheld); its env.toml's `build_settings`, values of its `BuildSettings` checked at put, are written to `.agentenv/build_settings.json` in its folder, so they're part of the image's key, its install steps read them, and a deploy can't change them). An env type whose services take environment data defines `load_environment_artifact` and `load_environment_universe_artifact`; `load_artifact` refuses an environment artifact aimed at a type that doesn't, at save and before it loads. An env built on the protocol SDK can serve agents a skill (`@agent_skill`, `skill/v1`), which `add_skills` `env_ids` installs under its environment name (behind a gateway, each server's under that server's name); a skill with files beside its SKILL.md goes as a bundle stored once per content under `agentenv-env-skills/` in the object store. `env/gateway/gateway.py` is the env-side gateway server, container-only code. Its site route, `/website//`, never forwards a site's data plane or card (`/agentenv/…`, `/.well-known/agent-env.json`), which agentenv reaches through `/svc//`; a site with no frontend image is served by its server image there. While its clock is armed it serves agents `get_time` (`env/gateway/get_time.py`), which answers in UTC and, given an IANA `time_zone`, also in that zone; executors' working `hours` (`env/gateway/triggers.py`) take a fixed `utc_offset` or an IANA `time_zone`. | +| `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset`, `close` and `load_file_artifact_universe`; teardown closes each env a run deployed before terminating its sandboxes by id. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`, one site: a server image, `backend_image` in its env.toml, serving the pages and API, ready when its card's `website/v1` `ready` answers a `WebsiteReadiness` that says so, as `agentenv_protocol.website` (`website_readiness`, `env_readiness`) defines a website env's health for the deploy and `DeployedEnv.readiness()` alike, or, for a card without `ready`, when its `health` path answers 200, which is deprecated; the gateway container's `env/gateway/constants.py` mirrors the URI and op, a test keeping them equal; a separate frontend image is deprecated, warned of at put and optional in a bundle, and a stored `backend_docker_image_artifact` reads as the server image), `GatewayEnv` (`gateway_server`), `ComputerDriverEnv` (`computer_driver`, `env/envs/computer/driver_env.py`: a computer driver's files and what its class declares for each platform it supports, read at put in a process of its own, `agentenv_protocol.computer._declare`, where the driver's own dependencies needn't be installed, with the `port` it serves on and its install by OS, `[install.]` (`env/envs/computer/utils/install.py`): files, apt `packages` with extra `sources`, a `python` requirements file for a venv, `downloads` by URL and sha256, `run` scripts, the services the base's init runs, and `ports`; it deploys only as a computer env's driver), `ComputerEnv` (`computer`, in `env/envs/computer/`: a base computer and a driver, matched at put by gate 1 before anything is stored, then composed by `composer.py` into an image for each form and arch on a builder of the put's sandbox provider, by default the env's (`utils/builder.py`; `--sandbox` on `env build`, `bundle register` and `run`): this machine's Docker for `local`, else a VM of the provider, which fetches the put's stored files by signed URL, takes the rest inline and pushes to the image store itself, so it builds only with stores a remote VM reaches; each arch goes to the first link that runs it natively, else one that emulates it (a VM registers QEMU; `--no-emulate` leaves it `skipped` instead, for a later put), and its pair records the provider that built it (`builder`); a run or register that reuses a computer env composes what it has left (`pending`), a run only the forms its sandbox's deploys boot (`booted_forms`: on each link, the first form it boots natively, by the deploy's own rule, `boot.first_native`; a compose of a disk adds its container), and a deploy composes first what it boots that no compose started (`ComputerEnv.compose_unstarted`); tagged `agentenv-computers:` in the image store by a key of the image's Dockerfile, build context and lock so any env with the same inputs reuses it; every layer's packages, requirements and downloads are resolved at put in a container of the base for each arch (`utils/packages.py`, `utils/resolve_in_base.py`), stored in the object store by sha256 under `agentenv-packages/`, and listed in a lock stored by a digest of its inputs, which any env with the same inputs reuses, and builds install only the lock's files, every install step `RUN --network=none`; each pair's status goes into its metadata, `composed`, the arches built are written as its computer, a `computer` artifact over one index, and `agent-env env build` resumes what a put left; gate 3 (`utils/probing.py`) then boots each native build on the put's sandbox provider and probes it at `full`, making it `ready` or `failed`, and a build no provider runs natively stays `unverified`. Its deploy, through the `computer` env provider (`providers/env_providers/env_computer_provider.py`), boots its first `ready` build on a sandbox provider that runs its arch natively (`native_platforms()`), with no gateway in front and the driver's card as the env card, then checks its `display`; `deploy_env` `env_params` take a `form`, `computer_settings` (deploy-phase values of the base's settings; a secret one, here or on the env, only as a bare `secret:`/`env:` reference) and `driver_settings` (the driver's `ComputerSettings`, such as its default coordinate space); a `network` that allows nothing runs it on local Docker's internal network behind a relay (`providers/sandbox_providers/local_docker/relay/`); `reset` and `close` have the driver end its recordings first (`computer-recording/v1` `end_all`, each uploading what's left and its manifest under the grant it started with), and `reset` then boots a fresh container in its place (`reset_container`), keeping the computer's address but not an agent's MCP session to the old driver; its `load_file_artifact_universe` writes files under the session user's home through the driver's `data/add`, `Downloads` by default, each file's bytes through a read grant when the store's grants reach it and its network allows everything, else inline (`env/envs/computer/utils/inside.py`)), `ComputerAppEnv` (`computer_app`, `env/envs/computer/app_env.py`: an app on a computer, a `ComputerAppEnvironment` class its env.toml's `module` names, read at put by the same `_declare`, which declares each OS with its own decorator (`@linux(...)`, or `@install(os_type, spec)` for any OS type) holding its install, its launch and, optionally, a skill and settings and build settings that add to the class's there, and install steps and ready checks for every OS or those their `os=` names; everything that reads its settings, launch or steps asks for the computer's OS type; deployed alone it is a one-app computer env, a `ComputerEnv` of the computer and driver its env.toml names, the composer installing it under `/opt/agentenv/apps//` with its services as supervisord programs beside its own server (`agentenv_protocol.computer.serve_app`, in a venv with the protocol's `app` extra, on `.agentenv/server.sock` in its folder), which adds its launcher, launches a `boot` app once per boot and says through `computer-app/v1` whether it's ready, as the driver's `computer-ready` asks, its card named after it, and `env_params` `app_settings` handing it values of its `Settings` in `AGENTENV_APP_SETTINGS`, a secret one only as a bare reference, over its env.toml's `[settings]` and a computer env's `settings` beside it in `apps`, each by top-level field (`params.app_settings`; checked at put, a required one left to the deploy; the deploy record keeps the merged values as `app_settings`, secrets withheld); its env.toml's `build_settings`, values of its `BuildSettings` checked at put, are written to `.agentenv/build_settings.json` in its folder, so they're part of the image's key, its install steps read them, and a deploy can't change them). An env type whose services take environment data defines `load_environment_artifact` and `load_environment_universe_artifact`; `load_artifact` refuses an environment artifact aimed at a type that doesn't, at save and before it loads. An env built on the protocol SDK can serve agents a skill (`@agent_skill`, `skill/v1`), which `add_skills` `env_ids` installs under its environment name (behind a gateway, each server's under that server's name); a skill with files beside its SKILL.md goes as a bundle stored once per content under `agentenv-env-skills/` in the object store. `env/gateway/gateway.py` is the env-side gateway server, container-only code. Its site route, `/website//`, never forwards a site's data plane or card (`/agentenv/…`, `/.well-known/agent-env.json`), which agentenv reaches through `/svc//`; a site with no frontend image is served by its server image there. While its clock is armed it serves agents `get_time` (`env/gateway/get_time.py`), which answers in UTC and, given an IANA `time_zone`, also in that zone; executors' working `hours` (`env/gateway/triggers.py`) take a fixed `utc_offset` or an IANA `time_zone`. | | `artifact/` | Immutable versioned artifacts; every `put()` creates a new version. The document goes to the document store, binary payloads to the object store. Every artifact but an environment universe, whose `metadata` key holds its legacy metadata file ids, carries a `metadata` dict outside its versioned fields, changed by compare-and-swap (`merge_metadata`). Built-ins: `CliArtifact` (`cli`), `ComputerArtifact` (`computer`, a base computer: its architectures, OS, session user, settings and container form, whose image `put` reads from its registry into `metadata.validated_images`), `FileArtifact` (`file`), `FileArtifactUniverse` (`file_artifact_universe`), `DockerImageArtifact` (`docker_image`), `EnvironmentArtifact` (`environment`: its environment's files by path, stored as the one wrapped `file_artifact_ref` every environment artifact had when it has one file at its filename, else as `file_artifact_refs`; `get_file_artifacts()` reads either, and a load hands each file to the env's `data/add` with its path, which a pre-v1 load, taking one file, refuses), `EnvironmentUniverseArtifact` (`environment_universe`), `SkillArtifact` (`skill`); `[artifacts] type_aliases` maps another stored spelling of a type onto one of them. A `DockerImageArtifact` holds a tar.gz of its image, or, written by `put_ref` (`--image-ref` on `env mcp-server put` and `a2a-agent put`), only a registry reference with its tag pinned to a digest, which sandboxes pull, or a published image's tag with its `image_digest` and its build context, which sandboxes pull by the digest and build from the context when they can't (the infra images, and a bundle agent built from a shipped agent's sources, `Plan.published`), or, written by `put_context`, only its build context, which a VM sandbox builds before running it (`VmSandbox.load_docker_images`) and Modal builds before creating its container (`ModalSandboxProvider.prepare_image`, outside a fallback chain's deadline); anything else that runs images by name (a container on the local provider, a lone server on any provider but Modal) refuses such an image. A build context (`utils/docker/context.py`) is what `docker build` would send, `.dockerignore` applied, written so equal files give equal bytes and an equal `source_digest`. | -| `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, among them `website_state_verifier`, which reads each checked session of a deployed website env's sites, or a multi env's, at the site's `/go?sid=` through `/svc//` and checks one dot path of its `current_state`, `initial_state` or `state_diff` each, optionally reduced by `len`, `keys` or `bool`, with `eq`, `ne`, `ge`, `gt`, `le`, `lt`, `in` or `contains`, its sites checked before a run against the `env_version` it pins, `start_computer_recording` and `end_computer_recording`, and more). On an env whose card serves computer-exec/v1 (a computer), `run_code` with `env_id` and `run_container_unit_tests_verifier` with `env_id` run inside it as its session user through the driver: files staged with `data/add` and read with `data/get`, commands run with `run`; `env_outcome_verifier` hands a `verify` that takes an `env_url` keyword the env's address. A computer's framework-only routes (everything but MCP and the card) take the per-deploy control token its provider boots it with (`DeployedEnv.control_token`, `AGENTENV_CONTROL_TOKEN`); framework calls send `deployed.control_headers`, `Task.run` runs every step under `protocol_v1.authorized` with each deployed env's (a `verify` included), and a `run_code` script inside the computer gets `input["env"]["headers"]`. `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | +| `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, among them `start_computer_recording` and `end_computer_recording`, and more). On an env whose card serves computer-exec/v1 (a computer), `run_code` with `env_id` and `run_container_unit_tests_verifier` with `env_id` run inside it as its session user through the driver: files staged with `data/add` and read with `data/get`, commands run with `run`; `env_outcome_verifier` hands a `verify` that takes an `env_url` keyword the env's address. A computer's framework-only routes (everything but MCP and the card) take the per-deploy control token its provider boots it with (`DeployedEnv.control_token`, `AGENTENV_CONTROL_TOKEN`); framework calls send `deployed.control_headers`, `Task.run` runs every step under `protocol_v1.authorized` with each deployed env's (a `verify` included), and a `run_code` script inside the computer gets `input["env"]["headers"]`. `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | | `store/` | Four store ABCs with local and cloud implementations: `DocumentStore` (SQLite, MongoDB, DynamoDB, Firestore with MongoDB compatibility), `ObjectStore` (filesystem, S3, Cloud Storage), `ImageStore` (local OCI registry, ECR), `SecretStore` (env vars or file, AWS Secrets Manager, Google Cloud Secret Manager). `VersionedEntityStore` implements the shared versioned get/put logic, `QueryBuilder` is the immutable chained query API, `store/base.py` holds the error types. `store/routing.py` is the local namespaces, `@local` and `@hf` (`is_local_id`), each with a documents file of its own: while the CLI or a `build_app` server runs, a local id and what a local task's run records go to local stores beside the configured ones, and that run's writes to the configured stores are refused. A new backend must pass the conformance kits in `tst/store/`. | | `config/` | The `Config` singleton (`get_config`, `configure`, `reset_config`) in `config/runtime.py`, file discovery in `config/loader.py`, and `load_impl`, which resolves `module:Class` pointers. `agent_env.store` re-exports the config names for compatibility. | | `providers/` | `providers/sandbox_providers/` holds the sandbox providers `local`, `modal`, `modal_vm`, `e2b`, `sail_vm` (the `sail` extra), `smol_vm` (the `smol` extra), `tensorlake` (the `tensorlake` extra), `tenki` (the `tenki` extra), `vercel` (the `vercel` extra), `boxd` (the `boxd` extra), `coreweave` (the `coreweave` extra), `daytona` (the `daytona` extra), `boat` (the `boat` extra); `[sandbox] default` and `agent_default` accept a comma-separated fallback chain, whose deploys skip a provider that can't run them. A sandbox provider declares what it runs (`ON_THIS_MACHINE`, and for VMs on this machine the address their requests to it arrive at, `REACHES_HOST_AT`; `CONTAINER_ACCEPTS`, `PRIVATE_NETWORK`; `SUPPORTED_SANDBOX_MODES`, derived from whether it implements `create_sandbox` and `create_vm`; and `SANDBOX_ACCEPTS`, the images an agent's sandbox runs, which defaults to those a VM loads when it implements `create_vm` and to `CONTAINER_ACCEPTS` otherwise, so only a provider that implements `create_vm` yet gives an agent a container, like the local one, declares it), each sandbox carries the `platform` of the images it runs (linux/amd64 unless its provider sets another when it creates it; None, any, unchecked, for a local sandbox), and code outside `providers/sandbox_providers/` reads those, never which provider it has. `providers/env_providers/` holds the environment providers: `EnvironmentProvider` (an env's containers and state store), `EnvironmentGatewayProvider` (`gateway`, an env's default), which renders a docker-compose for the gateway and its MCP servers inside the sandbox, and `EnvironmentServerProvider` (`server`), one MCP server in its own container with no gateway in front and no state store; `providers/env_state/` holds env-state providers (`local_postgres` built in). | diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index b6cc8c649..847455739 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -198,6 +198,16 @@ with any other transport are kept beside the SDK's entry. Card URLs are paths, r address the card was fetched from. On the client side, `client.mcp_path(card)` returns the declared path, or `/mcp` for a card without one. +A composed card, such as a gateway's, nests each backing environment's card under +`children_environments`, its `url` and endpoints rooted at the route the composer serves it at. +`client.children(card)` returns each as a `ChildEnv` (its `name`, `url` and `card`, with its +`extensions`, `operations` and `serves(method)`, and `base_url(address)`, its data plane resolved +against the composed card's address). Two children may share a name, as a website and an MCP +server can, so `client.child_at(card, url)` finds one by its url or route; `client.child_named(card, +name)` raises `AmbiguousChildError` for a shared name, and `client.find_child` returns the first. +`client.serves(card, method)` says whether a card serves a data-plane method by its declared +`operations`, None for a card that declares none. + Extensions are invoked from what the card advertises. `client.find_extension_method(card, uri, method)` returns one advertised method, whose `endpoint` is the method's own or else the extension's, and `client.invoke_extension(base_url, card, uri, params, method=...)` calls it with @@ -420,6 +430,33 @@ async with ToolSession(mcp_url=mcp_url, role="alice@example.com") as session: - **Tools:** `list_tools()` returns the tools the role may use as `EnvironmentTool`, the type a card's `capabilities.tools` holds. Each `inputSchema` is the JSON Schema a model API takes as the tool's parameters. +## Website readiness + +`agentenv_protocol.website` is the one definition of whether a website env is healthy. A site's card advertises +`urn:agentenv:website/v1`'s `ready` (`WEBSITE_READY_OP`), a GET answering a `WebsiteReadiness`: `ready`, a boolean, +and `problems`, the names of what keeps it from being ready (a `WebsiteEnvironment`'s are its `@ready_check` methods +not yet true). The model is strict about those two (`"yes"` is no `ready`) and ignores fields it doesn't know, so a +reader keeps reading a newer site. Everything that judges a site reads it through these helpers: agent-env's deploy, +`DeployedEnv.readiness()`, and the conformance checker's `health`, which fails an answer that isn't a +`WebsiteReadiness`. + +```python +from agentenv_protocol.website import env_readiness, website_readiness + +site = await website_readiness(base_url, card) # WebsiteReadiness, or None for a card without ready +env = await env_readiness(gateway_url, env_card) # WebsiteReadiness of every site of the env +print(env.ready, env.problems) # False ['shop: seeded'] +``` + +- **One site:** `website_readiness(base_url, card, *, timeout=READY_TIMEOUT_S, headers=None)` asks the site as its card + advertises `ready`. It raises httpx's error when the site can't be reached, `ExtensionCallError` for a non-2xx + answer and `ValueError` (pydantic's `ValidationError`) for an answer that isn't a `WebsiteReadiness`. +- **An env:** `env_readiness(base_url, card, ...)` asks, at once, the card itself when it advertises `ready` (a single + site's card), and every child under a composed card's `children_environments` that does, its endpoints rooted at + `base_url` as a gateway's are. A child's problems are prefixed with its name (`shop: seeded`); one that can't be + asked, or answers no `WebsiteReadiness`, is a problem itself. A card or child without `ready`, an MCP server's or an + older site's, isn't counted, so an env with no site serving it is ready with no problems. + ## Computer drivers A computer driver is what runs inside a computer an agent uses through screenshots and input. Write diff --git a/packages/agentenv-protocol/docs/website-spec.md b/packages/agentenv-protocol/docs/website-spec.md new file mode 100644 index 000000000..664b90342 --- /dev/null +++ b/packages/agentenv-protocol/docs/website-spec.md @@ -0,0 +1,359 @@ +# The website spec + +A website env used to be described in three places: its `env.toml` (`type = "website"`, `health`, `backend_image`), +the decorators in its code (`@website(domains, static, health, data)` and `@api` routes with names and roles), and the +card it serves (the `urn:agentenv:website/v1` extension: domains, health, routes, session cookie). The website spec +is one declaration of a site. Each way of serving a site either produces a spec or is checked against one, and specs +compose, so a site can be assembled from parts. + +The code is `agentenv_protocol.website.spec`. The JSON Schema of a spec file is +`python -m agentenv_protocol.website.spec schema`. + +## Shape + +A spec is a TOML file. Its top-level keys are fixed and every section is optional: + +| Section | What it says | +| --- | --- | +| `extends` | The path of a spec this one overlays (see [Overlays](#overlays)). | +| `[site]` | `name`, the card's name; `domains`, the host names the site answers for, with `*.` as the leftmost label matching any subdomain; `health`, the path that answers 200 once the site is usable (`/api/health` by default), read only for a server whose card has no `website/v1` `ready`, which is deprecated: a deploy and the conformance check read readiness from the card. | +| `[source]` | What serves the site, exactly one of `image`, `sdk`, `static`, `live` or `replay` (see [Sources](#sources)). | +| `[data]` | `collections` (each with `key`, `owner` and an optional JSON `schema`), or one `schema` of the whole state, or `models = "mcp:"` to bind the site's data to an MCP server's schema; and `seed`, the files it starts with, in order. | +| `[users]` | `personas` (ids, or `{id, name, role}` tables); `acting_user`, who a browser with no session is; `sign_in`, which is `cookie` (the site's signed session cookie, by default), `none` or `live`; and, for a cookie sign-in by a server not built on the SDK, its `session_cookie` and `session_path`. | +| `[roles]` | A role name mapped to the routes it may call. These roles are added to each route's own `roles`. | +| `[routes]` | `ops`, the named operations, each `name = "METHOD /path"` or a table `{method, path, roles, served_by}`; and `openapi`, an OpenAPI document whose operations, named by `operationId` (each name once) with roles from `x-agentenv-roles`, are routes too. Triggers and records call these names `op`. | +| `[clock]` | How the site follows the universe's time: `follow = "header"` (`X-Universe-Now` on every answer), `"date_patch"` (its pages' `Date` is shifted too), or `"none"` (wall time). | +| `[[uses]]` | Components: another spec mounted under a path prefix (see [Composition](#composition)). | + +Paths in a spec (Dockerfiles, folders, seeds, schemas, captures, the OpenAPI document and components) are relative to +the spec file. + +The Notes example's spec, `examples/website/notes/website.toml`: + +```toml +[site] +name = "notes" +domains = ["notes.test"] +health = "/healthz" + +[source] +sdk = "site:Notes" +assets = "web/dist" + +[data] +seed = ["data/seed.json"] + +[data.collections.users] + +[data.collections.notes] +owner = "owner" + +[users] +personas = [{ id = "ann", name = "Ann Lee" }, { id = "ben", name = "Ben Ortiz" }] +acting_user = "ann" +sign_in = "cookie" + +[roles] +admin = ["all_notes"] + +[routes.ops] +list_users = "GET /api/users" +sign_in = "POST /api/sign-in" +list_notes = "GET /api/notes" +create_note = "POST /api/notes" +get_note = "GET /api/notes/{note_id}" +edit_note = "PATCH /api/notes/{note_id}" +toggle_note = "POST /api/notes/{note_id}/toggle" +delete_note = "DELETE /api/notes/{note_id}" +all_notes = "GET /api/admin/notes" + +[clock] +follow = "header" +``` + +## Validation + +`load(path)` reads a spec, resolves `extends` and `uses`, and validates the result. If anything is wrong, it raises +`SpecError`, whose `problems` lists every problem as `: : `. For example, +`.../shop.toml: routes.ops: b and a are both GET /x`. The checks within a section are: unknown keys, exactly one +source, lower-case host names, health and route paths outside the data plane's `/agentenv` and the card's +`/.well-known/agent-env.json`, credentials that are `secret:`/`env:` references rather than values, and an +`acting_user` that is one of the `personas`. + +The checks across sections are: + +- No two routes share a verb and path, and no GET route is served at the health path. +- `[roles]` names only routes that exist, including the namespaced routes of components. +- `sign_in = "live"` needs a `live` or `replay` source. +- A `live` or `replay` site is read as `diff`, and only such a site is. +- An `sdk` site follows the clock, so `follow = "none"` is refused for it. +- A `static` site serves no routes. +- `served_by` is allowed only on a route of a live or replayed site. +- A collection with an `owner` needs a site that signs users in. + +`python -m agentenv_protocol.website.spec validate PATH` runs these checks. `show PATH` prints the resolved spec as +JSON, and `show --card PATH` prints the card params it advertises. + +## Sources + +Every source maps onto the same spec. Each row below says how a source is described, what it produces, and what it is +checked against. + +| Source | `[source]` | Where the spec comes from | Deployable today | +| --- | --- | --- | --- | +| SDK class | `sdk = "module:Class"`, `assets = ""` | `@website(spec="website.toml")` declares the class from the spec, and the class is refused if it doesn't serve what the spec declares. `from_environment(cls)` derives a spec from the decorators of a class declared the old way. | Yes, as a website env's server image | +| Any-stack image | `image = ""` or `image = { dockerfile, context }` | Written by hand. `conformance --spec` checks the running server against it. | Yes, through `backend_image` in env.toml | +| Static | `static = ""` | Written by hand. | Modeled only: no generic static server image ships yet | +| Live real site | `live = { allow_hosts, credentials }` | Written by hand; `from_card` reads one back from a proxy that serves a card. | Modeled only; a website env refuses it | +| Replayed capture | `replay = { capture, allow_hosts }` | Usually an overlay of the live spec. | Modeled only; a website env refuses it | + +For every source, `to_card_extension()` returns the `urn:agentenv:website/v1` params a server of the spec advertises: +`domains`, `health` and `routes` (each `{method, path, name, roles}`); `session_cookie` and `session_path` for a cookie +sign-in; and `clock` when it is declared. A WebsiteEnvironment builds its card from these params, so +the card and the spec cannot drift apart. `from_card(card)` reads a running site's card back into a spec. It recovers +the site, routes, sign-in and clock. A card does not carry a site's source, data, personas or the +`[roles]` table; roles are folded into each route. + +### The SDK + +```python +@website(spec="website.toml") +class Notes(WebsiteEnvironment): + ... +``` + +`@website(spec=...)` takes a path relative to the class's file, or a loaded `WebsiteSpec`. The spec supplies the +domains, health path, static folder (`assets`), seed files and clock patch. If any of those are also passed to +`@website`, it is refused. When the class is declared, it is checked against the spec: + +- Its `@api` routes must match the spec's routes by name, verb, path and roles, whenever the spec declares routes. +- `source.sdk` must name the class. +- The sign-in must be the SDK's cookie. +- `acting_user` must be the seed's `current_user`. + +If the class has no `@environment_card` name, the card is named by `site.name`. A card name that differs from +`site.name` is refused. Classes declared with `@website(domains=..., ...)` work as before; their card goes through +`from_environment(cls).to_card_extension()`. + +### env.toml + +A website env's `env.toml` can point to a spec with `spec = "website.toml"`, or carry one inline as a `[spec]` +table: + +```toml +type = "website" +spec = "website.toml" +backend_image = { dockerfile = "Dockerfile" } +``` + +The env takes its `environment_name` from `site.name` and its health path from `site.health` when the env.toml leaves +them out. If the env.toml sets one to a different value, the env is refused. The env stores the spec's domains +(`WebsiteEnv.domains`). A spec with a `live` or `replay` source is refused, because a server image can't serve it. +`backend_image` still says how the image is built. + +### Conformance + +`python -m agentenv_protocol.website.conformance --spec website.toml` runs the usual checks plus two more: + +- `spec` checks that the card's name, domains, health path, routes, session cookie and session path are the spec's, + and its `clock` too when the spec declares it. +- `spec_routes` checks that every declared route is served. An `OPTIONS` to the route's path, with each path parameter + filled in by a value the converter of the card's route of that name takes (a UUID for `uuid`, else `0`), must find + something there; a GET route without path parameters must answer neither 404 nor 405; and each + route must be an operation of the server's OpenAPI document when its card offers one. When the card offers no + document, a route only the `OPTIONS` can check (a non-GET route, or a GET route with path parameters) counts as + served only when that `OPTIONS` answers below 500 with an `Allow` listing its method. Any other answer, such as a + catch-all `OPTIONS` handler's 200 without `Allow`, a 500, or an `Allow` that doesn't list the method, leaves the + route unverified: the check is then a `warning` (a failure with `--strict`) rather than a pass, and fails only + when a route is shown missing. Only GET routes are called directly, so the check changes no state. + +The `health` check reads readiness from the card's `website/v1` `ready`. For a card without it, it reads the health path, which is deprecated: with `--spec`, the spec's rather than the card's. + +## Overlays + +`extends = ""` merges this spec over the one it names. The merge works like this: + +- Tables merge key by key, recursively. +- Arrays and scalars replace. For example, `domains = [...]` or `personas = [...]` in the overlay replace the base's. +- `[source]` replaces as a whole, so an overlay can turn a live site into a replay without leftover keys. +- `name = false` removes an inherited entry of `[routes.ops]`, `[roles]` or `[data.collections]`. A route removed + this way is removed from the operations of the `openapi` document too. +- A route written as `"METHOD /path"` is expanded to a table before merging, so an overlay can change one key of it, + such as `post_message = { served_by = "..." }`. An operation of the `openapi` document is one too: a table naming it, + such as `add_item = { roles = ["staff"] }`, is written over the document's operation. +- `[[uses]]` entries are added. An entry with the same `as` (or, without one, the same `spec`) replaces the inherited + entry. +- The base's relative paths are rewritten relative to the overlay's folder, so a base in another folder still finds + its files. + +`extends` chains are followed, and a cycle is refused. In code, `spec.overlay({...})` applies the same merge to a +loaded spec. The Notes test uses it to swap in a stand-in static folder. + +Overlays are how one site gets variants: + +- A frozen capture with one synthetic route: `examples/website/specs/slack.toml` extends `slack-live.toml`, replaces + the live source with a replay of a capture, and swaps `post_message` to an SDK class. +- The same site with another persona set: `examples/website/specs/intranet-ben.toml` signs in as `ben`, with personas + `ben` and `cy`, and drops the `editor` role. + +```toml +# slack-live.toml +[site] +name = "slack" +domains = ["slack.com", "*.slack.com"] +health = "/api/api.test" + +[source.live] +allow_hosts = ["*.slack-edge.com", "*.slack-imgs.com"] +credentials = { token = "secret:SLACK_USER_TOKEN" } + +[users] +sign_in = "live" + +[routes.ops] +list_conversations = "GET /api/conversations.list" +history = "GET /api/conversations.history" +post_message = "POST /api/chat.postMessage" +add_reaction = "POST /api/reactions.add" + +[clock] +follow = "none" +``` + +```toml +# slack.toml: the live site frozen as a replay, one route answered by a synthetic implementation +extends = "slack-live.toml" + +[source.replay] +capture = "captures/slack-workspace.har" +allow_hosts = ["*.slack-edge.com"] + +[routes.ops] +post_message = { served_by = "slack_synthetic:SlackChat" } + +[clock] +follow = "date_patch" +``` + +## Composition + +A `[[uses]]` entry mounts another spec as a component: + +```toml +[[uses]] +spec = "components/sso.toml" # relative to this spec +mount = "/sso" # the path prefix its routes are served under +as = "sso" # its namespace; the component's site.name by default +data = { accounts = "users" } # a collection of the component's, mapped onto one of this site's +``` + +When a spec is resolved, each component is resolved first, including its own `extends` and `uses`. It is then merged +into the site: + +- **Routes.** Each of the component's routes is served at `` and named `.`. The roles its own + `[roles]` grants come with it. The site's `[roles]` can name the composed routes, for example + `editor = ["sso.login"]`. +- **Data.** A collection named in `data` merges into the site's collection of that name. They must agree on `key`, + `owner` and `schema` where both set them. Any other collection is added as `.`. A schema file a + component names stays the component's file, its path made relative to the site's spec. A component's seed files + are not merged: the site seeds the shared collections. +- **Users.** The component's personas are added to the site's, merging the fields of a persona they share; fields + that contradict are a conflict. A component must sign in the way the site does, or not at all (`none`). +- **The rest.** The component's `site` (domains, health), `source` and `clock` are not merged. A mounted + component lives under the site's own domains and is served by the site's source. + +These are conflicts, each reported as `uses[]: ...`: + +- A composed route name that the site already declares. +- A composed verb and path that another route already serves. +- A route of the site's own under the component's mount prefix (unless the mount is `/`). +- A `data` mapping of a collection the component doesn't declare. +- A collection or persona that disagrees with the site's. +- A sign-in mismatch. +- A component with no name. +- A cycle of `uses`. + +A resolved spec has no `extends`, `uses` or `routes.openapi` left: the document's operations are in `routes.ops`. +`spec.components` lists what was mounted, each with its name, mount and own resolved spec. `to_dict()` of a resolved +spec reads back as the same spec. + +`examples/website/specs/intranet.toml` uses a fake SSO component, `components/sso.toml`, mounted at `/sso`. The +component's `accounts` collection is mapped onto the intranet's `users`, and its `grants` collection becomes +`sso.grants`: + +```toml +[site] +name = "intranet" +domains = ["intranet.test", "*.intranet.test"] + +[source.image] +dockerfile = "Dockerfile.intranet" + +[data.collections.users] + +[data.collections.pages] +owner = "author" + +[users] +personas = ["ann", "ben"] +acting_user = "ann" + +[roles] +editor = ["publish_page"] + +[routes.ops] +list_pages = "GET /api/pages" +publish_page = "POST /api/pages" + +[[uses]] +spec = "components/sso.toml" +mount = "/sso" +data = { accounts = "users" } +``` + +Resolved, its routes are `list_pages`, `publish_page`, `sso.authorize` (`GET /sso/authorize`), `sso.login`, +`sso.token` and `sso.userinfo`, and its collections are `users`, `pages` and `sso.grants`. + +## The name table + +A multi env routes every host name its sites answer for through one table. `name_table(specs)` builds it: + +- **Kinds.** A site's `domains` are `synthetic` when the env serves the site (an image, sdk, static or replay source) + and `live` when the site is the real one. A live or replay source's `allow_hosts` are `allowlist`: hosts let through + to the internet, not served. +- **Precedence.** When two kinds claim one pattern, the higher kind wins: synthetic, then live, then allowlist. The + loser is kept in `shadowed` beside the entry it lost to. This is how a synthetic overlay takes over a name a live + site or an allowlist also claims. +- **Resolution.** `table.resolve(host)` looks first at the highest kind of the entries matching the host, then at the + most specific pattern of that kind: an exact name, then the longest wildcard. So a synthetic `*.slack.com` beats a + live `status.slack.com`. A synthetic `inbox.example.test` beats a synthetic `*.example.test` of another site. +- **Wildcards.** `*.example.test` matches every subdomain at any depth, but not `example.test` itself. + +These are conflicts, each a `SpecError`: + +- Two sites claim the same pattern at the same kind, synthetic or live, whichever kind wins the pattern. A host that + two different patterns match (`*.example.test` and `inbox.example.test`) is not a conflict, because specificity + decides it. +- A site has no `site.name`. +- Two sites share a name. + +Two sites allowlisting the same host is not a conflict, because an allowlist only lets traffic through. + +## What is wired and what is only modeled + +These parts are wired: + +- The spec model, loader, validation, `extends`, `uses`, the card params, `from_card`, `from_environment`, the name + table, the JSON Schema and the command line. +- `@website(spec=...)` and the SDK card generated through the spec. +- `spec` in a website env's env.toml (name, health and domains). +- `conformance --spec`. + +These parts are only modeled: they are declared and validated, but nothing serves them yet. + +- The `static`, `live` and `replay` sources, and route `served_by`. +- `[data] models` and collection schemas. +- Mounting a component's implementation. An SDK class that uses a component must serve its routes itself. +- The name table's use by the gateway. +- Component references other than file paths. +- The env's stored domains: deploy doesn't yet check them against the card. diff --git a/packages/agentenv-protocol/examples/website/notes/.gitignore b/packages/agentenv-protocol/examples/website/notes/.gitignore new file mode 100644 index 000000000..545a6e274 --- /dev/null +++ b/packages/agentenv-protocol/examples/website/notes/.gitignore @@ -0,0 +1,2 @@ +web/node_modules/ +web/dist/ diff --git a/packages/agentenv-protocol/examples/website/notes/Dockerfile b/packages/agentenv-protocol/examples/website/notes/Dockerfile new file mode 100644 index 000000000..f8896a549 --- /dev/null +++ b/packages/agentenv-protocol/examples/website/notes/Dockerfile @@ -0,0 +1,26 @@ +ARG NOTES=packages/agentenv-protocol/examples/website/notes + +FROM mirror.gcr.io/library/node:22-slim@sha256:4b150b48b4116e2a22f2c39cc27025fbf4f153e2078fd3b14d159fdb5c7279da AS web +ARG NOTES +WORKDIR /web +COPY ${NOTES}/web/package.json ${NOTES}/web/package-lock.json ./ +RUN npm ci --no-audit --no-fund +COPY ${NOTES}/web/ ./ +RUN npm run build + +FROM mirror.gcr.io/library/python:3.12-slim@sha256:423ed6ab25b1921a477529254bfeeabf5855151dc2c3141699a1bfc852199fbf +ARG NOTES +WORKDIR /app +RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/* +COPY packages/agentenv-protocol /tmp/agentenv-protocol +RUN pip install --no-cache-dir /tmp/agentenv-protocol "uvicorn>=0.30" && rm -rf /tmp/agentenv-protocol +COPY ${NOTES}/site.py ${NOTES}/website.toml /app/ +COPY ${NOTES}/data /app/data +COPY --from=web /web/dist /app/web/dist +ENV PORT=8000 +EXPOSE 8000 +RUN useradd --create-home --uid 10001 appuser +USER appuser +HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \ + CMD python -c "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:%s/healthz' % os.environ.get('PORT', '8000'), timeout=2)" +CMD ["python", "site.py"] diff --git a/packages/agentenv-protocol/examples/website/notes/Dockerfile.dockerignore b/packages/agentenv-protocol/examples/website/notes/Dockerfile.dockerignore new file mode 100644 index 000000000..10ae9270c --- /dev/null +++ b/packages/agentenv-protocol/examples/website/notes/Dockerfile.dockerignore @@ -0,0 +1,6 @@ +**/node_modules +**/dist +**/__pycache__ +**/.pytest_cache +**/.venv +.git diff --git a/packages/agentenv-protocol/examples/website/notes/README.md b/packages/agentenv-protocol/examples/website/notes/README.md new file mode 100644 index 000000000..6bd558790 --- /dev/null +++ b/packages/agentenv-protocol/examples/website/notes/README.md @@ -0,0 +1,100 @@ +# Notes: a reference website/v1 site + +Notes is a small notes app built on `agentenv_protocol.website.WebsiteEnvironment`. It is the reference site for the +`urn:agentenv:website/v1` server contract. Each user keeps their own notes and can create, edit, mark done and delete +them. The page shows the universe's time. + +| File | What it is | +| --- | --- | +| `website.toml` | The site's spec: its name, domains, health path, source, seed, collections, personas, roles, routes, state read and clock (see `docs/website-spec.md`). | +| `site.py` | The server: the `Notes` WebsiteEnvironment, declared from `website.toml`, its `@api` routes, and `serve()` on `PORT` (8000). | +| `data/seed.json` | The seed data: two users (`ann`, `ben`), their notes, and `current_user`, the acting user. | +| `web/` | A minimal Vite app (`index.html`, `src/main.js`, `src/style.css`). Its assets (`base: "./"`) and API calls are relative to the page, so it works at `/` and under the gateway's `/website/notes/` alike; opened at a path without its final slash, such as `/website/notes`, the page first reloads itself with it. | +| `Dockerfile` | The server image: Node builds `web/`, then Python installs this repo's `packages/agentenv-protocol` with uvicorn. It installs `curl`, which the gateway's compose health check runs. | +| `env.toml` | The website env as a bundle entry, pointing at `website.toml`. | + +## What it exercises + +- **Site declaration.** `@website(spec="website.toml")`. The spec gives the name, domains, static folder, health path + and seed, and the class is refused when its `@api` routes aren't the spec's. The card's website/v1 params are the + spec's `to_card_extension()`: the domains, the health path, every route by name, the session, the state read and + the clock. +- **Routes.** `list_users`, `sign_in`, `list_notes` (with an optional `?done=`), `create_note` (201), `get_note`, + `edit_note`, `toggle_note` and `delete_note`. Errors are raised as `NotFound`, `BadRequest` and `Unauthorized`. +- **Roles.** `all_notes` (`GET /api/admin/notes`) is declared with `roles=["admin"]`, and the spec's `[roles]` grants + it to `admin`. It answers only a caller that + sends `AgentEnv-Role: admin` with `AgentEnv-Role-Key` equal to the container's `WEBSITE_ROLE_KEY`; any other caller + gets 403. A deploy with `keyed_roles` gives the site its own role key as `WEBSITE_ROLE_KEY` and its gateway sends it + on only beside a role it verified with the same key, so a page cannot claim `admin` through it. Any other deploy sets + no key, and its gateway passes on whatever role a browser sends; with no `WEBSITE_ROLE_KEY` set, every caller is + `default` and the route is closed. +- **Users.** `ApiContext.user` is the user in the signed session cookie, `agentenv_session_notes` (the site's own, so + it doesn't collide with another site's under the same gateway host; the card names it). With no cookie, it is the acting user (the + store's `current_user`), and `GET /api/session` signs that user in. `POST /api/sign-in` switches to another seeded + user. A user's notes are invisible to every other user, and another user's note answers 404. +- **State.** The site writes no data hooks, so it uses the default store. `data/add` loads the seed (as a data part, or + as a file part by bytes or `file://`), `data/get` exports it, and `data/reset` empties it. `data/get` is how + agentenv reads what a task left the site with. +- **Time.** Notes are stamped with `self.now()`, which follows the gateway's clock once `clock/v1`'s `sync_time` names + it. The frontend reads `X-Universe-Now` from every API response into `universeNow()` and shows it in the header. + It uses this helper rather than `clock_patch=True`, so the page works the same against a server written without the + SDK. +- **Readiness.** `@ready_check seeded` holds `/healthz` at 503 until the store has users, for example after a + `data/reset` and before the next `data/add`. + +## Run it + +From this folder: + +```bash +cd web && npm ci && npm run build && cd .. +PYTHONPATH=/packages/agentenv-protocol/src python site.py +``` + +Open http://localhost:8000. To work on the frontend, run `npm run dev` in `web/` while `site.py` runs; Vite proxies +`/api` to port 8000. + +## Test it + +```bash +python -m pytest packages/agentenv-protocol/tests/test_website_notes_example.py +``` + +The test imports `site.py`, runs the conformance checker, and checks sign-in, per-user notes, the admin-only route, +clock-synced timestamps, and the data plane's reset and seed load. It serves a stand-in page, so it needs no Vite +build. + +## Build and verify the image + +The build context is the repo root, because the image installs the protocol package from this repo's source: + +```bash +docker build -f packages/agentenv-protocol/examples/website/notes/Dockerfile -t notes-site . +docker run -d --name notes-site -p 18080:8000 notes-site +python -m agentenv_protocol.website.conformance http://localhost:18080 --clock-host host.docker.internal --strict +curl -s http://localhost:18080/api/notes +``` + +Use `--clock-host` so the container can reach the checker's clock, and add +`--spec packages/agentenv-protocol/examples/website/notes/website.toml` to check the server against its spec too. +`--strict` fails on a recommended check's warning too (the card, `X-Universe-Now`, the clock, forwarded headers, +cookies); without it only the contract's required checks, and the spec's when given, fail the run, and Notes passes +every check either way. The clock check leaves the server following a +clock that is gone once the check ends: later timestamps start from 2031 until the server is synced again, so restart +the container before using it. + +## Register it + +```bash +agent-env env website put --id notes-site \ + --server-dockerfile packages/agentenv-protocol/examples/website/notes/Dockerfile \ + --server-docker-context . --health-path /healthz +``` + +Run it from the repo root. `environment_name` comes from the card, which the spec names `notes`. The stored env +has a server image and no frontend image. Pass `--skip-validation` to register without deploying. + +`env.toml` describes the same env for a bundle. Its keys are `type`, `spec` and `backend_image`; the env takes its +`environment_name`, health path and domains from the spec. A bundle builds `backend_image` with this +folder as the build context, but this Dockerfile needs the repo root. Until the protocol package is installed from a +published release instead, register the site with `put` as above. diff --git a/packages/agentenv-protocol/examples/website/notes/data/seed.json b/packages/agentenv-protocol/examples/website/notes/data/seed.json new file mode 100644 index 000000000..0a52ab7be --- /dev/null +++ b/packages/agentenv-protocol/examples/website/notes/data/seed.json @@ -0,0 +1,15 @@ +{ + "current_user": "ann", + "users": [ + {"id": "ann", "name": "Ann Lee"}, + {"id": "ben", "name": "Ben Ortiz"} + ], + "notes": [ + {"id": 1, "owner": "ann", "title": "Buy milk", "body": "Oat, two cartons.", "done": false, + "created_at": "2026-01-05T09:00:00.000Z", "updated_at": "2026-01-05T09:00:00.000Z"}, + {"id": 2, "owner": "ann", "title": "Book dentist", "body": "", "done": true, + "created_at": "2026-01-04T15:30:00.000Z", "updated_at": "2026-01-06T08:10:00.000Z"}, + {"id": 3, "owner": "ben", "title": "Draft the quarterly report", "body": "Due Friday.", "done": false, + "created_at": "2026-01-03T11:00:00.000Z", "updated_at": "2026-01-03T11:00:00.000Z"} + ] +} diff --git a/packages/agentenv-protocol/examples/website/notes/env.toml b/packages/agentenv-protocol/examples/website/notes/env.toml new file mode 100644 index 000000000..483d79d77 --- /dev/null +++ b/packages/agentenv-protocol/examples/website/notes/env.toml @@ -0,0 +1,3 @@ +type = "website" +spec = "website.toml" +backend_image = { dockerfile = "Dockerfile" } diff --git a/packages/agentenv-protocol/examples/website/notes/site.py b/packages/agentenv-protocol/examples/website/notes/site.py new file mode 100644 index 000000000..777c83c40 --- /dev/null +++ b/packages/agentenv-protocol/examples/website/notes/site.py @@ -0,0 +1,129 @@ +"""Notes: the reference site of ``urn:agentenv:website/v1``, a small notes app on WebsiteEnvironment. Each user keeps +notes they create, edit, mark done and delete; a browser with no session is signed in as the acting user, the seed's +``current_user``, and can sign in as another seeded user. The site keeps its state in the default store, which the data +plane loads, exports and resets, and stamps each note with the universe's time. ``admin`` callers, as the gateway's +``AgentEnv-Role`` names them, can list every user's notes. + +Run it with ``python site.py`` after building ``web/`` into ``web/dist``; it serves on ``PORT`` (8000).""" +from __future__ import annotations + +from typing import Optional + +from pydantic import BaseModel + +from agentenv_protocol.website import ( + ApiContext, + BadRequest, + NotFound, + Unauthorized, + WebsiteEnvironment, + api, + ready_check, + website, +) +from agentenv_protocol.website.clock import iso + +NOTES = "notes" +USERS = "users" + + +class NewNote(BaseModel): + title: str + body: str = "" + + +class NoteEdit(BaseModel): + title: Optional[str] = None + body: Optional[str] = None + done: Optional[bool] = None + + +class SignIn(BaseModel): + user: str + + +@website(spec="website.toml") +class Notes(WebsiteEnvironment): + @ready_check + def seeded(self) -> bool: + """The site is usable once its store knows who may sign in.""" + return bool(self.store[USERS]) + + @api.get("/api/users", name="list_users") + def list_users(self) -> list: + """The users a browser can sign in as.""" + return self.store[USERS] + + @api.post("/api/sign-in", name="sign_in") + def sign_in(self, body: SignIn, ctx: ApiContext) -> dict: + """Sign the browser in as one of the site's users.""" + if self.store.get(USERS, body.user) is None: + raise NotFound(f"no user {body.user!r}") + ctx.sign_in(body.user) + return {"user": ctx.user} + + @api.get("/api/notes", name="list_notes") + def list_notes(self, ctx: ApiContext, done: Optional[bool] = None) -> list: + """The signed-in user's notes, newest first; with ``done``, only those done or not.""" + match = {"owner": self._user(ctx)} if done is None else {"owner": self._user(ctx), "done": done} + return sorted(self.store.find(NOTES, **match), key=lambda note: note["created_at"], reverse=True) + + @api.post("/api/notes", name="create_note", status=201) + def create_note(self, note: NewNote, ctx: ApiContext) -> dict: + """Add a note for the signed-in user, not done, created and updated now.""" + now = iso(self.now()) + return self.store.insert(NOTES, {"owner": self._user(ctx), "title": _title(note.title), "body": note.body, + "done": False, "created_at": now, "updated_at": now}) + + @api.get("/api/notes/{note_id:int}", name="get_note") + def get_note(self, note_id: int, ctx: ApiContext) -> dict: + return self._note(note_id, ctx) + + @api.patch("/api/notes/{note_id:int}", name="edit_note") + def edit_note(self, note_id: int, edit: NoteEdit, ctx: ApiContext) -> dict: + """Change a note's title, body or done, and its ``updated_at``.""" + note = self._note(note_id, ctx) + fields = edit.model_dump(exclude_none=True) + if "title" in fields: + fields["title"] = _title(fields["title"]) + note.update(fields, updated_at=iso(self.now())) + return note + + @api.post("/api/notes/{note_id:int}/toggle", name="toggle_note") + def toggle_note(self, note_id: int, ctx: ApiContext) -> dict: + """Mark a note done, or not done when it was.""" + note = self._note(note_id, ctx) + note.update(done=not note["done"], updated_at=iso(self.now())) + return note + + @api.delete("/api/notes/{note_id:int}", name="delete_note") + def delete_note(self, note_id: int, ctx: ApiContext) -> dict: + self.store.delete(NOTES, self._note(note_id, ctx)["id"]) + return {"deleted": note_id} + + @api.get("/api/admin/notes", name="all_notes", roles=["admin"]) + def all_notes(self) -> list: + """Every user's notes, for an ``admin`` caller only.""" + return self.store[NOTES] + + def _user(self, ctx: ApiContext) -> str: + if ctx.user is None: + raise Unauthorized("sign in first") + return ctx.user + + def _note(self, note_id: int, ctx: ApiContext) -> dict: + """The signed-in user's note ``note_id``; another user's is as missing as one that doesn't exist.""" + note = self.store.get(NOTES, note_id) + if note is None or note["owner"] != self._user(ctx): + raise NotFound(f"no note {note_id}") + return note + + +def _title(title: str) -> str: + if not title.strip(): + raise BadRequest("a note needs a title", code="empty_title") + return title.strip() + + +if __name__ == "__main__": + Notes().serve() diff --git a/packages/agentenv-protocol/examples/website/notes/web/index.html b/packages/agentenv-protocol/examples/website/notes/web/index.html new file mode 100644 index 000000000..c714b7a6c --- /dev/null +++ b/packages/agentenv-protocol/examples/website/notes/web/index.html @@ -0,0 +1,37 @@ + + + + + + Notes + + + +
+

Notes

+
+ + +
+
+
+
+ + + +
+ + +
    +
    + + + diff --git a/packages/agentenv-protocol/examples/website/notes/web/package-lock.json b/packages/agentenv-protocol/examples/website/notes/web/package-lock.json new file mode 100644 index 000000000..938f9c2cd --- /dev/null +++ b/packages/agentenv-protocol/examples/website/notes/web/package-lock.json @@ -0,0 +1,1020 @@ +{ + "name": "notes-site-web", + "version": "0.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "notes-site-web", + "version": "0.0.0", + "devDependencies": { + "vite": "5.4.10" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.21.5.tgz", + "integrity": "sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.21.5.tgz", + "integrity": "sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.21.5.tgz", + "integrity": "sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.21.5.tgz", + "integrity": "sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.21.5.tgz", + "integrity": "sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.21.5.tgz", + "integrity": "sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.21.5.tgz", + "integrity": "sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.21.5.tgz", + "integrity": "sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.21.5.tgz", + "integrity": "sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.21.5.tgz", + "integrity": "sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.21.5.tgz", + "integrity": "sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.21.5.tgz", + "integrity": "sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.21.5.tgz", + "integrity": "sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.21.5.tgz", + "integrity": "sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.21.5.tgz", + "integrity": "sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.21.5.tgz", + "integrity": "sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.21.5.tgz", + "integrity": "sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.21.5.tgz", + "integrity": "sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.21.5.tgz", + "integrity": "sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.21.5.tgz", + "integrity": "sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.21.5.tgz", + "integrity": "sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.21.5.tgz", + "integrity": "sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.21.5.tgz", + "integrity": "sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@napi-rs/lzma-linux-x64-gnu": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/@napi-rs/lzma-linux-x64-gnu/-/lzma-linux-x64-gnu-1.5.1.tgz", + "integrity": "sha512-oTXEIha4SsuXdTA4Iyskj0kpdx2yVXdhd75c2v3xGrHFfVMsbhTPZU/nMPL4sWKo4pBHm3aucLaqGlF696dTyQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^22.20 || ^24.12 || >=25" + } + }, + "node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.67.2.tgz", + "integrity": "sha512-qB7Gzl1QI45zP1zXgqLKzjFmOBGQw2s2MLtdw0eB4xdFGPNE8LhmOnItFgyYJXXNSW8Go9c2ovvXrUUzw5sAMQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-arm64": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.67.2.tgz", + "integrity": "sha512-8+hXNO1J+GsYCGHwyJuBlnhyozivEvIEm/P6NkJeuoVUN+60TDvYYXk38UTm5bKRTLhIpqEQYx4d9CeN6kjh0Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-x64": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-x64/-/rollup-android-x64-4.67.2.tgz", + "integrity": "sha512-/uEZY+tcWDE8DXa/rajU3Bwv6IfvQavvzjmbkn39GgQenT+UYtAxfEMdB7VKr74y8aonncm0eQa1EbiNfg+Cjg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.67.2.tgz", + "integrity": "sha512-3tCLEgakEvb5M3/KyYdhPjoteyWYziMTeznoXfcs/31eeFu+Vgs0VNRlb6Zc+YC07/sLa84OwTDF5zHOMFk92Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-darwin-x64": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.67.2.tgz", + "integrity": "sha512-z31ODzkdkX7OOJCssvpz++hhtkQKWKw2L0eO2HAxFP7RMSXXedzkrTjesqCUxs4XLckRShjNfxsNxjGY5DQCGg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-freebsd-arm64": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.67.2.tgz", + "integrity": "sha512-CTvrb0KTAQqcrpcj7rg8FgAI0+jRWnCl+En6exqXDZztCo7vspxlRUX8z9Hp/J/ja2XRnQ1JDbexkBtRyD9jSw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-freebsd-x64": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.67.2.tgz", + "integrity": "sha512-SJdblZpBkbOh1e4JwMM5pJJ2PdVmD3WWbMTXvncMSRLA3DdZuwU+P93RaSChOAXN4Mu53tBoFp1ADPzi3dIVAA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.67.2.tgz", + "integrity": "sha512-pUBEd7uI9dAu3TpbPcvi9eU2aJslsjIomormeTaQHmzWNhyqNx3mmq+ekC6blGNZu8F3uTHngTibKp7tbblF0A==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm-musleabihf": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.67.2.tgz", + "integrity": "sha512-wW6ARw2Ah0/teOBTgTeSJIpVdPKaxQmEgZ/irkUfr3F9sgimqpApCv/NbQzjm0Nfi5EFilHh1H8kz01uQLaq5Q==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.67.2.tgz", + "integrity": "sha512-xcFYMHNL42fRu2MFipvxznr+zd+DRqwL98bhNvDWbMPRcFtMYf34NAz4xVOlnEhzgpNRE5htSemcqAsAxf2eQg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.67.2.tgz", + "integrity": "sha512-U9HrnJ9PFXqA//P/oEGaXBIJraEqzYMS9yuQOdOcPCvHZIC1FFOlnd8ocAaeDDPDpY8xOrhfxMRtr6KqKi3gyg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-gnu": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.67.2.tgz", + "integrity": "sha512-2hZN8CFGfsbDw+DNtImkPRitpLNjVM2XEbi4ZQ5igLxMFD4gI/P6Zo9czGxuvQXtRyZtZyZp8Kr4ERsAipplBw==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-musl": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.67.2.tgz", + "integrity": "sha512-HZO7UfWEsReAnkdVByyQaFsNXJCtvdfun/LFb2Xd4qnhgF1ZfhikpbgqrTRtNMmB5OgNnSwFLiPobOT8oNT9iw==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-gnu": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.67.2.tgz", + "integrity": "sha512-AdYIihuLkJ2TScy9BpSPiDex3zOMjkNJLsFlk9NeJ0U3WEtOx4W8z0nOkPEhDS2S4gwsRWBa6ZZexJ9e6OxyzQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-musl": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.67.2.tgz", + "integrity": "sha512-iZE0NXcAGqLoCWeHRPkBxJe93JT4Q981z1ZqR+y7qAiIksdRlWhw8hmqHFo1B4ZnU66He31NUb52Iyv+vwMEBQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.67.2.tgz", + "integrity": "sha512-vC7EAtJpSqSrrbg8huMfgxtwlR8fUsFCe5rStDk1WA8POky/6EEp52dxENMmqLKjSfdM3TT1WWPcZ01hiYZcdw==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-musl": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.67.2.tgz", + "integrity": "sha512-DdqG6x72MHpQOlkK/cqrru9b2QMJHLkNojCB3bteB5fmkE/7XEAkReJMANp8wgef8eiEC99SvdP/BH7Ho/BpLA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-s390x-gnu": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.67.2.tgz", + "integrity": "sha512-F10mjdnRmGg9K+LGdqQVeUYFr08f3fVBOh7YNE5p1KIpTzhtKKOLqcQEsqZG6yEM2eouXaMo2AZzeDAcseVTrQ==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.67.2.tgz", + "integrity": "sha512-ChzK9f3iE4N5GInBquiZtzWOqh+CYS5nlLlXGJxeG3d2T9rk8L1Hxg0s+v2Eh6vmAc/1T6IhOVkQU0ORP2eV7g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-musl": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.67.2.tgz", + "integrity": "sha512-95WwzC+KH9hPHtbgwvwwW6MkKME6vMAdAQLS2diCvoi/iUiYdC0Y1RIPYVESx0iWqkN5ZaZq6qD1BflLcyTMEQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-openbsd-x64": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.67.2.tgz", + "integrity": "sha512-eKbpwlolQmDKd/dbFjvqB1+2qdYQLrgxi3Mtbijp7/3yBBvO9OoHtQ3ecYIaCdW46ijSjy5/yydyrrxHI416/g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ] + }, + "node_modules/@rollup/rollup-openharmony-arm64": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.67.2.tgz", + "integrity": "sha512-is9lnlv6khapl+H9Hjp0gIVsUVtDxQga2nwMHpQbhDXMCMGcNmlel4/q9ihf/3FvvbL4GdGOPOGvWoei2RCggg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ] + }, + "node_modules/@rollup/rollup-win32-arm64-msvc": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.67.2.tgz", + "integrity": "sha512-yTq+7YjsgbBcMdlk7Wq+VoJilGQtdtRc4PyTvN7vhO958atCO2nQeIjCF/YANSD/Z/fnrCr27fYsGgC6PFTndQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-ia32-msvc": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.67.2.tgz", + "integrity": "sha512-nAvZib4Xz1zx8OKhtRuyCsSmCmbfegv1AGbfpG+CowQKxCBW1pLlfYWjcr1+GI3BzRKQbBl7l9640GwWAwOF7g==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-gnu": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.67.2.tgz", + "integrity": "sha512-R3CVbEbNa1OXjpldD1tOLarb5rB08xjQiRMpe7PZKtT8eHCZhrV4ClSPfLLbSgRMM1RfLtmAfouBRf14f47lMA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-msvc": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.67.2.tgz", + "integrity": "sha512-DeuVFilvaGtoS9TFIbTIQ0Kt4CBqP3xnjJ8SFqWWbXlFbrpmCdivEW7Rm38trRA8rZzQQ8lhs73nEetWotJpTg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@types/estree": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.10.tgz", + "integrity": "sha512-lTyOMnOcmRBzdCv6/4zUZCrEhV0xrqQiUZBZtVcbmLHBXKLnN8q89v/wKRF/Q0HbwVsqbIG3o5HDoFl9sduM+g==", + "dev": true, + "license": "MIT" + }, + "node_modules/esbuild": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.21.5.tgz", + "integrity": "sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=12" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.21.5", + "@esbuild/android-arm": "0.21.5", + "@esbuild/android-arm64": "0.21.5", + "@esbuild/android-x64": "0.21.5", + "@esbuild/darwin-arm64": "0.21.5", + "@esbuild/darwin-x64": "0.21.5", + "@esbuild/freebsd-arm64": "0.21.5", + "@esbuild/freebsd-x64": "0.21.5", + "@esbuild/linux-arm": "0.21.5", + "@esbuild/linux-arm64": "0.21.5", + "@esbuild/linux-ia32": "0.21.5", + "@esbuild/linux-loong64": "0.21.5", + "@esbuild/linux-mips64el": "0.21.5", + "@esbuild/linux-ppc64": "0.21.5", + "@esbuild/linux-riscv64": "0.21.5", + "@esbuild/linux-s390x": "0.21.5", + "@esbuild/linux-x64": "0.21.5", + "@esbuild/netbsd-x64": "0.21.5", + "@esbuild/openbsd-x64": "0.21.5", + "@esbuild/sunos-x64": "0.21.5", + "@esbuild/win32-arm64": "0.21.5", + "@esbuild/win32-ia32": "0.21.5", + "@esbuild/win32-x64": "0.21.5" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/nanoid": { + "version": "3.3.20", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.20.tgz", + "integrity": "sha512-uKdg2G3GNCKQn9byYOpxbGqrT2fGO5KRt5J/8b3pok8rT6qxGWF6hxMyJiEYtAf+FVyYuD9hRaDqX5uPFYJ4ZQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/postcss": { + "version": "8.5.32", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.32.tgz", + "integrity": "sha512-LJlKCCAUFKOuZxipUdYw1NzIITEsPcgyOG9LkvRJvuKj/bRzbuRY0ARXvL0pWGpsZvn8jku+LMep2eU0seHp8g==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.20", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.2" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/rollup": { + "version": "4.67.2", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.67.2.tgz", + "integrity": "sha512-sFZk5BeRUQzkjmuClrJI41BowIP4yv/40d5gM9vtjxVLEt+8JG9FGlBps6cliTP2KhFiw22oqNGo0G9pVpPZlw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.10" + }, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "@napi-rs/lzma-linux-x64-gnu": "1.5.1", + "@rollup/rollup-android-arm-eabi": "4.67.2", + "@rollup/rollup-android-arm64": "4.67.2", + "@rollup/rollup-android-x64": "4.67.2", + "@rollup/rollup-darwin-arm64": "4.67.2", + "@rollup/rollup-darwin-x64": "4.67.2", + "@rollup/rollup-freebsd-arm64": "4.67.2", + "@rollup/rollup-freebsd-x64": "4.67.2", + "@rollup/rollup-linux-arm-gnueabihf": "4.67.2", + "@rollup/rollup-linux-arm-musleabihf": "4.67.2", + "@rollup/rollup-linux-arm64-gnu": "4.67.2", + "@rollup/rollup-linux-arm64-musl": "4.67.2", + "@rollup/rollup-linux-loong64-gnu": "4.67.2", + "@rollup/rollup-linux-loong64-musl": "4.67.2", + "@rollup/rollup-linux-ppc64-gnu": "4.67.2", + "@rollup/rollup-linux-ppc64-musl": "4.67.2", + "@rollup/rollup-linux-riscv64-gnu": "4.67.2", + "@rollup/rollup-linux-riscv64-musl": "4.67.2", + "@rollup/rollup-linux-s390x-gnu": "4.67.2", + "@rollup/rollup-linux-x64-gnu": "4.67.2", + "@rollup/rollup-linux-x64-musl": "4.67.2", + "@rollup/rollup-openbsd-x64": "4.67.2", + "@rollup/rollup-openharmony-arm64": "4.67.2", + "@rollup/rollup-win32-arm64-msvc": "4.67.2", + "@rollup/rollup-win32-ia32-msvc": "4.67.2", + "@rollup/rollup-win32-x64-gnu": "4.67.2", + "@rollup/rollup-win32-x64-msvc": "4.67.2", + "fsevents": "~2.3.2" + } + }, + "node_modules/source-map-js": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.2.tgz", + "integrity": "sha512-KGj/8Y43x35aZVDtt+J4mK1hoLGHULMYfSkODJNQjNDC3oW1PqPoxMwo0pLUsWM/UEGzON/NxeHywEfNXNP3Vw==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/vite": { + "version": "5.4.10", + "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.10.tgz", + "integrity": "sha512-1hvaPshuPUtxeQ0hsVH3Mud0ZanOLwVTneA1EgbAM5LhaZEqyPWGRQ7BtaMvUrTDeEaC8pxtj6a6jku3x4z6SQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "^0.21.3", + "postcss": "^8.4.43", + "rollup": "^4.20.0" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^18.0.0 || >=20.0.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^18.0.0 || >=20.0.0", + "less": "*", + "lightningcss": "^1.21.0", + "sass": "*", + "sass-embedded": "*", + "stylus": "*", + "sugarss": "*", + "terser": "^5.4.0" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + } + } + } + } +} diff --git a/packages/agentenv-protocol/examples/website/notes/web/package.json b/packages/agentenv-protocol/examples/website/notes/web/package.json new file mode 100644 index 000000000..c741095dd --- /dev/null +++ b/packages/agentenv-protocol/examples/website/notes/web/package.json @@ -0,0 +1,14 @@ +{ + "name": "notes-site-web", + "private": true, + "version": "0.0.0", + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "preview": "vite preview" + }, + "devDependencies": { + "vite": "5.4.10" + } +} diff --git a/packages/agentenv-protocol/examples/website/notes/web/src/main.js b/packages/agentenv-protocol/examples/website/notes/web/src/main.js new file mode 100644 index 000000000..61ead9d31 --- /dev/null +++ b/packages/agentenv-protocol/examples/website/notes/web/src/main.js @@ -0,0 +1,252 @@ +import "./style.css"; + +let offsetMs = 0; +let filter = "all"; +let renders = 0; +const saving = new Map(); +const lastSent = new Map(); +const landed = new Map(); +const drafts = new Map(); +const CLOCK_REFRESH_MS = 5000; + +export function universeNow() { + return new Date(Date.now() + offsetMs); +} + +function followUniverse(response) { + const at = Date.parse(response.headers.get("X-Universe-Now") || ""); + if (!Number.isNaN(at)) offsetMs = at - Date.now(); +} + +async function api(path, { method = "GET", body } = {}) { + const response = await fetch(path, { + method, + headers: body === undefined ? {} : { "Content-Type": "application/json" }, + body: body === undefined ? undefined : JSON.stringify(body), + }); + followUniverse(response); + const data = await response.json().catch(() => null); + if (!response.ok) throw new Error(data?.error || `${method} ${path}: ${response.status}`); + return data; +} + +const $ = (id) => document.getElementById(id); + +function setDraft(id, kind, value, saved) { + const draft = { ...drafts.get(id) }; + if (value === saved) delete draft[kind]; + else draft[kind] = value; + if (Object.keys(draft).length) drafts.set(id, draft); + else drafts.delete(id); +} + +function settleDraft(id, sent) { + const draft = drafts.get(id); + if (!draft) return; + for (const kind of Object.keys(sent)) if (draft[kind] === sent[kind]) delete draft[kind]; + if (!Object.keys(draft).length) drafts.delete(id); +} + +function queueSave(id, fields) { + lastSent.set(id, fields); + const queued = saving.get(id); + if (queued) { + queued.next = fields; + return; + } + const state = { next: fields }; + saving.set(id, state); + state.done = run(async () => { + try { + while (state.next) { + const next = state.next; + state.next = null; + await api(`api/notes/${id}`, { method: "PATCH", body: next }); + settleDraft(id, next); + } + landed.set(id, renders); + } catch (error) { + state.error = error; + lastSent.set(id, {}); + landed.delete(id); + throw error; + } finally { + saving.delete(id); + } + }); +} + +function showError(error) { + $("error").textContent = error ? error.message : ""; + $("error").hidden = !error; +} + +function formatTime(value) { + return new Date(value).toLocaleString(undefined, { dateStyle: "medium", timeStyle: "short", timeZone: "UTC" }); +} + +function tick() { + $("universe-now").textContent = `${formatTime(universeNow())} UTC`; + $("universe-now").dateTime = universeNow().toISOString(); +} + +function noteItem(note) { + const item = document.createElement("li"); + item.className = `note card${note.done ? " done" : ""}`; + item.dataset.id = note.id; + + const done = document.createElement("input"); + done.type = "checkbox"; + done.checked = note.done; + done.setAttribute("aria-label", `Mark "${note.title}" ${note.done ? "not done" : "done"}`); + done.addEventListener("change", () => run(() => api(`api/notes/${note.id}/toggle`, { method: "POST" }))); + + const draft = drafts.get(note.id) ?? {}; + const title = document.createElement("input"); + title.className = "title"; + title.defaultValue = note.title; + title.value = draft.title ?? note.title; + title.setAttribute("aria-label", "Title"); + + const body = document.createElement("textarea"); + body.className = "body"; + body.rows = 2; + body.defaultValue = note.body; + body.value = draft.body ?? note.body; + body.placeholder = "Details"; + body.setAttribute("aria-label", "Details"); + + const save = () => { + const sent = lastSent.get(note.id) ?? { title: note.title, body: note.body }; + if (title.value === sent.title && body.value === sent.body) return; + queueSave(note.id, { title: title.value, body: body.value }); + }; + for (const [kind, field] of [["title", title], ["body", body]]) { + field.addEventListener("input", () => setDraft(note.id, kind, field.value, note[kind])); + field.addEventListener("change", save); + field.addEventListener("blur", save); + } + + const meta = document.createElement("small"); + meta.textContent = `Created ${formatTime(note.created_at)} · updated ${formatTime(note.updated_at)}`; + + const remove = document.createElement("button"); + remove.className = "delete"; + remove.textContent = "Delete"; + remove.addEventListener("click", () => run(async () => { + await api(`api/notes/${note.id}`, { method: "DELETE" }); + drafts.delete(note.id); + })); + + const text = document.createElement("div"); + text.className = "text"; + text.append(title, body, meta); + item.append(done, text, remove); + return item; +} + +function focused() { + const field = document.activeElement; + if (!field?.matches?.("#notes .title, #notes .body")) return null; + return { id: field.closest("li").dataset.id, kind: field.className, start: field.selectionStart, + end: field.selectionEnd }; +} + +function refocus(kept) { + if (!kept) return; + const field = document.querySelector(`#notes li[data-id="${kept.id}"] .${kept.kind}`); + if (!field) return; + field.focus(); + field.setSelectionRange(kept.start, kept.end); +} + +async function render() { + const query = filter === "all" ? "" : `?done=${filter === "done"}`; + const request = ++renders; + const notes = await api(`api/notes${query}`); + if (request !== renders) return; + for (const [id, after] of landed) { + if (request > after && !saving.has(id)) { + lastSent.delete(id); + landed.delete(id); + } + } + const kept = focused(); + $("notes").replaceChildren(...notes.map(noteItem)); + refocus(kept); + if (!notes.length) { + const empty = document.createElement("li"); + empty.className = "empty"; + empty.textContent = "No notes here yet."; + $("notes").append(empty); + } +} + +async function run(action) { + try { + await action(); + showError(null); + await render(); + } catch (error) { + showError(error); + } +} + +async function start() { + const [session, users] = await Promise.all([api("api/session"), api("api/users")]); + $("user").replaceChildren(...users.map((user) => new Option(user.name || user.id, user.id))); + let signedInAs = session.user ?? ""; + $("user").value = signedInAs; + $("user").addEventListener("change", (event) => { + const user = event.target.value; + $("user").disabled = true; + run(async () => { + try { + while (saving.size) { + const pending = [...saving.values()]; + await Promise.all(pending.map((state) => state.done)); + const failed = pending.find((state) => state.error); + if (failed) throw failed.error; + } + await api("api/sign-in", { method: "POST", body: { user } }); + signedInAs = user; + drafts.clear(); + } finally { + $("user").value = signedInAs; + $("user").disabled = false; + } + }); + }); + + const add = $("new-note").querySelector("button[type=submit]"); + $("new-note").addEventListener("submit", (event) => { + event.preventDefault(); + if (add.disabled) return; + const body = { title: $("new-title").value, body: $("new-body").value }; + add.disabled = true; + run(async () => { + try { + await api("api/notes", { method: "POST", body }); + } finally { + add.disabled = false; + } + if ($("new-title").value === body.title) $("new-title").value = ""; + if ($("new-body").value === body.body) $("new-body").value = ""; + }); + }); + + for (const button of document.querySelectorAll("[data-filter]")) { + button.addEventListener("click", () => { + filter = button.dataset.filter; + document.querySelectorAll("[data-filter]").forEach((b) => b.classList.toggle("active", b === button)); + run(async () => {}); + }); + } + + tick(); + setInterval(tick, 1000); + setInterval(() => fetch("healthz").then(followUniverse).catch(() => {}), CLOCK_REFRESH_MS); + await render(); +} + +start().catch(showError); diff --git a/packages/agentenv-protocol/examples/website/notes/web/src/style.css b/packages/agentenv-protocol/examples/website/notes/web/src/style.css new file mode 100644 index 000000000..8ecd48b6a --- /dev/null +++ b/packages/agentenv-protocol/examples/website/notes/web/src/style.css @@ -0,0 +1,57 @@ +:root { + --bg: #f6f6f3; + --card: #ffffff; + --text: #1d1d1b; + --muted: #6b6b66; + --line: #deded8; + --accent: #2f5bd3; + font-family: system-ui, -apple-system, "Segoe UI", sans-serif; + color: var(--text); + background: var(--bg); +} + +* { box-sizing: border-box; } +body { margin: 0; } + +.bar { + display: flex; + flex-wrap: wrap; + gap: 12px; + align-items: center; + justify-content: space-between; + padding: 12px 16px; + border-bottom: 1px solid var(--line); + background: var(--card); +} +.bar h1 { margin: 0; font-size: 1.25rem; } +.who { display: flex; flex-wrap: wrap; gap: 12px; align-items: center; color: var(--muted); font-size: 0.9rem; } +.who time { font-variant-numeric: tabular-nums; } + +main { max-width: 720px; margin: 0 auto; padding: 16px; } + +.card { background: var(--card); border: 1px solid var(--line); border-radius: 8px; padding: 12px; } + +input, textarea, select, button { font: inherit; color: inherit; } +input, textarea { width: 100%; border: 1px solid var(--line); border-radius: 6px; padding: 6px 8px; background: transparent; } +textarea { resize: vertical; } +button { border: 1px solid var(--line); border-radius: 6px; padding: 6px 12px; background: var(--card); cursor: pointer; } +button[type="submit"], .filters .active { background: var(--accent); border-color: var(--accent); color: #fff; } + +#new-note { display: grid; gap: 8px; } +#new-note button { justify-self: end; } + +.filters { display: flex; gap: 8px; margin: 16px 0; } + +#error { color: #b3261e; margin: 0 0 12px; } + +#notes { list-style: none; margin: 0; padding: 0; display: grid; gap: 8px; } +.note { display: grid; grid-template-columns: auto 1fr auto; gap: 12px; align-items: start; } +.note .text { display: grid; gap: 6px; min-width: 0; } +.note .title { font-weight: 600; border-color: transparent; } +.note .body { border-color: transparent; color: var(--muted); } +.note .title:focus, .note .body:focus { border-color: var(--line); } +.note small { color: var(--muted); } +.note.done .title { text-decoration: line-through; color: var(--muted); } +.note input[type="checkbox"] { width: auto; margin-top: 10px; } +.delete { color: #b3261e; } +.empty { color: var(--muted); text-align: center; padding: 24px; } diff --git a/packages/agentenv-protocol/examples/website/notes/web/vite.config.js b/packages/agentenv-protocol/examples/website/notes/web/vite.config.js new file mode 100644 index 000000000..d07ecd2c2 --- /dev/null +++ b/packages/agentenv-protocol/examples/website/notes/web/vite.config.js @@ -0,0 +1,11 @@ +import { defineConfig } from "vite"; + +export default defineConfig({ + base: "./", + server: { + proxy: { + "/api": "http://127.0.0.1:8000", + "/healthz": "http://127.0.0.1:8000", + }, + }, +}); diff --git a/packages/agentenv-protocol/examples/website/notes/website.toml b/packages/agentenv-protocol/examples/website/notes/website.toml new file mode 100644 index 000000000..e0bb05473 --- /dev/null +++ b/packages/agentenv-protocol/examples/website/notes/website.toml @@ -0,0 +1,38 @@ +[site] +name = "notes" +domains = ["notes.test"] +health = "/healthz" + +[source] +sdk = "site:Notes" +assets = "web/dist" + +[data] +seed = ["data/seed.json"] + +[data.collections.users] + +[data.collections.notes] +owner = "owner" + +[users] +personas = [{ id = "ann", name = "Ann Lee" }, { id = "ben", name = "Ben Ortiz" }] +acting_user = "ann" +sign_in = "cookie" + +[roles] +admin = ["all_notes"] + +[routes.ops] +list_users = "GET /api/users" +sign_in = "POST /api/sign-in" +list_notes = "GET /api/notes" +create_note = "POST /api/notes" +get_note = "GET /api/notes/{note_id}" +edit_note = "PATCH /api/notes/{note_id}" +toggle_note = "POST /api/notes/{note_id}/toggle" +delete_note = "DELETE /api/notes/{note_id}" +all_notes = "GET /api/admin/notes" + +[clock] +follow = "header" diff --git a/packages/agentenv-protocol/examples/website/specs/components/sso.toml b/packages/agentenv-protocol/examples/website/specs/components/sso.toml new file mode 100644 index 000000000..5d5594985 --- /dev/null +++ b/packages/agentenv-protocol/examples/website/specs/components/sso.toml @@ -0,0 +1,19 @@ +[site] +name = "sso" + +[source] +sdk = "fake_sso:FakeSso" + +[data.collections.accounts] + +[data.collections.grants] +owner = "user" + +[users] +sign_in = "cookie" + +[routes.ops] +authorize = "GET /authorize" +login = "POST /login" +token = "POST /token" +userinfo = "GET /userinfo" diff --git a/packages/agentenv-protocol/examples/website/specs/intranet-ben.toml b/packages/agentenv-protocol/examples/website/specs/intranet-ben.toml new file mode 100644 index 000000000..7349d418e --- /dev/null +++ b/packages/agentenv-protocol/examples/website/specs/intranet-ben.toml @@ -0,0 +1,8 @@ +extends = "intranet.toml" + +[users] +personas = ["ben", "cy"] +acting_user = "ben" + +[roles] +editor = false diff --git a/packages/agentenv-protocol/examples/website/specs/intranet.toml b/packages/agentenv-protocol/examples/website/specs/intranet.toml new file mode 100644 index 000000000..b4f8965a9 --- /dev/null +++ b/packages/agentenv-protocol/examples/website/specs/intranet.toml @@ -0,0 +1,27 @@ +[site] +name = "intranet" +domains = ["intranet.test", "*.intranet.test"] + +[source.image] +dockerfile = "Dockerfile.intranet" + +[data.collections.users] + +[data.collections.pages] +owner = "author" + +[users] +personas = ["ann", "ben"] +acting_user = "ann" + +[roles] +editor = ["publish_page"] + +[routes.ops] +list_pages = "GET /api/pages" +publish_page = "POST /api/pages" + +[[uses]] +spec = "components/sso.toml" +mount = "/sso" +data = { accounts = "users" } diff --git a/packages/agentenv-protocol/examples/website/specs/slack-live.toml b/packages/agentenv-protocol/examples/website/specs/slack-live.toml new file mode 100644 index 000000000..35d7eec71 --- /dev/null +++ b/packages/agentenv-protocol/examples/website/specs/slack-live.toml @@ -0,0 +1,20 @@ +[site] +name = "slack" +domains = ["slack.com", "*.slack.com"] +health = "/api/api.test" + +[source.live] +allow_hosts = ["*.slack-edge.com", "*.slack-imgs.com"] +credentials = { token = "secret:SLACK_USER_TOKEN" } + +[users] +sign_in = "live" + +[routes.ops] +list_conversations = "GET /api/conversations.list" +history = "GET /api/conversations.history" +post_message = "POST /api/chat.postMessage" +add_reaction = "POST /api/reactions.add" + +[clock] +follow = "none" diff --git a/packages/agentenv-protocol/examples/website/specs/slack.toml b/packages/agentenv-protocol/examples/website/specs/slack.toml new file mode 100644 index 000000000..77635c391 --- /dev/null +++ b/packages/agentenv-protocol/examples/website/specs/slack.toml @@ -0,0 +1,11 @@ +extends = "slack-live.toml" + +[source.replay] +capture = "captures/slack-workspace.har" +allow_hosts = ["*.slack-edge.com"] + +[routes.ops] +post_message = { served_by = "slack_synthetic:SlackChat" } + +[clock] +follow = "date_patch" diff --git a/packages/agentenv-protocol/pyproject.toml b/packages/agentenv-protocol/pyproject.toml index 73559b1c2..91e2985b0 100644 --- a/packages/agentenv-protocol/pyproject.toml +++ b/packages/agentenv-protocol/pyproject.toml @@ -25,6 +25,7 @@ dependencies = [ "pydantic>=2.11", "starlette>=0.37", "httpx>=0.27", + "tomli>=1.1; python_version < '3.11'", ] [project.optional-dependencies] @@ -49,6 +50,12 @@ computer = [ "Pillow>=10.4", "uvicorn>=0.30", ] +# A website whose state is an MCP server's database: @website(models=) and self.db, on a +# postgresql:// DATABASE_URL through psycopg 3, SQLAlchemy 2.1's driver for it. +db = [ + "sqlalchemy>=2.1", + "psycopg[binary]>=3.1", +] dev = [ "a2a-sdk[http-server]>=0.3.26,<0.4", "python-multipart>=0.0.13", @@ -59,6 +66,7 @@ dev = [ "fastmcp>=3.4,<4", # the standalone-fastmcp mount path "Pillow>=10.4", # the computer extra's frame crops and encodings "av>=17", # the computer extra's encoder, and the tests' decoder + "sqlalchemy>=2.1", # the db extra's models= site, tested on sqlite ] [project.urls] Homepage = "https://www.agentenvframework.com" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/client.py b/packages/agentenv-protocol/src/agentenv_protocol/client.py index dcddaf541..a1e352879 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/client.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/client.py @@ -7,6 +7,7 @@ import logging from collections.abc import Iterator from contextvars import ContextVar +from dataclasses import dataclass, field import httpx @@ -210,6 +211,83 @@ def find_tool(card: dict, name: str) -> dict | None: return None +def serves(card: dict, method: str) -> bool | None: + """Whether a card serves the data-plane ``method`` (``data/get`` say): True or False by its declared + ``capabilities.operations``, or None when it declares none, as an older card doesn't, so the caller decides as it + did before cards declared them (a probe, say).""" + declared = operations(card) + return None if declared is None else method in declared + + +@dataclass(frozen=True) +class ChildEnv: + """One child env a composed card nests under ``children_environments``: its ``name`` and ``url`` as its card gives + them, and that ``card``, which says what it offers. A composer roots each child's ``url`` (and its endpoints) at + the route it serves the child at, so two children sharing a name have distinct urls; ``url`` is None for a child + whose card names none.""" + + name: str | None + url: str | None + card: dict = field(hash=False, repr=False) + + @property + def extensions(self) -> list[dict]: + """The extensions the child's card advertises (``capabilities.extensions``).""" + return list((self.card.get("capabilities") or {}).get("extensions") or []) + + @property + def operations(self) -> list[str] | None: + """The data-plane methods the child's card declares (``operations``), or None when it declares none.""" + return operations(self.card) + + def find_extension(self, uri: str) -> dict | None: + """The extension the child's card advertises under ``uri``, or None (``find_extension``).""" + return find_extension(self.card, uri) + + def serves(self, method: str) -> bool | None: + """Whether the child's card serves the data-plane ``method``, None when it declares no operations (``serves``).""" + return serves(self.card, method) + + def base_url(self, address: str) -> str: + """Where the child's data plane is, its ``url`` without the RPC path, resolved against ``address``, the composed + card's: a path joins onto it, a full URL stands alone, and a child naming no url is at ``address`` itself.""" + url = (self.url or RPC_PATH).removesuffix(RPC_PATH) + return url if "://" in url else f"{address.rstrip('/')}{url}" + + +def children(card: dict) -> list[ChildEnv]: + """Each child env a composed card nests under ``children_environments``, in its order; none for a leaf card, or + one whose ``children_environments`` is missing or null. An entry that isn't an object is skipped.""" + return [ChildEnv(name=child.get("name"), url=child.get("url"), card=child) + for child in card.get("children_environments") or [] if isinstance(child, dict)] + + +def child_at(card: dict, url: str) -> ChildEnv | None: + """The child env of a composed card at ``url``: the one whose ``url`` is ``url``, or whose data plane is there + (``url`` without the RPC path, ``/svc/mcp-slack`` for ``/svc/mcp-slack/agentenv`` say), a trailing slash aside; + None when no child is. A composer serves each child at its own route, so this is the lookup two children sharing a + name can't confuse.""" + wanted = url.rstrip("/") + for child in children(card): + if child.url is not None and wanted in (child.url.rstrip("/"), child.url.removesuffix(RPC_PATH).rstrip("/")): + return child + return None + + +class AmbiguousChildError(LookupError): + """More than one child env of a composed card has the name asked for, so the name can't say which is meant.""" + + +def child_named(card: dict, name: str) -> ChildEnv | None: + """The one child env of a composed card named ``name``, or None when none is; raises ``AmbiguousChildError`` when + several are, as a website and an MCP server sharing a name are, which ``child_at`` tells apart by their urls.""" + found = [child for child in children(card) if child.name == name] + if len(found) > 1: + raise AmbiguousChildError(f"{len(found)} child envs are named {name!r} ({', '.join(str(c.url) for c in found)}); " + "look one up by its url") + return found[0] if found else None + + def find_child(card: dict, name: str) -> dict | None: """Return the nested child EnvironmentCard named `name`, or None. @@ -218,6 +296,8 @@ def find_child(card: dict, name: str) -> dict | None: rewritten to be reachable from the composed card's base_url. Callers resolve a service by name here, then use the child card with `find_extension`/`invoke_extension`. Tolerates a missing/null `children_environments` so single/leaf cards return None rather than raising. + Two children sharing `name` return the first; `child_named` refuses that, and `child_at` + finds a child by its url, which tells them apart. """ for child in card.get("children_environments") or []: if child.get("name") == name: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/__init__.py b/packages/agentenv-protocol/src/agentenv_protocol/website/__init__.py new file mode 100644 index 000000000..455f9010c --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/__init__.py @@ -0,0 +1,35 @@ +"""The website SDK: a site's server, meeting ``urn:agentenv:website/v1``. Subclass WebsiteEnvironment, declare the +site with ``@website(domains=[...], static="dist/")``, mark its API with ``@api.get/post/put/patch/delete(path)``, and +``serve()`` it, or declare it from a website spec with ``@website(spec="website.toml")``; it needs only the plain +package, and uvicorn to serve. The class is optional: a server in any language +that meets the contract works, which ``python -m agentenv_protocol.website.conformance `` checks. Whether a site +is ready is its ``WebsiteReadiness`` answer to ``ready``, which ``website_readiness`` reads of one site and +``env_readiness`` of every site of an env. What a task left a site with is its data plane's ``data/get``.""" + +from __future__ import annotations + +from .clock import CLOCK_EXTENSION_URI, UNIVERSE_NOW_HEADER, UniverseClock +from .environment import ( + DEFAULT_HEALTH_PATH, + SESSION_PATH, + WebsiteDeclaration, + WebsiteEnvironment, + ready_check, + website, +) +from .errors import ApiError, BadRequest, Conflict, Forbidden, NotFound, Unauthorized +from .readiness import READY_TIMEOUT_S, WEBSITE_EXTENSION_URI, WEBSITE_READY_OP, WebsiteReadiness, env_readiness, website_readiness +from .routes import ROLE_KEY_ENV, ROLE_KEY_HEADER, ApiContext, ApiRoute, api, reserved_path +from .session import SESSION_COOKIE, SessionSigner +from .spec import NameTable, SpecError, WebsiteSpec, from_card, from_environment, load as load_spec, name_table +from .store import Collections + +__all__ = [ + "WebsiteEnvironment", "WebsiteDeclaration", "website", "api", "ApiContext", "ApiRoute", "ready_check", + "ApiError", "BadRequest", "Unauthorized", "Forbidden", "NotFound", "Conflict", + "Collections", "UniverseClock", "SessionSigner", "reserved_path", + "WebsiteSpec", "SpecError", "NameTable", "load_spec", "from_card", "from_environment", "name_table", + "WEBSITE_EXTENSION_URI", "CLOCK_EXTENSION_URI", "UNIVERSE_NOW_HEADER", "DEFAULT_HEALTH_PATH", "SESSION_PATH", + "SESSION_COOKIE", "ROLE_KEY_ENV", "ROLE_KEY_HEADER", + "WEBSITE_READY_OP", "READY_TIMEOUT_S", "WebsiteReadiness", "website_readiness", "env_readiness", +] diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/clock.py b/packages/agentenv-protocol/src/agentenv_protocol/website/clock.py new file mode 100644 index 000000000..dec301780 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/clock.py @@ -0,0 +1,122 @@ +"""The universe's clock as a site reads it: ``clock/v1``'s ``sync_time`` hands the server the gateway's ``/clock/time`` +URL, as it hands it to MCP servers, and the site's time then follows that clock's ``{"virtual_time": }``, +read at most once a second and extrapolated between reads. The rate it extrapolates at is the one the clock answers +with, ``virtual_seconds_per_real_second``, as the gateway's does; for a clock that answers none, it is one two +successive intervals between reads agreed on, so a clock set or advanced between two reads is taken for a jump, not a +speed, and until two agree it is real time's. An unsynced site reads wall time. Every answer of a site carries the time it answered at as ``X-Universe-Now``, +RFC3339 in UTC. ``sync_time`` may also name the universe's time zone, an IANA name, which the clock keeps for the site +to show local times in; the time itself stays UTC.""" +from __future__ import annotations + +import asyncio +import datetime +import json +import logging +import math +import time + +import httpx + +logger = logging.getLogger(__name__) + +CLOCK_EXTENSION_URI = "urn:agentenv:clock/v1" +UNIVERSE_NOW_HEADER = "X-Universe-Now" +REFRESH_S = 1.0 +READ_TIMEOUT_S = 5.0 +RATE_AGREEMENT = 0.1 + + +def iso(moment: datetime.datetime) -> str: + return moment.astimezone(datetime.timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z") + + +def parse_time(value: str) -> datetime.datetime: + moment = datetime.datetime.fromisoformat(value.replace("Z", "+00:00")) + return moment if moment.tzinfo else moment.replace(tzinfo=datetime.timezone.utc) + + +class UniverseClock: + """The time a site answers with: wall time until ``sync`` names a gateway clock, that clock's after.""" + + def __init__(self) -> None: + self.url: str | None = None + self.timezone: str | None = None + self._virtual: datetime.datetime | None = None + self._read_at = 0.0 + self._tried_at = 0.0 + self._rate = 1.0 + self._measured: float | None = None + self._lock: asyncio.Lock | None = None + + @property + def synced(self) -> bool: + return self._virtual is not None + + def now(self) -> datetime.datetime: + if self._virtual is None: + return datetime.datetime.now(datetime.timezone.utc) + return self._virtual + datetime.timedelta(seconds=(time.monotonic() - self._read_at) * self._rate) + + async def sync(self, url: str, timezone: str | None = None) -> datetime.datetime: + """Follow the clock at ``url`` from now on, in ``timezone`` (none when the sync names none); raises when it + can't be read, keeping the clock and zone it followed. It waits for a refresh reading the clock it followed, + so that read never lands on the new clock.""" + async with self._reading(): + virtual, rate = await _read(url) + self.url, self._virtual, self._rate, self._measured = url, virtual, 1.0 if rate is None else rate, None + self.timezone = timezone + self._read_at = self._tried_at = time.monotonic() + return virtual + + async def refresh(self) -> None: + """Read the synced clock again when the last read is a second old; a failed read keeps extrapolating.""" + if self.url is None or time.monotonic() - self._tried_at < REFRESH_S: + return + async with self._reading(): + started = time.monotonic() + if started - self._tried_at < REFRESH_S: + return + self._tried_at = started + try: + virtual, rate = await _read(self.url) + except Exception as e: + logger.debug("reading the universe's clock at %s failed: %r", self.url, e) + return + read_at = time.monotonic() + if rate is not None: + self._rate = rate + elif self._virtual is not None and read_at > self._read_at: + measured = max(0.0, (virtual - self._virtual).total_seconds() / (read_at - self._read_at)) + if self._measured is not None and abs(measured - self._measured) <= RATE_AGREEMENT * max(measured, self._measured, 1.0): + self._rate = measured + self._measured = measured + self._virtual, self._read_at = virtual, read_at + + def _reading(self) -> asyncio.Lock: + """The lock a sync and a refresh each read the clock under.""" + self._lock = self._lock or asyncio.Lock() + return self._lock + + +async def _read(url: str) -> tuple[datetime.datetime, float | None]: + """The clock's time, and the rate it says it runs at: None when it says none, or none that is a finite rate.""" + async with httpx.AsyncClient() as client: + response = await client.get(url, timeout=READ_TIMEOUT_S) + response.raise_for_status() + body = response.json() + rate = body.get("virtual_seconds_per_real_second") + ok = isinstance(rate, (int, float)) and not isinstance(rate, bool) and math.isfinite(rate) and rate >= 0 + return parse_time(body["virtual_time"]), float(rate) if ok else None + + +def clock_patch_script(now: datetime.datetime) -> bytes: + """An inline script that shifts the page's ``Date`` to the universe's time ``now``, and follows the + ``X-Universe-Now`` header of every ``fetch`` the page makes after. The ``Date`` it puts in place answers called, + with the time's string, as constructed, and keeps ``Date.parse``, ``Date.UTC`` and ``instanceof Date``.""" + return ("" % (int(now.timestamp() * 1000), json.dumps(UNIVERSE_NOW_HEADER))).encode() diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/conformance.py b/packages/agentenv-protocol/src/agentenv_protocol/website/conformance.py new file mode 100644 index 000000000..0a515c80c --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/conformance.py @@ -0,0 +1,520 @@ +"""Check a running site's server against ``urn:agentenv:website/v1``, whatever it's written in: ``python -m +agentenv_protocol.website.conformance `` prints each check and exits 1 when a required one failed (or, with +``--strict``, when a recommended one warned), 2 when the server couldn't be reached. Everything goes over HTTP. + +The required checks are the contract, what the gateway and a browser rely on of any server: ``/`` serves HTML under a +made-up host, the site is ready, by the card's ``website/v1`` ``ready``, whose answer must be a +``WebsiteReadiness``, or, for a card without it, by the health path (the card's, else ``/api/health``) answering 200, which is deprecated, the data plane and card's paths are not +the site's pages, and, when the card advertises the data plane, a round trip at the card's ``url`` (data/get, +data/reset, data/get, which no longer returns the exported data, data/add of what the first data/get returned, data/get +again) returns what it was given. A failed one is ``failed``. The recommended checks are what an SDK site does and the +gateway uses when a server offers it: a card declaring the site's domains, health path and routes, ``X-Universe-Now`` +on every answer the other checks got, ``clock/v1``'s ``sync_time`` against a clock the checker serves, the forwarded +headers wherever an answer shows them (on ``/`` and on a GET route with its trailing slash toggled, which a server may +redirect), and cookies tied to no domain, the session path's the one the card names. A failed one is a ``warning``. + +With ``--spec``, a website spec, the server is checked against it too, and those checks are required: its card's name, +domains, health path, routes and session against the spec's, and every route the spec declares served (an ``OPTIONS`` +that finds something at its path, and a GET route without path parameters answering neither 404 nor 405), each in the +server's OpenAPI document when its card offers one; for a card without one, a route only the ``OPTIONS`` can check is +unverified, a ``warning``, unless that ``OPTIONS`` answers with an ``Allow`` listing its method. The health check +then reads the spec's health path for a card without ``website/v1`` ``ready``. + +The round trip leaves the server's data as it found it, adding the export back when a step after the reset fails too. +The clock check leaves the server following a clock that is gone once the check ends, so a server it ran against +keeps extrapolating from that clock's time until it's synced again. The checker's clock starts at ``CLOCK_START``, far +enough from wall time that a synced server shows it, and a synced server's ``X-Universe-Now`` may be ``CLOCK_SLACK_S`` +seconds from it.""" +from __future__ import annotations + +import argparse +import contextlib +import dataclasses +import datetime +import json +import re +import sys +import threading +import time +import uuid +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Callable, Iterator, Literal + +import httpx +from pydantic import ValidationError + +from ..client import find_extension, find_extension_method +from ..types import METHOD_ADD, METHOD_GET, METHOD_RESET, RPC_PATH, WELL_KNOWN_PATH +from .clock import CLOCK_EXTENSION_URI, UNIVERSE_NOW_HEADER, iso, parse_time +from .environment import DEFAULT_HEALTH_PATH +from .readiness import WEBSITE_EXTENSION_URI, WEBSITE_READY_OP, WebsiteReadiness +from .spec import SpecError, WebsiteSpec, load + +Status = Literal["passed", "failed", "warning", "skipped"] +CLOCK_START = datetime.datetime(2031, 5, 6, 7, 8, 9, tzinfo=datetime.timezone.utc) +CLOCK_SLACK_S = 120 +REQUIRED = ("any_host", "health", "reserved_paths", "data_plane", "spec", "spec_routes") + + +@dataclasses.dataclass(frozen=True) +class Result: + """One check's outcome; ``required`` for a check of the contract, whose failure is ``failed``, and not for a + recommended one, whose failure is a ``warning``.""" + + name: str + status: Status + detail: str = "" + required: bool = True + + @property + def passed(self) -> bool: + return self.status != "failed" + + def fails(self, strict: bool = False) -> bool: + """Whether the result fails the run: a failed check, or with ``strict`` a warning too.""" + return self.status == "failed" or (strict and self.status == "warning") + + +def check(base_url: str, *, client: httpx.Client | None = None, clock_host: str = "127.0.0.1", + timeout: float = 30, spec: WebsiteSpec | None = None) -> list[Result]: + """Each check's result against the server at ``base_url``. ``clock_host`` is the address the server reaches the + checker's clock at; with ``spec``, the server is checked against it too. Raises httpx's error when the server + can't be reached at all.""" + own = client is None + client = client or httpx.Client(timeout=timeout) + try: + return _Checker(base_url.rstrip("/"), client, clock_host, spec).run() + finally: + if own: + client.close() + + +class _Checker: + def __init__(self, base_url: str, client: httpx.Client, clock_host: str, spec: WebsiteSpec | None = None) -> None: + self.base_url, self.client, self.clock_host, self.spec = base_url, client, clock_host, spec + self.rpc_url = base_url + RPC_PATH + self.responses: list[httpx.Response] = [] + + def get(self, path: str, **kwargs) -> httpx.Response: + return self._seen(self.client.get(self.base_url + path, **kwargs)) + + def _seen(self, response: httpx.Response) -> httpx.Response: + self.responses.append(response) + return response + + def run(self) -> list[Result]: + response = self.get(WELL_KNOWN_PATH) + try: + card = response.json() if response.status_code == 200 else {} + except ValueError: + card = {} + card = card if isinstance(card, dict) else {} + self.rpc_url = _rpc_url(self.base_url, card) + site = (find_extension(card, WEBSITE_EXTENSION_URI) or {}).get("params") or {} + health = site.get("health") if isinstance(site.get("health"), str) else DEFAULT_HEALTH_PATH + if self.spec is not None: + health = self.spec.health + steps = [("card", lambda: self._card(response, site)), ("any_host", self._any_host), + ("health", lambda: self._health(card, health)), ("universe_now", self._universe_now), + ("reserved_paths", self._reserved_paths), ("data_plane", lambda: self._data_plane(card)), + ("clock", lambda: self._clock(card, health)), ("forwarded", lambda: self._forwarded(site)), + ("cookies", lambda: self._cookies(site))] + if self.spec is not None: + steps += [("spec", lambda: self._spec_card(card, site)), ("spec_routes", lambda: self._spec_routes(card, site))] + outcomes = {name: _outcome(step) for name, step in steps if name != "universe_now"} + outcomes["universe_now"] = _outcome(self._universe_now) + results = [] + for name, _step in steps: + status, detail = outcomes[name] + required = name in REQUIRED + results.append(Result(name, "warning" if status == "failed" and not required else status, detail, required)) + return results + + def _card(self, response: httpx.Response, site: dict) -> tuple[Status, str]: + if response.status_code != 200: + return "failed", f"{WELL_KNOWN_PATH} answered {response.status_code}" + problems = [] + if not site: + problems.append(f"declares no {WEBSITE_EXTENSION_URI} with params") + if not isinstance(site.get("domains"), list): + problems.append("its params have no domains list") + if not (isinstance(site.get("health"), str) and site["health"].startswith("/")): + problems.append("its params have no health path") + routes = site.get("routes") + if not isinstance(routes, list) or not all(isinstance(r, dict) and {"method", "path", "name"} <= set(r) + for r in routes): + problems.append("its params' routes aren't a list of {method, path, name}") + return ("failed" if problems else "passed"), "; ".join(problems) + + def _any_host(self) -> tuple[Status, str]: + host = f"conformance-{uuid.uuid4().hex[:8]}.invalid" + response = self.get("/", headers={"Host": host}) + kind = response.headers.get("content-type", "") + if response.status_code == 200 and kind.startswith("text/html"): + return "passed", f"/ served HTML for Host {host}" + return "failed", f"/ for Host {host} answered {response.status_code} {kind or 'without a content type'}" + + def _health(self, card: dict, health: str) -> tuple[Status, str]: + ready = find_extension_method(card, WEBSITE_EXTENSION_URI, WEBSITE_READY_OP) + if ready is None: + response = self.get(health) + return (("passed" if response.status_code == 200 else "failed"), + f"{health} answered {response.status_code}; the card has no website/v1 ready, so the health path " + "was read, which is deprecated") + response = self._seen(self.client.request(ready.get("method", "GET"), self.base_url + ready["endpoint"])) + try: + answer = response.json() + except ValueError: + return "failed", f"website/v1 ready answered {response.status_code} with no JSON: {response.text[:200]!r}" + try: + readiness = WebsiteReadiness.model_validate(answer) + except ValidationError as e: + return "failed", (f"website/v1 ready answered {answer!r}, not a WebsiteReadiness: " + f"{'; '.join(_field_error(error) for error in e.errors())}") + if not response.is_success: + return "failed", (f"website/v1 ready answered {response.status_code}: " + f"{', '.join(readiness.problems) or 'no problem named'}") + if readiness.ready: + return "passed", "website/v1 ready says it is ready" + return "failed", f"website/v1 ready says it is not ready: {', '.join(readiness.problems) or 'no problem named'}" + + def _universe_now(self) -> tuple[Status, str]: + missing = [str(r.url.path) for r in self.responses if UNIVERSE_NOW_HEADER not in r.headers] + if missing: + return "failed", f"no {UNIVERSE_NOW_HEADER} on {', '.join(missing)}" + for response in self.responses: + parse_time(response.headers[UNIVERSE_NOW_HEADER]) + return "passed", f"on all {len(self.responses)} answers" + + def _reserved_paths(self) -> tuple[Status, str]: + paths = [RPC_PATH, WELL_KNOWN_PATH, f"{RPC_PATH}/conformance-{uuid.uuid4().hex[:8]}", f"{RPC_PATH}/", f"{WELL_KNOWN_PATH}/"] + for path in paths: + response = self.get(path) + if response.is_success and response.headers.get("content-type", "").startswith("text/html"): + return "failed", f"{path} answered a page, but the gateway keeps {RPC_PATH} and {WELL_KNOWN_PATH}" + return "passed", f"no page at {', '.join(paths)}" + + def _rpc(self, method: str, params: dict | None = None) -> dict: + response = self._seen(self.client.post(self.rpc_url, json={"jsonrpc": "2.0", "id": method, + "method": method, "params": params or {}})) + body = response.json() + if isinstance(body, dict) and "error" in body: + raise RuntimeError(f"{method} answered {body['error']}") + if not response.is_success or not isinstance(body, dict) or "result" not in body: + raise RuntimeError(f"{method} answered {response.status_code} with no JSON-RPC result: {response.text[:200]}") + return body["result"] or {} + + def _data_plane(self, card: dict) -> tuple[Status, str]: + operations = (card.get("capabilities") or {}).get("operations") + operations = operations if isinstance(operations, list) else [] + missing = [m for m in (METHOD_GET, METHOD_RESET, METHOD_ADD) if m not in operations] + if len(missing) == 3: + return "skipped", "the card advertises no data plane" + if missing: + return "failed", f"the card's operations lack {', '.join(missing)}" + exported = self._rpc(METHOD_GET).get("parts") or [] + self._rpc(METHOD_RESET) + added_back = False + try: + cleared = self._rpc(METHOD_GET).get("parts") or [] + if _holds_data(exported) and cleared == exported: + added_back = True + return "failed", "data/get after data/reset returned the data it returned before" + if not exported: + added_back = True + return "passed", "data/get, data/reset and data/get answered; data/get returned nothing to add back" + added_back = True + self._rpc(METHOD_ADD, {"parts": exported}) + again = self._rpc(METHOD_GET).get("parts") or [] + finally: + if not added_back: + with contextlib.suppress(Exception): + self._rpc(METHOD_ADD, {"parts": exported}) + if again != exported: + return "failed", "data/get after data/reset and data/add of its own export returned other data" + return "passed", f"{len(exported)} part(s) round-tripped through reset and add" + + def _clock(self, card: dict, health: str) -> tuple[Status, str]: + if find_extension(card, CLOCK_EXTENSION_URI) is None: + return "failed", f"the card declares no {CLOCK_EXTENSION_URI}" + method = find_extension_method(card, CLOCK_EXTENSION_URI, "sync_time") + if method is None or not method.get("endpoint"): + return "failed", f"{CLOCK_EXTENSION_URI} advertises no sync_time endpoint" + with _clock_server(self.clock_host) as (url, virtual_now): + response = self._seen(self.client.request(method["method"], self.base_url + method["endpoint"], + json={"env_get_time_url": url})) + if response.status_code >= 400: + return "failed", f"sync_time answered {response.status_code}: {response.text[:200]}" + seen = self.get(health).headers.get(UNIVERSE_NOW_HEADER) + expected = virtual_now() + if seen is None: + return "failed", f"no {UNIVERSE_NOW_HEADER} after sync_time" + off = abs((parse_time(seen) - expected).total_seconds()) + if off > CLOCK_SLACK_S: + return "failed", f"after sync_time {UNIVERSE_NOW_HEADER} is {seen}, {off:.0f}s from the clock's {iso(expected)}" + return "passed", f"{UNIVERSE_NOW_HEADER} followed the clock to {seen}" + + def _forwarded(self, site: dict) -> tuple[Status, str]: + internal = f"internal-{uuid.uuid4().hex[:8]}.invalid" + forwarded = f"site-{uuid.uuid4().hex[:8]}.example" + headers = {"Host": internal, "X-Forwarded-Host": forwarded, "X-Forwarded-Proto": "https"} + paths = ["/", *_slash_toggled(site.get("routes"))] + for path in paths: + response = self.get(path, headers=headers, follow_redirects=False) + location = response.headers.get("location", "") + shown = [where for where, text in (("body", response.text), ("Location", location)) if internal in text] + if shown: + return "failed", f"the {' and '.join(shown)} of {path} named the internal Host {internal}, not {forwarded}" + if location.startswith(f"http://{forwarded}"): + return "failed", f"the Location of {path} is {location}, not https as X-Forwarded-Proto said" + return "passed", f"no answer to {', '.join(paths)} named the internal Host" + + def _cookies(self, site: dict) -> tuple[Status, str]: + session = [] + if isinstance(site.get("session_path"), str): + session = self.get(site["session_path"]).headers.get_list("set-cookie") + cookies = [c for r in self.responses for c in r.headers.get_list("set-cookie")] + if not cookies: + return "skipped", "no answer set a cookie" + scoped = [c.split("=", 1)[0] for c in cookies if any(a.strip().lower().startswith("domain=") for a in c.split(";"))] + if scoped: + return "failed", f"cookies tied to a domain: {', '.join(sorted(set(scoped)))}" + named = site.get("session_cookie") + if isinstance(named, str) and session and named not in {c.split("=", 1)[0].strip() for c in session}: + return "failed", f"{site['session_path']} set {', '.join(c.split('=', 1)[0] for c in session)}, not the card's {named}" + return "passed", f"{len(cookies)} cookie(s), none tied to a domain" + + + def _spec_card(self, card: dict, site: dict) -> tuple[Status, str]: + """The card's name, domains, health path, routes and session cookie and path against the spec's, none of the + latter for a spec that doesn't sign in by cookie, and its state and clock when the spec declares them.""" + spec, declared = self.spec, self.spec.to_card_extension() + problems = [] + if spec.site.name is not None and card.get("name") != spec.site.name: + problems.append(f"the card is named {card.get('name')!r}, the spec's site {spec.site.name!r}") + if set(site.get("domains") or []) != set(declared["domains"]): + problems.append(f"the card's domains are {site.get('domains')}, the spec's {declared['domains']}") + if site.get("health") != declared["health"]: + problems.append(f"the card's health is {site.get('health')!r}, the spec's {declared['health']!r}") + theirs = {r.get("name"): _route_key(r) for r in site.get("routes") or [] if isinstance(r, dict)} + for route in declared["routes"]: + if route["name"] not in theirs: + problems.append(f"the card lacks route {route['name']}") + elif theirs[route["name"]] != _route_key(route): + problems.append(f"the card's {route['name']} is {_shown(theirs[route['name']])}, the spec's " + f"{_shown(_route_key(route))}") + if extra := sorted(set(theirs) - {r["name"] for r in declared["routes"]}): + problems.append(f"the card has routes the spec doesn't declare: {', '.join(map(str, extra))}") + for key in ("session_cookie", "session_path", "clock"): + if key in declared and site.get(key) != declared[key]: + problems.append(f"the card's {key} is {site.get(key)!r}, the spec's {declared[key]!r}") + if spec.users.sign_in != "cookie": + for key in ("session_cookie", "session_path"): + if site.get(key) is not None: + problems.append(f"the card's {key} is {site[key]!r}, a cookie sign-in, and the spec's sign-in " + f"is {spec.users.sign_in}") + if problems: + return "failed", "; ".join(problems) + return "passed", f"name, {len(declared['domains'])} domain(s), health and {len(declared['routes'])} route(s) match" + + def _spec_routes(self, card: dict, site: dict) -> tuple[Status, str]: + """Every route the spec declares served: an OPTIONS finding something at its path, a GET route without path + parameters answering neither 404 nor 405, and an operation of the server's OpenAPI document. A path + parameter is probed with a value its converter on the card's route of that name takes (``_PROBES``), ``0`` + when the card gives it none. When the card offers no document, a route the OPTIONS alone can check is + served only when the OPTIONS answers below 500 with an ``Allow`` listing its method; any other answer, such + as a catch-all's 200 without ``Allow``, a 500, or an ``Allow`` naming only the methods of another route at + that path, leaves it unverified, which, when nothing failed, is a ``warning``.""" + routes = self.spec.route_table() + served = {r.get("name"): r.get("path") for r in site.get("routes") or [] + if isinstance(r, dict) and isinstance(r.get("path"), str)} + if not routes: + return "skipped", "the spec declares no routes" + problems, unverified = [], [] + offered = find_extension_method(card, WEBSITE_EXTENSION_URI, "openapi") is not None + try: + document = self._openapi(card) + except RuntimeError as e: + problems.append(str(e)) + document = None + for name, route in routes.items(): + card_path = served.get(name) + template = card_path if card_path and _CONVERTER.sub(r"{\1}", card_path) == route.path else route.path + path = _PARAM.sub(_probe, template) + options = self.client.options(self.base_url + path) + if options.status_code == 404: + problems.append(f"{name}: nothing is at {route.method} {route.path}") + continue + if route.method == "GET" and path == route.path: + status = self.get(path).status_code + if status in (404, 405) or status >= 500: + problems.append(f"{name}: GET {route.path} answered {status}") + elif not offered: + allowed = _allowed(options) + if allowed is None or options.status_code >= 500 or route.method not in allowed: + shown = " without an Allow" if allowed is None else f" allowing {', '.join(sorted(allowed))}" + unverified.append(f"{name}: OPTIONS {path} answered {options.status_code}{shown}, which doesn't " + f"show {route.method} {route.path} is served") + if document is not None: + operation = ((document.get("paths") or {}).get(route.path) or {}).get(route.method.lower()) + if not isinstance(operation, dict) or operation.get("operationId") != name: + problems.append(f"{name}: the OpenAPI document has no operation {name} at {route.method} {route.path}") + if problems: + return "failed", "; ".join(problems + unverified) + if unverified: + return "warning", "unverified: " + "; ".join(unverified) + return "passed", f"all {len(routes)} route(s) served" + (", each in the OpenAPI document" if document else "") + + def _openapi(self, card: dict) -> dict | None: + """The OpenAPI document the card's ``website/v1`` ``openapi`` serves, None when it offers none; raises + RuntimeError when one it offers can't be read.""" + method = find_extension_method(card, WEBSITE_EXTENSION_URI, "openapi") + if method is None or not method.get("endpoint"): + return None + response = self.client.request(method["method"], self.base_url + method["endpoint"], + **({"json": {}} if method["method"] != "GET" else {})) + try: + document = response.json() if response.status_code == 200 else None + except ValueError: + document = None + if not isinstance(document, dict): + raise RuntimeError(f"the card's openapi, {method['method']} {method['endpoint']}, answered " + f"{response.status_code} and no OpenAPI document") + return document + + +def _allowed(response: httpx.Response) -> set[str] | None: + """The methods ``response``'s ``Allow`` lists, None when it has none.""" + allow = response.headers.get("allow") + return None if allow is None else {m.strip().upper() for m in allow.split(",") if m.strip()} + + +_PARAM = re.compile(r"{[^}]+}") +_PROBES = {"uuid": "00000000-0000-0000-0000-000000000000"} + + +def _probe(param: re.Match) -> str: + """A value the path parameter ``param`` matches: its converter's from ``_PROBES``, else ``0``, which an int, + float, str or path parameter all take.""" + _, _, converter = param.group(0)[1:-1].partition(":") + return _PROBES.get(converter, "0") + + +_CONVERTER = re.compile(r"{(\w+):[^}]+}") + + +def _route_key(route: dict) -> tuple: + """A route as the card and the spec can both say it: its method, its path with any converter dropped + (``{rest:path}`` is ``{rest}``), and its roles.""" + return route.get("method"), _CONVERTER.sub(r"{\1}", route.get("path") or ""), tuple(sorted(route.get("roles") or [])) + + +def _shown(key: tuple) -> str: + method, path, roles = key + return f"{method} {path}" + (f" for {list(roles)}" if roles else "") + + +def _rpc_url(base_url: str, card: dict) -> str: + """Where the card says its data plane is: its ``url``, a path joined onto ``base_url`` or a full URL, else + ``RPC_PATH``.""" + url = card.get("url") if isinstance(card.get("url"), str) and card.get("url") else RPC_PATH + return url if "://" in url else base_url + url + + +def _outcome(step: Callable[[], tuple[Status, str]]) -> tuple[Status, str]: + try: + return step() + except Exception as e: + return "failed", f"{type(e).__name__}: {e}" + + +def _field_error(error: dict) -> str: + """One of a ``ValidationError``'s errors as a person reads it: where, then what.""" + where = ".".join(map(str, error.get("loc") or ())) or "the answer" + return f"{where}: {error.get('msg')}" + + +def _holds_data(parts: list) -> bool: + """Whether exported parts carry task data: a part that isn't a data part, or a data part with a value in it.""" + return any(not (isinstance(p, dict) and p.get("kind") == "data") or _filled(p.get("data")) for p in parts) + + +def _filled(value: object) -> bool: + if isinstance(value, (dict, list)): + return any(_filled(v) for v in (value.values() if isinstance(value, dict) else value)) + return value not in (None, "") + + +def _slash_toggled(routes: object) -> list[str]: + """The first GET route without path parameters, with its trailing slash added or taken off: a path a server + may redirect to the route's own, which shows the address the redirect names.""" + for route in routes if isinstance(routes, list) else []: + path = route.get("path") if isinstance(route, dict) and route.get("method") == "GET" else None + if isinstance(path, str) and path.startswith("/") and path != "/" and "{" not in path: + return [path[:-1] if path.endswith("/") else path + "/"] + return [] + + +@contextlib.contextmanager +def _clock_server(host: str) -> Iterator[tuple[str, Callable[[], datetime.datetime]]]: + """A gateway-shaped ``/clock/time`` running from ``CLOCK_START`` at real speed, and its time now.""" + started = time.monotonic() + + def virtual_now() -> datetime.datetime: + return CLOCK_START + datetime.timedelta(seconds=time.monotonic() - started) + + class Clock(BaseHTTPRequestHandler): + def do_GET(self) -> None: + body = json.dumps({"virtual_time": iso(virtual_now()), "virtual_seconds_per_real_second": 1.0}).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args) -> None: + return None + + server = ThreadingHTTPServer(("127.0.0.1" if host in ("127.0.0.1", "localhost") else "0.0.0.0", 0), Clock) + threading.Thread(target=server.serve_forever, daemon=True).start() + try: + address = f"[{host}]" if ":" in host else host + yield f"http://{address}:{server.server_port}/clock/time", virtual_now + finally: + server.shutdown() + server.server_close() + + +_MARK = {"passed": "PASS", "failed": "FAIL", "warning": "WARN", "skipped": "----"} + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="python -m agentenv_protocol.website.conformance", + description="Check a running site's server against urn:agentenv:website/v1.") + parser.add_argument("url", help="where the server answers, such as http://localhost:8000") + parser.add_argument("--clock-host", default="127.0.0.1", help="the address the server reaches this checker at") + parser.add_argument("--timeout", type=float, default=30, help="seconds for each request") + parser.add_argument("--strict", action="store_true", help="fail on a recommended check's warning too") + parser.add_argument("--spec", help="a website spec file to check the server against too") + args = parser.parse_args(argv) + try: + spec = load(args.spec) if args.spec else None + except SpecError as e: + print("\n".join(e.problems), file=sys.stderr) + return 2 + try: + results = check(args.url, clock_host=args.clock_host, timeout=args.timeout, spec=spec) + except httpx.TransportError as e: + print(f"couldn't reach {args.url}: {type(e).__name__}: {e}", file=sys.stderr) + return 2 + for result in results: + kind = "required" if result.required else "recommended" + print(f"{_MARK[result.status]} {result.name} ({kind}){': ' + result.detail if result.detail else ''}") + counts = {status: sum(r.status == status for r in results) for status in _MARK} + print(f"{args.url}: {counts['passed']} passed, {counts['failed']} failed, {counts['warning']} warning(s), " + f"{counts['skipped']} skipped{' (strict)' if args.strict else ''}") + return 1 if any(r.fails(args.strict) for r in results) else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/database.py b/packages/agentenv-protocol/src/agentenv_protocol/website/database.py new file mode 100644 index 000000000..2e18d002b --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/database.py @@ -0,0 +1,277 @@ +"""A site whose state is a database's: ``@website(models=...)`` names the data an MCP server keeps, so the site's pages +and the server's tools read and write the same rows under the same constraints (and the same rules, where the site +calls the server's own service code). ``models`` is either an object with +``load(data)``, ``export()`` and ``reset()`` (each may be async), the data hooks themselves, or SQLAlchemy models: a +declarative ``Base``, a ``MetaData``, or a module holding one (its ``Base``, else its one metadata), as the +synthetic_servers kit's ``database.py`` declares them. For models, the site's ``self.db`` is a SQLAlchemy session +factory on ``DATABASE_URL``, which agent-env points at the env's own schema of the service database (its +``search_path``), creating the models' tables there on first use; its data is a JSON object of each table's rows by +table name, which data/add inserts, data/get exports, and data/reset deletes. SQLAlchemy is imported only for models, +from the ``db`` extra, which carries SQLAlchemy 2.1 and its ``postgresql://`` driver, ``psycopg``; a site whose +``DATABASE_URL`` names another driver installs that one, and an engine whose driver isn't installed raises ImportError +naming it. + +Beside the tables, the data's string ``current_user`` names the user a browser starts signed in as, as the default +store's does.""" +from __future__ import annotations + +import datetime +import decimal +import enum +import itertools +import types +import uuid +from typing import Any, Callable, Iterable + +from pydantic_core import to_jsonable_python + +from ..agent_env_environment import _maybe_await +from .store import CURRENT_USER, part_data + +HOOKS = ("load", "export", "reset") + + +def is_hooks(models: Any) -> bool: + """Whether ``models`` is data hooks of its own: an object with callable ``load``, ``export`` and ``reset``.""" + return all(callable(getattr(models, name, None)) for name in HOOKS) + + +def metadata_of(models: Any) -> Any: + """The SQLAlchemy ``MetaData`` ``models`` names: itself, its ``metadata``, or a module's ``Base``'s or one + metadata; raises TypeError for anything else.""" + sqlalchemy = _sqlalchemy() + if isinstance(models, sqlalchemy.MetaData): + return models + if isinstance(getattr(models, "metadata", None), sqlalchemy.MetaData): + return models.metadata + if isinstance(models, types.ModuleType): + if isinstance(getattr(getattr(models, "Base", None), "metadata", None), sqlalchemy.MetaData): + return models.Base.metadata + found = {id(m): m for value in vars(models).values() if isinstance( + m := value if isinstance(value, sqlalchemy.MetaData) else getattr(value, "metadata", None), + sqlalchemy.MetaData)} + if len(found) == 1: + return next(iter(found.values())) + if found: + raise TypeError(f"module {models.__name__} holds {len(found)} MetaData; pass models= its Base") + raise TypeError(f"models= is an object with load/export/reset, or SQLAlchemy models (a declarative Base, a " + f"MetaData, or a module holding one), not {models!r}") + + +def _sqlalchemy() -> Any: + try: + import sqlalchemy + except ImportError as e: + raise ImportError("@website(models=...) of SQLAlchemy models needs sqlalchemy: " + "pip install 'agentenv-framework-protocol[db]'") from e + return sqlalchemy + + +class Tables: + """The data hooks of SQLAlchemy ``metadata`` on the database at ``url()``: each table's rows by table name.""" + + def __init__(self, metadata: Any, url: Callable[[], str | None]) -> None: + self.metadata, self._url = metadata, url + self._engine: Any = None + self._sessions: Any = None + + @property + def engine(self) -> Any: + """The engine on ``DATABASE_URL``, made on first use with the models' tables created there; raises + ImportError naming the driver when the URL's isn't installed.""" + if self._engine is None: + url = self._url() + if not url: + raise RuntimeError("a site of SQLAlchemy models keeps its rows in DATABASE_URL, which isn't set") + sqlalchemy = _sqlalchemy() + try: + engine = sqlalchemy.create_engine(url) + except ImportError as e: + scheme = sqlalchemy.engine.make_url(url).drivername + raise ImportError(f"DATABASE_URL's {scheme}:// needs the database driver {e.name or 'it names'}, which " + f"isn't installed; the db extra carries psycopg for postgresql://, and a site installs any " + f"other driver itself (pip install {e.name or 'the driver'})") from e + self.metadata.create_all(engine) + self._engine = engine + return self._engine + + @property + def sessions(self) -> Any: + if self._sessions is None: + from sqlalchemy.orm import sessionmaker + + self._sessions = sessionmaker(bind=self.engine) + return self._sessions + + def export(self) -> dict: + sqlalchemy = _sqlalchemy() + with self.engine.connect() as connection: + return {table.name: [to_jsonable_python(dict(row._mapping)) for row in connection.execute( + sqlalchemy.select(table).order_by(*table.primary_key.columns))] for table in self.metadata.sorted_tables} + + def load(self, data: dict) -> None: + """Insert ``data``'s rows, each table's after the tables it refers to, in their order but each row of a table that + refers to itself after the row it refers to (``_parents_first``), each run of rows that name the same columns as + one insert (one insert takes its columns from its first row, so a row that leaves an optional column out goes in + its own), leaving out each computed column, which the database works out itself, so what ``export`` gave loads + back; refuses a name that is no table.""" + tables = {table.name: table for table in self.metadata.sorted_tables} + unknown = sorted(set(data) - set(tables)) + if unknown: + raise ValueError(f"no table {', '.join(map(repr, unknown))}; the tables are {sorted(tables)}") + with self.engine.begin() as connection: + for name, table in tables.items(): + rows = data.get(name) or [] + if not (isinstance(rows, list) and all(isinstance(row, dict) for row in rows)): + raise ValueError(f"table {name!r}'s data is not a list of rows") + for _, run in itertools.groupby(_parents_first(table, rows), key=lambda row: sorted(row)): + connection.execute(table.insert(), [_row(table, row) for row in run]) + if connection.dialect.name == "postgresql": + _advance_sequences(connection, tables.values()) + + def reset(self) -> None: + with self.engine.begin() as connection: + for table in reversed(self.metadata.sorted_tables): + connection.execute(table.delete()) + + +def _parents_first(table: Any, rows: list[dict]) -> list[dict]: + """``rows`` in their order, but each row that refers to another row of ``table`` through a foreign key of the table's + own after that row, each key matched whole, every column of a composite one together, so a constraint that isn't + deferred holds at each insert; rows in a cycle keep their order. The walk keeps its own stack, so a chain of any + length is ordered.""" + links = [([element.parent.name for element in constraint.elements], + [element.column.name for element in constraint.elements]) + for constraint in table.foreign_key_constraints if constraint.referred_table is table] + if not links: + return rows + by_key: dict = {} + for index, row in enumerate(rows): + for n, (_, referenced) in enumerate(links): + key = tuple(row.get(column) for column in referenced) + if None not in key: + by_key[(n, key)] = index + ordered, placed, visiting = [], set(), set() + + def parents(index: int) -> list[int]: + found = [] + for n, (local, _) in enumerate(links): + key = tuple(rows[index].get(column) for column in local) + parent = by_key.get((n, key)) if None not in key else None + if parent is not None and parent != index: + found.append(parent) + return found + + for start in range(len(rows)): + if start in placed: + continue + visiting.add(start) + stack = [(start, iter(parents(start)))] + while stack: + index, pending = stack[-1] + parent = next((p for p in pending if p not in placed and p not in visiting), None) + if parent is not None: + visiting.add(parent) + stack.append((parent, iter(parents(parent)))) + continue + stack.pop() + visiting.discard(index) + placed.add(index) + ordered.append(rows[index]) + return ordered + + +_PARSERS: dict = {datetime.datetime: datetime.datetime.fromisoformat, datetime.date: datetime.date.fromisoformat, + datetime.time: datetime.time.fromisoformat, decimal.Decimal: decimal.Decimal, uuid.UUID: uuid.UUID} + + +def _row(table: Any, row: dict) -> dict: + """``row`` without its computed columns' values, with each JSON string a date, time, decimal or UUID column holds + parsed to that type, and each an enum column holds made its member again, found by its value, as ``export`` writes + it, else by its name.""" + parsed = dict(row) + for column in table.columns: + if column.computed is not None: + parsed.pop(column.name, None) + continue + value = parsed.get(column.name) + if isinstance(value, str): + kind = _python_type(column) + if kind in _PARSERS: + parsed[column.name] = _PARSERS[kind](value.replace("Z", "+00:00") if kind is datetime.datetime else value) + elif isinstance(kind, type) and issubclass(kind, enum.Enum): + parsed[column.name] = _member(kind, value) + return parsed + + +def _member(kind: type[enum.Enum], value: str) -> Any: + """The member of ``kind`` whose value is ``value``, else the one so named, else ``value`` as given.""" + try: + return kind(value) + except ValueError: + return kind.__members__.get(value, value) + + +def _python_type(column: Any) -> type | None: + try: + return column.type.python_type + except NotImplementedError: + return None + + +def _advance_sequences(connection: Any, tables: Iterable[Any]) -> None: + """Move each serial key's sequence past the rows just loaded, so the next row the site or server adds takes a key + of its own.""" + sqlalchemy = _sqlalchemy() + for table in tables: + keys = list(table.primary_key.columns) + if len(keys) == 1 and keys[0].autoincrement in (True, "auto") and _python_type(keys[0]) is int: + quote = connection.dialect.identifier_preparer + name = quote.format_table(table) + connection.execute(sqlalchemy.text( + f"SELECT setval(pg_get_serial_sequence(:table, :column), COALESCE(MAX({quote.quote(keys[0].name)}), 0) + 1, " + f"false) FROM {name}"), {"table": name, "column": keys[0].name}) + + +class Sessions: + """A site's ``self.db``: ``self.db()`` is a SQLAlchemy ``Session`` on the site's database, and ``self.db.begin()`` + one in a transaction that commits as its ``with`` ends; the engine is made on first use, not at mount.""" + + def __init__(self, tables: Tables) -> None: + self._tables = tables + + def __call__(self, **options: Any) -> Any: + return self._tables.sessions(**options) + + def begin(self) -> Any: + return self._tables.sessions.begin() + + @property + def engine(self) -> Any: + return self._tables.engine + + +class ModelState: + """A models site's data plane: ``hooks`` loads, exports and resets its data, and the data's ``current_user`` is + kept beside it.""" + + def __init__(self, hooks: Any) -> None: + self.hooks = hooks + self.current_user: str | None = None + + async def reset(self) -> None: + await _maybe_await(self.hooks.reset()) + self.current_user = None + + async def load_parts(self, parts: list) -> None: + for part in parts: + data = dict(part_data(part)) + if CURRENT_USER in data: + self.current_user = data.pop(CURRENT_USER) + await _maybe_await(self.hooks.load(data)) + + async def export(self) -> dict: + data = dict(await _maybe_await(self.hooks.export())) + if self.current_user is not None: + data[CURRENT_USER] = self.current_user + return data diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/environment.py b/packages/agentenv-protocol/src/agentenv_protocol/website/environment.py new file mode 100644 index 000000000..230ab28f4 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/environment.py @@ -0,0 +1,524 @@ +"""WebsiteEnvironment: an AgentEnvEnvironment that is a site's server, meeting ``urn:agentenv:website/v1``. It serves +plain HTTP on ``PORT`` for any host name: its ``@api`` routes, its static folder at ``/`` with ``index.html`` for the +pages a single-page app routes itself, its health path, and the data plane and card the base class serves. Every answer +carries ``X-Universe-Now``, the universe's time, which ``self.now()`` reads and ``clock/v1``'s ``sync_time`` ties to +the gateway's virtual clock. Without data hooks of its own a site keeps its data in ``self.store``, or with +``@website(models=...)`` in an MCP server's database, which ``self.db`` opens sessions of (``website.database``); the +data plane loads, exports and resets it, and ``data/get`` is how agentenv reads what a task left a site with. + +One route is built in, replaced by a site's own ``@api`` route at its very path and answering before one whose path +parameters would also match it: ``/api/session``, which a frontend calls to learn who is signed in, signing the acting +user in when nobody is (GET), or to sign out (DELETE).""" +from __future__ import annotations + +import inspect +import json +import os +import re +from pathlib import Path +from typing import Any, ClassVar, NamedTuple, Optional +from urllib.parse import unquote + +from starlette.applications import Starlette +from starlette.datastructures import MutableHeaders +from starlette.middleware import Middleware +from starlette.requests import Request +from starlette.responses import FileResponse, JSONResponse, Response +from starlette._utils import get_route_path +from starlette.routing import Route + +from ..agent_env_environment import ( + _CARD_CONFIG_ATTR, + _OP_ATTR, + AgentEnvEnvironment, + AgentEnvStarletteApplication, + _marked_members, + _maybe_await, + add_data, + extension, + get_data, + reset_data, +) +from ..types import DataPart +from .clock import CLOCK_EXTENSION_URI, UNIVERSE_NOW_HEADER, UniverseClock, clock_patch_script, iso +from .database import ModelState, Sessions, Tables, is_hooks, metadata_of +from .errors import NotFound +from .readiness import WEBSITE_EXTENSION_URI, WEBSITE_READY_OP, WebsiteReadiness +from .routes import (ApiContext, ApiRoute, endpoint, error_response, forwarded_prefix, openapi, openapi_path, request_role, + reserved_path, routes_of) +from .session import SessionSigner, expires_at, session_cookie_name +from .spec import (DEFAULT_HEALTH_PATH, SESSION_PATH, SpecError, WebsiteSpec, from_environment, load, + problems_with_environment) +from .store import Collections + +_WEBSITE_ATTR = "_aee_website" +_READY_ATTR = "_aee_website_ready" +_DEFAULT_HOOK_ATTR = "_aee_default_store_hook" +_DEFAULT_HOOKS = {"reset_data": "_store_reset", "add_data": "_store_add", "get_data": "_store_get"} +_HEAD = re.compile(rb"]*>", re.IGNORECASE) + + +class WebsiteDeclaration(NamedTuple): + """What ``@website`` declares: the site's domains, its static folder and seed data file (relative to the class's + file), its health path, whether its HTML pages read the universe's time, the models whose database keeps its state + instead of the default store, and the spec it was declared from.""" + + domains: tuple[str, ...] = () + static: Optional[str] = None + health: str = DEFAULT_HEALTH_PATH + clock_patch: bool = False + data: Optional[str] = None + models: Any = None + spec: Optional[WebsiteSpec] = None + + +def website(_cls: type | None = None, *, domains: list[str] | tuple[str, ...] = (), static: str | None = None, + health: str = DEFAULT_HEALTH_PATH, clock_patch: bool = False, data: str | None = None, + models: Any = None, spec: str | Path | WebsiteSpec | None = None) -> Any: + """Declare a WebsiteEnvironment's site. ``data`` names a JSON file the default store starts with; ``models`` + keeps the site's data in a database instead, an MCP server's (see ``website.database``), and the site's + data plane is that database's. With ``spec``, a website spec file (relative to the class's file) or a loaded one, + the site is the spec's, and the class must serve what it declares; the other arguments but ``models`` are then + the spec's to give.""" + if _cls is not None: + raise TypeError("use @website(...) with parentheses") + if spec is not None and (domains or static is not None or health != DEFAULT_HEALTH_PATH or clock_patch + or data is not None): + raise TypeError("@website(spec=...) takes the site from its spec; set domains, health, assets, seed and " + "clock there") + if not health.startswith("/") or health == "/" or reserved_path(health): + raise ValueError(f"health is a path of its own under '/', not {health!r}") + if models is not None and data is not None: + raise ValueError("data= seeds the default store; a site of models= is seeded through data/add") + if models is not None and not is_hooks(models): + metadata_of(models) + + def deco(cls: type) -> type: + if not (isinstance(cls, type) and issubclass(cls, WebsiteEnvironment)): + raise TypeError(f"@website declares a WebsiteEnvironment subclass, not {cls!r}") + if spec is None: + setattr(cls, _WEBSITE_ATTR, WebsiteDeclaration(tuple(domains), static, health, clock_patch, data, models)) + else: + declared = spec if isinstance(spec, WebsiteSpec) else load(cls._site_path(str(spec))) + if models is not None and declared.data.seed: + raise ValueError("the spec's data.seed seeds the default store; a site of models= is seeded through " + "data/add") + source = declared.source + assets = str(declared.path(source.assets)) if source is not None and source.assets else None + setattr(cls, _WEBSITE_ATTR, WebsiteDeclaration( + declared.site.domains, assets, declared.health, + declared.clock is not None and declared.clock.follow == "date_patch", None, models, declared)) + if problems := problems_with_environment(declared, cls): + raise SpecError([f"{cls.__name__} doesn't serve its spec: {p}" for p in problems]) + _check_health(cls) + return cls + + return deco + + +def ready_check(fn: Any) -> Any: + """Mark a method that says whether the site is usable yet: the health path answers 200 once every one returns a + truthy value, and 503 while one returns a falsy one or raises.""" + setattr(fn, _READY_ATTR, True) + return fn + + +class WebsiteEnvironment(AgentEnvEnvironment): + """Subclass it, declare the site with ``@website(...)``, mark its API with ``@api``, and ``serve()`` it. A + subclass's ``__init__`` calls ``super().__init__()``. ``app`` may be a Starlette or FastAPI app answering what + ``@api`` and the static folder don't (streaming, uploads); its lifespan is not run.""" + + app: ClassVar[Any] = None + asgi: Any = None + db: Any = None + + def __init_subclass__(cls, **kwargs: Any) -> None: + super().__init_subclass__(**kwargs) + routes_of(cls) + own = {getattr(member, _OP_ATTR) for klass in cls.__mro__ for member in vars(klass).values() + if getattr(member, _OP_ATTR, None) is not None and not getattr(member, _DEFAULT_HOOK_ATTR, False)} + for op, name in _DEFAULT_HOOKS.items(): + if op in own: + setattr(cls, name, None) + + def __init__(self) -> None: + self.store = Collections() + for seed in self._seed_files(): + self.store.load(json.loads(seed.read_text(encoding="utf-8"))) + self.clock = UniverseClock() + self._signer = SessionSigner(cookie_name=session_cookie_name(self.environment_name())) + models = self.declaration().models + self._tables = (None if models is None or is_hooks(models) + else Tables(metadata_of(models), lambda: self.database_url)) + self._models = None if models is None else ModelState(models if self._tables is None else self._tables) + self.db = Sessions(self._tables) if self._tables is not None else None + + @classmethod + def declaration(cls) -> WebsiteDeclaration: + return getattr(cls, _WEBSITE_ATTR, None) or WebsiteDeclaration() + + @classmethod + def routes(cls) -> list[ApiRoute]: + return routes_of(cls) + + def environment_name(self) -> str: + """The site's name, as its card names it: ``ENVIRONMENT_NAME``, else the card's declared name, else the + spec's, else the class's.""" + declared = (getattr(self, _CARD_CONFIG_ATTR, None) or {}).get("name") + spec = self.declaration().spec + return (os.environ.get("ENVIRONMENT_NAME") or declared or (spec.site.name if spec is not None else None) + or type(self).__name__) + + @property + def session_cookie(self) -> str: + """The name of the site's session cookie, ``agentenv_session_``.""" + return self._session_signer().cookie_name + + @classmethod + def site_spec(cls) -> WebsiteSpec: + """The site's spec: the one ``@website(spec=...)`` names with what the class serves filled in, else the one + its decorators declare. Its card's website/v1 params are this spec's ``to_card_extension()``, with the + session cookie the running site's.""" + return from_environment(cls) + + @classmethod + def _seed_files(cls) -> list[Path]: + declaration = cls.declaration() + if declaration.spec is not None: + return [declaration.spec.path(seed) for seed in declaration.spec.data.seed] + return [cls._site_path(declaration.data)] if declaration.data is not None else [] + + @property + def database_url(self) -> str | None: + """Where the site keeps its task state: its own schema of the env's database.""" + return os.environ.get("DATABASE_URL") + + @property + def uses_store(self) -> bool: + """Whether the site's data plane exports ``self.store``: it declares no ``@get_data`` of its own.""" + return type(self)._store_get is not None + + def now(self) -> Any: + """The universe's time, an aware UTC datetime: the gateway's virtual clock once synced, else wall time.""" + return self._clock().now() + + @property + def timezone(self) -> str | None: + """The universe's time zone, an IANA name such as ``America/Chicago``, as the last ``sync_time`` named it; + None when it named none. ``now()`` is UTC either way.""" + return self._clock().timezone + + def acting_user(self) -> str | None: + """Who a browser with no session is signed in as: the store's ``current_user``, or the one a models site's data + named, unless a site says else.""" + if getattr(self, "_models", None) is not None: + return self._models.current_user + return self.store.current_user if hasattr(self, "store") else None + + def context(self, request: Request) -> ApiContext: + """The ``ApiContext`` of ``request``, for a mounted ``app``'s routes as for ``@api`` ones.""" + session = self._session_signer().read(request.cookies) + user = session["user"] if session is not None else self.acting_user() + return ApiContext(request, user, request_role(request), session, self._session_signer()) + + def create_app(self) -> Starlette: + if self._mounted: + raise RuntimeError(f"{type(self).__name__} is already mounted") + declaration = self.declaration() + _check_health(type(self)) + api_routes = self.routes() + taken = {openapi_path(r.path) for r in api_routes} + root = self._site_path(declaration.static) if declaration.static is not None else None + asgi = Starlette(middleware=[Middleware(_Forwarded), Middleware(_Universe, env=self)]) + AgentEnvStarletteApplication(self._build_card(), self).add_routes_to_app(asgi) + asgi.router.routes.append(Route(declaration.health, self._health, methods=["GET"])) + if SESSION_PATH not in taken: + asgi.router.routes.append(Route(SESSION_PATH, self._session, methods=["GET", "DELETE"])) + asgi.router.routes.extend(Route(r.path, _unreserved(endpoint(getattr(self, r.fn.__name__), r, self.context)), + methods=[r.method]) for r in api_routes) + asgi.router.default = _Fallback(root, type(self).app) + self.asgi = asgi + self._mounted = True + return asgi + + def mount(self, app: Any) -> Any: + raise TypeError(f"{type(self).__name__} builds its own app with create_app(); set app = ... to serve more") + + def serve(self) -> None: + """Serve the site on ``PORT`` (8000 by default) with uvicorn, of the ``agent`` or ``app`` extra.""" + try: + import uvicorn + except ImportError as e: + raise ImportError("WebsiteEnvironment.serve needs uvicorn: pip install 'uvicorn>=0.30'") from e + uvicorn.run(self.asgi or self.create_app(), host=os.environ.get("HOST", "0.0.0.0"), + port=int(os.environ.get("PORT", "8000"))) + + def extension_params(self, uri: str) -> dict: + """``website/v1``'s params are the spec's card params, each route named by the path the class serves it at, + converters and all (``/files/{rest:path}``), so a gateway matches the requests it answers, nested paths + included; a spec names a path by its template.""" + if uri == WEBSITE_EXTENSION_URI: + params = {**self.site_spec().to_card_extension(), "session_cookie": self.session_cookie, + "session_path": SESSION_PATH} + served = {r.name: r.path for r in self.routes()} + params["routes"] = [{**r, "path": served.get(r["name"], r["path"])} for r in params["routes"]] + return params + return super().extension_params(uri) + + def _build_card(self) -> Any: + """The card, named by the spec's site name when neither ``ENVIRONMENT_NAME`` nor ``@environment_card`` + names it; a card name and a spec name that differ are refused.""" + card = super()._build_card() + spec = self.declaration().spec + named = (getattr(self, _CARD_CONFIG_ATTR, None) or {}).get("name") + if spec is None or spec.site.name is None: + return card + if named is not None and named != spec.site.name: + raise SpecError([f"{type(self).__name__}: @environment_card names it {named!r}, its spec " + f"{spec.site.name!r}"]) + return card if named is not None or os.environ.get("ENVIRONMENT_NAME") else card.model_copy( + update={"name": spec.site.name}) + + @extension(CLOCK_EXTENSION_URI, description="Sync this site to the gateway's virtual clock.") + async def sync_time(self, env_get_time_url: str, timezone: Optional[str] = None) -> dict: + """Follow the gateway's virtual time at ``env_get_time_url`` from now on, in ``timezone`` when the task names + one.""" + virtual = await self._clock().sync(env_get_time_url, timezone.strip() if timezone else None) + return {"env_get_time_url": env_get_time_url, "gateway_virtual_time": iso(virtual), + **({"timezone": self.timezone} if self.timezone else {})} + + @extension(WEBSITE_EXTENSION_URI, op="openapi", method="GET", description=( + "A site's server: domains, its health path and its @api routes by name; openapi returns the OpenAPI document " + "of its routes, ready whether the site is usable now and why not, and session the cookies a browser adds to " + "start signed in as user, or signed out without one.")) + async def _website_openapi(self) -> dict: + name = self._application.environment_card.name if self._application is not None else type(self).__name__ + return openapi(name, self.routes()) + + @extension(WEBSITE_EXTENSION_URI, op=WEBSITE_READY_OP, method="GET") + async def _website_ready(self) -> dict: + """Whether the site is usable now, a ``WebsiteReadiness`` by its ``@ready_check`` methods: ``problems`` names + each not yet true. A deploy reads it through the card, as ``computer-app/v1``'s ``ready`` of an app.""" + problems = await self._not_ready() + return WebsiteReadiness(ready=not problems, problems=problems).model_dump() + + @extension(WEBSITE_EXTENSION_URI, op="session") + async def _website_session(self, user: Optional[str] = None) -> dict: + cookie = self._session_signer().cookie(user) + return {"user": user, "cookies": [cookie], "expires_at": expires_at(cookie["expires"])} + + @reset_data + async def _store_reset(self) -> None: + if self._models is not None: + return await self._models.reset() + self.store.reset() + + @add_data + async def _store_add(self, parts: list) -> None: + if self._models is not None: + return await self._models.load_parts(parts) + self.store.load_parts(parts) + + @get_data + async def _store_get(self) -> list: + return [DataPart(data=await self._models.export() if self._models is not None else self.store.export())] + + async def _not_ready(self) -> list[str]: + return [check.__name__ for check, _marker in _marked_members(self, _READY_ATTR) if not await _ready(check)] + + async def _health(self, request: Request) -> Response: + waiting = await self._not_ready() + if waiting: + return JSONResponse({"status": "starting", "waiting_for": waiting}, status_code=503) + return JSONResponse({"status": "ok"}) + + async def _session(self, request: Request) -> Response: + context = self.context(request) + if request.method == "DELETE": + context.sign_out() + elif context.session is None and context.user is not None: + context.sign_in(context.user) + return context.apply(JSONResponse({"user": context.user, "role": context.role, + "signed_in": context.user is not None})) + + def _clock(self) -> UniverseClock: + if not hasattr(self, "clock"): + raise TypeError(f"{type(self).__qualname__}.__init__ must call super().__init__()") + return self.clock + + def _session_signer(self) -> SessionSigner: + if not hasattr(self, "_signer"): + raise TypeError(f"{type(self).__qualname__}.__init__ must call super().__init__()") + return self._signer + + @classmethod + def _site_path(cls, path: str) -> Path: + given = Path(path) + return given if given.is_absolute() else Path(inspect.getfile(cls)).resolve().parent / given + + +for _hook in _DEFAULT_HOOKS.values(): + setattr(getattr(WebsiteEnvironment, _hook), _DEFAULT_HOOK_ATTR, True) + + +async def _ready(check: Any) -> bool: + try: + return bool(await _maybe_await(check())) + except Exception: + return False + + +def _check_health(cls: type) -> None: + health = cls.declaration().health + clash = [r.name for r in routes_of(cls) if r.method == "GET" and openapi_path(r.path) == health] + if clash: + raise ValueError(f"{cls.__name__}'s health path {health} is served by @api {clash[0]}; use @ready_check") + + +class _Forwarded: + """Puts the address the browser reached the site at in the scope, from ``X-Forwarded-Host``, + ``X-Forwarded-Proto`` and ``X-Forwarded-Prefix``, before anything routes it, so redirects, ``request.url`` and a + mounted app's ``url_for`` name that address and not the internal one the proxy reached the server at. A prefix + becomes the scope's ``root_path`` and leads its ``path``, as ASGI has a mounted app's, so routing still sees the + site's own path.""" + + def __init__(self, app: Any) -> None: + self.app = app + + async def __call__(self, scope: dict, receive: Any, send: Any) -> None: + if scope["type"] in ("http", "websocket"): + headers = dict(scope["headers"]) + host = _first(headers.get(b"x-forwarded-host")) + proto = _first(headers.get(b"x-forwarded-proto")).lower() + scope = dict(scope) + if host: + scope["headers"] = [(k, v) for k, v in scope["headers"] if k != b"host"] + [(b"host", host.encode())] + if proto in ("http", "https"): + scope["scheme"] = proto if scope["type"] == "http" else {"http": "ws", "https": "wss"}[proto] + prefix = forwarded_prefix(_first(headers.get(b"x-forwarded-prefix"))) + if prefix: + scope["root_path"] = unquote(prefix) + scope.get("root_path", "") + scope["path"] = unquote(prefix) + scope["path"] + if scope.get("raw_path") is not None: + scope["raw_path"] = prefix.encode("latin-1") + scope["raw_path"] + await self.app(scope, receive, send) + + +def _first(value: bytes | None) -> str: + return value.decode("latin-1").split(",")[0].strip() if value else "" + + +class _Universe: + """Stamps ``X-Universe-Now`` on every answer, and with ``clock_patch`` puts the clock script in its whole HTML (never a range of it), holding + only the HTML up to its ```` (at most ``HEAD_SCAN_BYTES``), so a page streamed in chunks reaches the browser + as it's sent.""" + + HEAD_SCAN_BYTES = 64 * 1024 + + def __init__(self, app: Any, env: WebsiteEnvironment) -> None: + self.app, self.env = app, env + + async def __call__(self, scope: dict, receive: Any, send: Any) -> None: + if scope["type"] != "http": + return await self.app(scope, receive, send) + clock = self.env._clock() + await clock.refresh() + patch = self.env.declaration().clock_patch and scope["method"] != "HEAD" + held: dict = {} + + async def stamped(message: dict) -> None: + if message["type"] == "http.response.start": + headers = MutableHeaders(scope=message) + now = clock.now() + headers[UNIVERSE_NOW_HEADER] = iso(now) + if (patch and headers.get("content-type", "").startswith("text/html") and "content-encoding" not in headers + and message.get("status") != 206 and "content-range" not in headers): + held.update(start=message, body=b"", script=clock_patch_script(now)) + return + elif held and message["type"] == "http.response.body": + held["body"] += message.get("body", b"") + more = message.get("more_body", False) + if more and not _HEAD.search(held["body"]) and len(held["body"]) < self.HEAD_SCAN_BYTES: + return + start, script = held.pop("start"), held.pop("script") + body = _with_script(held.pop("body"), script) + held.clear() + headers = MutableHeaders(scope=start) + if "content-length" in headers: + headers["content-length"] = str(int(headers["content-length"]) + len(script)) + await send(start) + message = {"type": "http.response.body", "body": body, "more_body": more} + await send(message) + + await self.app(scope, receive, stamped) + + +def _with_script(html: bytes, script: bytes) -> bytes: + head = _HEAD.search(html) + return html[:head.end()] + script + html[head.end():] if head else script + html + + +def _unreserved(handler: Any) -> Any: + """``handler``, an ``@api`` route's endpoint, answering 404 at the data plane's and card's paths, so a broad route + (``/{rest:path}``) never serves a page there; the routes agentenv registers at those paths answer before it.""" + async def answer(request: Request) -> Response: + if reserved_path(request.scope["path"]): + return error_response(NotFound("not found")) + return await handler(request) + return answer + + +class _Fallback: + """What no route answers: a file of the static folder, then the mounted app, then for a page's path the folder's + ``index.html``; the data plane's and card's paths answer 404 however they're spelled.""" + + def __init__(self, root: Path | None, app: Any) -> None: + self.root = root.resolve() if root is not None else None + self.app = app + + async def __call__(self, scope: dict, receive: Any, send: Any) -> None: + if scope["type"] != "http": + if self.app is not None: + await self.app(scope, receive, send) + return + path, reading = get_route_path(scope) or "/", scope["method"] in ("GET", "HEAD") + if reserved_path(path): + return await error_response(NotFound("not found"))(scope, receive, send) + if reading and (file := self._file(path)) is not None: + return await FileResponse(file)(scope, receive, send) + index = self._file("/") if reading and _page(path) else None + if self.app is not None: + return await self._app_or(index, scope, receive, send) + if index is not None: + return await FileResponse(index)(scope, receive, send) + await error_response(NotFound("not found"))(scope, receive, send) + + def _file(self, path: str) -> Path | None: + if self.root is None: + return None + candidate = (self.root / path.lstrip("/")).resolve() + if candidate.is_dir(): + candidate = (candidate / "index.html").resolve() + if not candidate.is_relative_to(self.root): + return None + return candidate if candidate.is_file() else None + + async def _app_or(self, index: Path | None, scope: dict, receive: Any, send: Any) -> None: + """The mounted app's answer, or ``index`` instead of the app's 404 when there is one.""" + missed = False + + async def unless_missed(message: dict) -> None: + nonlocal missed + if message["type"] == "http.response.start" and message["status"] == 404 and index is not None: + missed = True + if not missed: + await send(message) + + await self.app(scope, receive, unless_missed) + if missed: + await FileResponse(index)(scope, receive, send) + + +def _page(path: str) -> bool: + """Whether ``path`` is a page a single-page app routes itself: not the API's, and not a file's.""" + return not (path == "/api" or path.startswith("/api/")) and "." not in path.rsplit("/", 1)[-1] diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/errors.py b/packages/agentenv-protocol/src/agentenv_protocol/website/errors.py new file mode 100644 index 000000000..13e349b88 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/errors.py @@ -0,0 +1,39 @@ +"""Errors an ``@api`` method raises to answer with an HTTP status: the body is ``{"error": message, "code": code}``.""" +from __future__ import annotations + + +class ApiError(Exception): + """An answer other than success, ``status`` with ``{"error": message, "code": code}``; a subclass sets both + defaults, and either may be given per raise.""" + + status: int = 500 + code: str = "error" + + def __init__(self, message: str, *, code: str | None = None, status: int | None = None) -> None: + super().__init__(message) + self.message = message + self.code = code or type(self).code + self.status = status or type(self).status + + def body(self) -> dict: + return {"error": self.message, "code": self.code} + + +class BadRequest(ApiError): + status, code = 400, "bad_request" + + +class Unauthorized(ApiError): + status, code = 401, "unauthorized" + + +class Forbidden(ApiError): + status, code = 403, "forbidden" + + +class NotFound(ApiError): + status, code = 404, "not_found" + + +class Conflict(ApiError): + status, code = 409, "conflict" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/readiness.py b/packages/agentenv-protocol/src/agentenv_protocol/website/readiness.py new file mode 100644 index 000000000..958f828e7 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/readiness.py @@ -0,0 +1,73 @@ +"""How a website env's health is determined, the one definition everything that judges it reads: a site says whether +it is usable now through its card's ``urn:agentenv:website/v1`` ``ready`` (``WEBSITE_READY_OP``), answering a +``WebsiteReadiness``. ``website_readiness`` asks one site as its card advertises it; ``env_readiness`` asks every site +of an env, a composed card's children or a single site's own card, and answers for the env as a whole; asking one +site may take ``READY_TIMEOUT_S`` by default.""" +from __future__ import annotations + +import asyncio + +import httpx +from pydantic import BaseModel, ConfigDict + +from ..client import find_extension_method, invoke_extension + +WEBSITE_EXTENSION_URI = "urn:agentenv:website/v1" +WEBSITE_READY_OP = "ready" +READY_TIMEOUT_S = 10.0 + + +class WebsiteReadiness(BaseModel): + """What a site says of its readiness, ``website/v1``'s ``ready``: whether it is ``ready``, and the ``problems`` + that keep it from being so, each a name a person reads, such as a ``@ready_check`` method's. Fields it doesn't + name are ignored rather than refused, so a reader keeps reading a newer site that says more, as ``ready`` and + ``problems`` keep their meaning. Its types are strict, as JSON's are: ``"yes"`` or ``1`` is no ``ready``.""" + + model_config = ConfigDict(extra="ignore", strict=True) + + ready: bool + problems: list[str] = [] + + +def _advertises_ready(card: dict) -> bool: + return find_extension_method(card, WEBSITE_EXTENSION_URI, WEBSITE_READY_OP) is not None + + +async def website_readiness(base_url: str, card: dict, *, timeout: float = READY_TIMEOUT_S, + headers: dict[str, str] | None = None) -> WebsiteReadiness | None: + """What the site whose ``card`` is read from ``base_url`` says of its readiness, None when the card doesn't + advertise ``website/v1``'s ``ready``. Raises httpx's error when it can't be asked, ``ExtensionCallError`` for a + non-2xx answer, and ``ValueError`` (pydantic's ``ValidationError`` among them) for an answer that isn't a + ``WebsiteReadiness``.""" + if not _advertises_ready(card): + return None + answer = await invoke_extension(base_url, card, WEBSITE_EXTENSION_URI, timeout=timeout, method=WEBSITE_READY_OP, + headers=headers) + return WebsiteReadiness.model_validate(answer) + + +async def env_readiness(base_url: str, card: dict, *, timeout: float = READY_TIMEOUT_S, + headers: dict[str, str] | None = None) -> WebsiteReadiness: + """What the env whose ``card`` is read from ``base_url`` says of its readiness, from every site it has, asked at + once: the card itself when it advertises ``website/v1``'s ``ready``, as a single site's does, its problems as the + site names them, and each child under a composed card's ``children_environments`` that advertises it, its endpoints + rooted at ``base_url`` as a gateway's are, its problems prefixed with its name. A site that can't be asked, or + answers something that isn't a ``WebsiteReadiness``, is a problem of its own, and one not ready that names no + problem is one too. A child, or a card, that doesn't advertise ``ready`` isn't counted, so an env with no site + serving it, as one of MCP servers alone, is ready with no problems.""" + sites = [("", card)] if _advertises_ready(card) else [] + sites += [(f"{child.get('name') or 'child'}: ", child) for child in card.get("children_environments") or [] + if isinstance(child, dict) and _advertises_ready(child)] + answers = await asyncio.gather(*(_problems(base_url, site, timeout, headers) for _prefix, site in sites)) + problems = [prefix + problem for (prefix, _site), found in zip(sites, answers) for problem in found] + return WebsiteReadiness(ready=not problems, problems=problems) + + +async def _problems(base_url: str, card: dict, timeout: float, headers: dict[str, str] | None) -> list[str]: + try: + readiness = await website_readiness(base_url, card, timeout=timeout, headers=headers) + except (httpx.HTTPError, ValueError) as e: + return [f"{WEBSITE_READY_OP} could not be read: {type(e).__name__}: {e}"] + if readiness.ready: + return [] + return readiness.problems or [f"{WEBSITE_READY_OP} says not ready"] diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/routes.py b/packages/agentenv-protocol/src/agentenv_protocol/website/routes.py new file mode 100644 index 000000000..15158ddc8 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/routes.py @@ -0,0 +1,385 @@ +"""A site's API: ``@api.get/post/put/patch/delete(path)`` methods, each bound from its type hints. A parameter named in +the path is a path parameter, one annotated with a pydantic model or ``dict`` is the JSON body, one annotated +``ApiContext`` receives the request's context, and every other one is a query parameter; path and query values are +coerced to their annotated types (int, float, bool, datetime, UUID, enum, Literal). What the method returns is the +JSON answer, a Starlette ``Response`` as it is; an ``ApiError`` it raises answers with its status, and a request its +parameters can't be bound from answers 400.""" +from __future__ import annotations + +import dataclasses +import datetime +import hmac +import inspect +import json +import logging +import os +import posixpath +import re +from typing import Any, Callable, Literal, NamedTuple, Optional, get_args, get_origin, get_type_hints + +from pydantic import BaseModel, TypeAdapter, ValidationError +from pydantic_core import to_jsonable_python +from starlette.requests import Request +from starlette.responses import JSONResponse, Response + +from ..agent_env_environment import _coerce_scalar, _maybe_await, _schema_for_annotation, _unwrap_annotation +from ..tool_context import DEFAULT_ROLE, caller_from_request +from ..types import RPC_PATH, WELL_KNOWN_PATH +from .errors import ApiError, BadRequest, Forbidden +from .session import SessionSigner + +logger = logging.getLogger(__name__) + +_ROUTE_ATTR = "_aee_api_route" +_PATH_PARAM = re.compile(r"{(\w+)(?::\w+)?}") +Kind = Literal["path", "query", "body", "context"] + + +def reserved_path(path: str) -> bool: + """Whether ``path`` is the data plane's or the card's, however it's spelled, as the gateway decides it.""" + normalized = posixpath.normpath("/" + path.lstrip("/")) + return normalized == RPC_PATH or normalized.startswith(RPC_PATH + "/") or normalized == WELL_KNOWN_PATH + + +class ApiRoute(NamedTuple): + """One ``@api`` method: its verb, its Starlette path, its name on the card and in the OpenAPI document, the roles + it answers (any when empty), the status it succeeds with, and its function.""" + + method: str + path: str + name: str + roles: tuple[str, ...] + status: int + fn: Callable + + def advertised(self) -> dict: + """The route as the card names it: its path is the Starlette path it is served at, converters and all + (``/files/{rest:path}``), so a gateway matches the requests it answers, nested paths included.""" + return {"method": self.method, "path": self.path, "name": self.name, "roles": list(self.roles) or None} + + +class _Api: + """``api.get(path, name=..., roles=[...])`` and its siblings mark a method as one route of the site's API.""" + + def get(self, path: str, **options: Any) -> Callable: + return self.route("GET", path, **options) + + def post(self, path: str, **options: Any) -> Callable: + return self.route("POST", path, **options) + + def put(self, path: str, **options: Any) -> Callable: + return self.route("PUT", path, **options) + + def patch(self, path: str, **options: Any) -> Callable: + return self.route("PATCH", path, **options) + + def delete(self, path: str, **options: Any) -> Callable: + return self.route("DELETE", path, **options) + + def route(self, method: str, path: str, *, name: str | None = None, roles: list[str] | None = None, + status: int = 200) -> Callable: + if not path.startswith("/"): + raise ValueError(f"an @api path starts with '/', not {path!r}") + if reserved_path(path): + raise ValueError(f"@api path {path!r} is under the data plane's {RPC_PATH} or the card's {WELL_KNOWN_PATH}") + + def mark(fn: Callable) -> Callable: + missing = sorted(set(_PATH_PARAM.findall(path)) - set(inspect.signature(fn).parameters)) + if missing: + raise TypeError(f"{fn.__qualname__} takes no parameter for the path's {missing}") + setattr(fn, _ROUTE_ATTR, ApiRoute(method.upper(), path, name or fn.__name__, tuple(roles or ()), status, fn)) + return fn + + return mark + + +api = _Api() + + +def routes_of(cls: type) -> list[ApiRoute]: + """The class's ``@api`` routes in the order they were first defined, each as the class resolves its name; + a name or a verb and path served twice is refused.""" + names = dict.fromkeys(name for klass in reversed(cls.__mro__) for name in vars(klass)) + routes = [route for name in names if (route := getattr(getattr(cls, name, None), _ROUTE_ATTR, None)) is not None] + for label, keys in (("name", [r.name for r in routes]), ("route", [f"{r.method} {r.path}" for r in routes])): + if twice := sorted({k for k in keys if keys.count(k) > 1}): + raise ValueError(f"{cls.__name__} serves the @api {label} {twice} twice") + return routes + + +def openapi_path(path: str) -> str: + return _PATH_PARAM.sub(r"{\1}", path) + + +@dataclasses.dataclass +class ApiContext: + """The request an ``@api`` method answers: its headers, cookies and query, the user it's signed in as (the + session cookie's, else the site's acting user), the caller's role (``request_role``), and the site's own address as the browser reached it. ``sign_in`` and ``sign_out`` set the + session cookie on the answer.""" + + request: Request + user: Optional[str] + role: str + session: Optional[dict] + _signer: SessionSigner = dataclasses.field(repr=False, default_factory=SessionSigner) + _set_cookie: list = dataclasses.field(repr=False, default_factory=list) + + @property + def headers(self) -> Any: + return self.request.headers + + @property + def cookies(self) -> dict[str, str]: + return self.request.cookies + + @property + def query(self) -> Any: + return self.request.query_params + + @property + def base_url(self) -> str: + return base_url(self.request) + + def sign_in(self, user: str) -> None: + self._set_session(user) + + def sign_out(self) -> None: + self._set_session(None) + + def _set_session(self, user: str | None) -> None: + value, expires = self._signer.encode(user) + self._set_cookie.append((value, expires)) + self.user, self.session = user, {"user": user} + + def apply(self, response: Response) -> Response: + for value, expires in self._set_cookie: + at = datetime.datetime.fromtimestamp(expires, datetime.timezone.utc) + response.set_cookie(self._signer.cookie_name, value, expires=at, path="/", httponly=True, samesite="lax") + return response + + +def base_url(request: Request) -> str: + """The site's address as the browser reached it: ``X-Forwarded-Proto`` and ``X-Forwarded-Host`` when a proxy + sent them, the request's own scheme and ``Host`` otherwise, under the path ``X-Forwarded-Prefix`` names when the + proxy serves the site below one (the gateway's ``/website/``).""" + proto = request.headers.get("x-forwarded-proto", "").split(",")[0].strip() or request.url.scheme + host = (request.headers.get("x-forwarded-host", "").split(",")[0].strip() or request.headers.get("host") + or request.url.netloc) + return f"{proto}://{host}{forwarded_prefix(request.headers.get('x-forwarded-prefix', ''))}" + + +def forwarded_prefix(value: str) -> str: + """The path an ``X-Forwarded-Prefix`` value names, without its trailing slash; empty for none, and for one that is + not an absolute path or would make the site's URLs name another host (``//host``). + + >>> [forwarded_prefix(v) for v in ("/website/notes/", "/a, /b", "", "/", "website", "//evil.example", "/a//b")] + ['/website/notes', '/a', '', '', '', '', ''] + """ + prefix = value.split(",")[0].strip().rstrip("/") + return prefix if prefix.startswith("/") and "//" not in prefix and prefix.isprintable() else "" + + +ROLE_KEY_ENV = "WEBSITE_ROLE_KEY" +ROLE_KEY_HEADER = "AgentEnv-Role-Key" + + +def request_role(request: Request) -> str: + """The role ``AgentEnv-Role`` names, believed only from a caller that also sends ``AgentEnv-Role-Key`` equal to + ``WEBSITE_ROLE_KEY``; every other caller, and every caller of a site with no key set, is ``default``. A deploy with + ``keyed_roles`` sets the key and its gateway sends it on beside a role it verified the same way; any other gateway + forwards whatever role a browser sends, so without the key a page could claim any role.""" + claimed = caller_from_request(None, request.headers).role or DEFAULT_ROLE + key = os.environ.get(ROLE_KEY_ENV, "") + if claimed == DEFAULT_ROLE or not key: + return DEFAULT_ROLE + given = request.headers.get(ROLE_KEY_HEADER, "") + return claimed if hmac.compare_digest(given.encode(), key.encode()) else DEFAULT_ROLE + + +class _Param(NamedTuple): + name: str + kind: Kind + annotation: Any + default: Any + + +def plan(route: ApiRoute) -> list[_Param]: + """How each of the route's parameters is bound; at most one is the JSON body, and never on a GET.""" + hints = _hints(route.fn) + path_params = set(_PATH_PARAM.findall(route.path)) + params = [] + for name, param in list(inspect.signature(route.fn).parameters.items())[1:]: + if param.kind in (inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD): + continue + annotation = hints.get(name, param.annotation) + kind: Kind = ("context" if _is_context(annotation) else "path" if name in path_params + else "body" if _body_type(annotation) is not None else "query") + params.append(_Param(name, kind, annotation, param.default)) + bodies = [p.name for p in params if p.kind == "body"] + if len(bodies) > 1 or (bodies and route.method == "GET"): + raise TypeError(f"{route.fn.__qualname__}: a route takes at most one JSON body and a GET route none, " + f"not {bodies}") + return params + + +def _is_context(annotation: Any) -> bool: + if isinstance(annotation, str): + return annotation.strip("'\"").rsplit(".", 1)[-1] in ("ApiContext", "Optional[ApiContext]") + return any(t is ApiContext for t in _unwrap_annotation(annotation)) + + +def _body_type(annotation: Any) -> type | None: + for t in _unwrap_annotation(annotation): + base = get_origin(t) or t + if base is dict or (isinstance(base, type) and issubclass(base, BaseModel)): + return t + return None + + +async def bind(params: list[_Param], request: Request, context: ApiContext) -> dict: + """The method's arguments from ``request``; raises BadRequest for one that is missing or doesn't fit.""" + kwargs: dict = {} + for p in params: + if p.kind == "context": + kwargs[p.name] = context + elif p.kind == "body": + kwargs[p.name] = await _body(p, request) + else: + source = request.path_params if p.kind == "path" else request.query_params + if p.name not in source: + if p.default is inspect.Parameter.empty: + raise BadRequest(f"missing {p.kind} parameter {p.name!r}", code="invalid_params") + kwargs[p.name] = p.default + continue + many = _list_type(p.annotation) if p.kind == "query" else None + try: + kwargs[p.name] = ([_coerce_scalar(v, get_args(many)[0] if get_args(many) else None) + for v in source.getlist(p.name)] if many is not None + else _coerce_scalar(source[p.name], p.annotation)) + except (TypeError, ValueError) as e: + raise BadRequest(f"{p.kind} parameter {p.name!r}: {e}", code="invalid_params") from None + return kwargs + + +def _list_type(annotation: Any) -> Any: + """The ``list[T]`` an annotation names, Optional or Annotated or not; None when it names no list.""" + inner = _unwrap_annotation(annotation) + return inner[0] if len(inner) == 1 and (inner[0] is list or get_origin(inner[0]) is list) else None + + +async def _body(p: _Param, request: Request) -> Any: + raw = await request.body() + if not raw: + if p.default is not inspect.Parameter.empty: + return p.default + raise BadRequest("this route takes a JSON body", code="invalid_params") + try: + value = json.loads(raw) + except ValueError: + raise BadRequest("the request body is not JSON", code="invalid_params") from None + try: + return TypeAdapter(p.annotation).validate_python(value) + except ValidationError as e: + problems = "; ".join(f"{'.'.join(map(str, err['loc'])) or 'body'}: {err['msg']}" for err in e.errors()) + raise BadRequest(f"invalid body: {problems}", code="invalid_params") from None + + +def endpoint(fn: Callable, route: ApiRoute, context_for: Callable[[Request], ApiContext]) -> Callable: + """The Starlette endpoint of one route, calling the bound method ``fn``.""" + params = plan(route) + + async def handle(request: Request) -> Response: + context = None + try: + context = context_for(request) + if route.roles and context.role not in route.roles: + raise Forbidden(f"role {context.role!r} may not call {route.name}") + result = await _maybe_await(fn(**await bind(params, request, context))) + if isinstance(result, Response): + response = result + elif not _has_body(route.status): + response = Response(status_code=route.status) + else: + response = JSONResponse(to_jsonable_python(result), status_code=route.status) + except ApiError as e: + response = error_response(e) + except Exception: + logger.exception("@api %s failed", route.name) + response = error_response(ApiError("internal error", code="internal")) + return context.apply(response) if context is not None else response + + return handle + + +def _has_body(status: int) -> bool: + """Whether an answer with ``status`` may carry a body: not a 1xx, 204 or 304.""" + return status >= 200 and status not in (204, 304) + + +def error_response(error: ApiError) -> Response: + return JSONResponse(error.body(), status_code=error.status) + + +def openapi(title: str, routes: list[ApiRoute]) -> dict: + """The OpenAPI 3.1 document of ``routes``, each operation's id its route's name; the models their bodies and + answers name, nested ones too, are its ``components.schemas``, which their schemas reference. Bodies are + described as they are validated and answers as they are serialized; a model the two modes see differently is in + ``components.schemas`` twice, as ``-Input`` and ``-Output``.""" + paths: dict = {} + schemas: list = [] + for route in routes: + doc = inspect.getdoc(route.fn) or "" + operation: dict = {"operationId": route.name, "parameters": [], + "responses": {str(route.status): {"description": "Success"}}} + if doc: + operation["summary"], operation["description"] = doc.splitlines()[0], doc + if route.roles: + operation["x-agentenv-roles"] = list(route.roles) + for p in plan(route): + if p.kind in ("path", "query"): + listed = _list_type(p.annotation) if p.kind == "query" else None + schema = _schema_for_annotation(p.annotation) if listed is None else { + "type": "array", **({"items": _schema_for_annotation(get_args(listed)[0])} if get_args(listed) else {})} + if p.default is not inspect.Parameter.empty: + schema["default"] = to_jsonable_python(p.default) + operation["parameters"].append({"name": p.name, "in": p.kind, "schema": schema, + "required": p.kind == "path" or p.default is inspect.Parameter.empty}) + elif p.kind == "body": + media: dict = {} + operation["requestBody"] = {"required": p.default is inspect.Parameter.empty, + "content": {"application/json": media}} + schemas.append((media, p.annotation, "validation")) + returns = _hints(route.fn).get("return") + if _has_body(route.status) and _adapter(returns) is not None: + media = {} + operation["responses"][str(route.status)]["content"] = {"application/json": media} + schemas.append((media, returns, "serialization")) + paths.setdefault(openapi_path(route.path), {})[route.method.lower()] = operation + document: dict = {"openapi": "3.1.0", "info": {"title": title, "version": "1.0"}, "paths": paths} + adapters = [(i, mode, adapter) for i, (_media, annotation, mode) in enumerate(schemas) + if (adapter := _adapter(annotation)) is not None] + found, definitions = TypeAdapter.json_schemas(adapters, ref_template="#/components/schemas/{model}") + for i, (media, _annotation, mode) in enumerate(schemas): + media["schema"] = found.get((i, mode), {}) + if definitions.get("$defs"): + document["components"] = {"schemas": definitions["$defs"]} + return document + + +def _hints(fn: Callable) -> dict: + try: + return get_type_hints(fn) + except Exception: + return {} + + +def _adapter(annotation: Any) -> TypeAdapter | None: + """The adapter of a type a JSON schema can describe; None for no type, None itself, or one it can't describe.""" + if annotation is None or annotation is type(None) or annotation is inspect.Parameter.empty: + return None + try: + adapter = TypeAdapter(annotation) + adapter.json_schema() + except Exception: + return None + return adapter diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/session.py b/packages/agentenv-protocol/src/agentenv_protocol/website/session.py new file mode 100644 index 000000000..b84b47be5 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/session.py @@ -0,0 +1,88 @@ +"""A site's signed session cookie: who the browser is signed in as, or that it signed out, signed with HMAC-SHA256 +under ``WEBSITE_SESSION_SECRET``, or a secret of the process's own when that isn't set. The cookie names no domain, +so it holds under whatever host the site is reached at, and its name is the site's own, ``agentenv_session_``, so +two sites a browser reaches under one gateway host (``/website/a/``, ``/website/b/``) keep a session each; a site still +reads the shared ``agentenv_session`` of older sites, for one release.""" +from __future__ import annotations + +import base64 +import binascii +import datetime +import hashlib +import hmac +import json +import os +import re +import secrets +import time + +SESSION_COOKIE = "agentenv_session" +SECRET_ENV = "WEBSITE_SESSION_SECRET" +SESSION_TTL_S = 7 * 24 * 3600 +SECRET_BYTES = 32 +_NOT_TOKEN = re.compile(r"[^A-Za-z0-9_.-]") + + +def _cookie_suffix(environment_name: str) -> str: + """The part of a site's cookie names that is its own: the name itself when a cookie name can hold each of its + characters, else the name with each such character made ``_`` and ``~`` and a hash of the whole name appended. No + name kept as itself holds a ``~``, so two distinct names never make one suffix.""" + if not _NOT_TOKEN.search(environment_name): + return environment_name + digest = hashlib.sha256(environment_name.encode("utf-8", "surrogatepass")).hexdigest()[:12] + return f"{_NOT_TOKEN.sub('_', environment_name)}~{digest}" + + +def session_cookie_name(environment_name: str | None) -> str: + """The session cookie of the site named ``environment_name``: ``agentenv_session_``, a name a cookie name + can't hold spelled as ``_cookie_suffix`` spells it; the shared ``agentenv_session`` for a site with no name.""" + return f"{SESSION_COOKIE}_{_cookie_suffix(environment_name)}" if environment_name else SESSION_COOKIE + + +class SessionSigner: + def __init__(self, secret: str | bytes | None = None, cookie_name: str = SESSION_COOKIE) -> None: + secret = secret if secret is not None else os.environ.get(SECRET_ENV) + self._key = (secret.encode() if isinstance(secret, str) else secret) or secrets.token_bytes(SECRET_BYTES) + self.cookie_name = cookie_name + + def encode(self, user: str | None, ttl_s: int = SESSION_TTL_S) -> tuple[str, int]: + """The cookie value for ``user`` signed in, or signed out when None, and when it expires, in epoch + seconds.""" + expires = int(time.time()) + ttl_s + payload = base64.urlsafe_b64encode(json.dumps({"u": user, "exp": expires}).encode()).decode().rstrip("=") + return f"{payload}.{self._sign(payload)}", expires + + def decode(self, value: str | None) -> dict | None: + """The session a cookie value carries, ``{"user": ...}``; None for one that is absent, forged or expired.""" + payload, _, signature = (value or "").partition(".") + if not payload or not hmac.compare_digest(signature.encode("utf-8", "surrogateescape"), + self._sign(payload).encode()): + return None + try: + data = json.loads(base64.urlsafe_b64decode(payload + "=" * (-len(payload) % 4))) + except (ValueError, binascii.Error): + return None + if not isinstance(data, dict) or not isinstance(data.get("exp"), int) or data["exp"] < time.time(): + return None + return {"user": data.get("u")} + + def read(self, cookies: dict) -> dict | None: + """The session of a request's ``cookies``: its own cookie's, else the shared legacy cookie's.""" + session = self.decode(cookies.get(self.cookie_name)) + if session is None and self.cookie_name != SESSION_COOKIE: + session = self.decode(cookies.get(SESSION_COOKIE)) + return session + + def cookie(self, user: str | None, ttl_s: int = SESSION_TTL_S) -> dict: + """The cookie a browser sets to be signed in as ``user``, as a browser automation adds one; its domain is + left to whoever adds it.""" + value, expires = self.encode(user, ttl_s) + return {"name": self.cookie_name, "value": value, "path": "/", "expires": expires, "httpOnly": True, + "sameSite": "Lax"} + + def _sign(self, payload: str) -> str: + return hmac.new(self._key, payload.encode("utf-8", "surrogateescape"), hashlib.sha256).hexdigest() + + +def expires_at(epoch: int) -> str: + return datetime.datetime.fromtimestamp(epoch, datetime.timezone.utc).isoformat().replace("+00:00", "Z") diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/spec/__init__.py b/packages/agentenv-protocol/src/agentenv_protocol/website/spec/__init__.py new file mode 100644 index 000000000..aa0fe5ce1 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/spec/__init__.py @@ -0,0 +1,1027 @@ +"""The website spec: one declarative description of a site that every way of serving one produces or is checked +against. A spec is a TOML file of fixed sections, each optional: ``[site]`` (name, domains, health path), +``[source]`` (exactly one of ``image``, ``sdk``, ``static``, ``live``, ``replay``), ``[data]``, ``[users]``, +``[roles]``, ``[routes]``, ``[clock]`` and ``[[uses]]``, with ``extends`` naming a spec it overlays. + +``load(path)`` reads one, resolves ``extends`` (a deep merge: tables merge key by key, arrays and scalars replace, +``[source]`` replaces whole, ``false`` removes an inherited route, role or collection, and ``[[uses]]`` entries are +added, one with the same ``as`` replacing the inherited one) and composes ``[[uses]]`` (each component's routes +mounted under its path prefix and named ``.``, its collections namespaced ``.`` unless mapped +into one of the site's, its personas merged), and validates the result, raising ``SpecError`` with every problem. +A resolved spec has no ``extends``, ``uses`` or ``routes.openapi`` left, the document's operations being in +``routes.ops`` (less any an overlay removed); ``components`` says what was mounted. + +``to_card_extension()`` is the ``urn:agentenv:website/v1`` params a server of the spec advertises, ``from_card`` +reads a running site's card back into a spec, ``from_environment`` derives one from a WebsiteEnvironment class's +decorators, and ``name_table`` builds a multi env's one table of host names, synthetic before live before +allowlisted. ``python -m agentenv_protocol.website.spec validate|show|schema`` is its command line.""" +from __future__ import annotations +from ..readiness import WEBSITE_EXTENSION_URI + +import argparse +import copy +import dataclasses +import inspect +import json +import os +import re +import sys +from pathlib import Path +from typing import Any, Iterable, Literal, Optional, Union + +from pydantic import (BaseModel, ConfigDict, Field, PrivateAttr, ValidationError, ValidationInfo, field_validator, + model_validator) + +from ...agent_env_environment import _CARD_CONFIG_ATTR +from ...client import find_extension +from ..routes import openapi_path, reserved_path +from ..session import session_cookie_name + +try: + import tomllib +except ImportError: + try: + import tomli as tomllib + except ImportError: + tomllib = None + +DEFAULT_HEALTH_PATH = "/api/health" +SESSION_PATH = "/api/session" +METHODS = ("GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS") +SOURCE_KINDS = ("image", "sdk", "static", "live", "replay") +NameKind = Literal["synthetic", "live", "allowlist"] +_PRECEDENCE: dict[str, int] = {"synthetic": 0, "live": 1, "allowlist": 2} +_HOST = re.compile(r"(\*\.)?([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?\.)*[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?") +_NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_-]*(\.[A-Za-z_][A-Za-z0-9_-]*)*") +_SDK_REF = re.compile(r"[A-Za-z_][\w.]*:[A-Za-z_<][\w.<>]*") +_SECRET_REF = re.compile(r"(secret|env):[A-Za-z_][A-Za-z0-9_./-]*") +_PATH = re.compile(r"/(?:[A-Za-z0-9._~!$&'()*+,;=:@/{}-]|%[0-9A-Fa-f]{2})*") +_PATH_FIELDS: tuple[tuple[str, ...], ...] = ( + ("source", "image", "dockerfile"), ("source", "image", "context"), ("source", "static"), ("source", "assets"), + ("source", "replay", "capture"), ("data", "schema"), ("routes", "openapi")) +_REMOVABLE = {("routes", "ops"), ("roles",), ("data", "collections")} + + +class SpecError(ValueError): + """A spec that can't be read or doesn't hold together; ``problems`` lists each, ``: ``.""" + + def __init__(self, problems: Iterable[str]) -> None: + self.problems = list(problems) + super().__init__("\n".join(self.problems)) + + +class _Section(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True, populate_by_name=True) + + +class Site(_Section): + """Who the site is: its name (the card's), the host names it answers for, ``*.`` as a leftmost label matching + any subdomain, and the path that answers 200 once it's usable.""" + + name: Optional[str] = None + domains: tuple[str, ...] = () + health: Optional[str] = None + + @field_validator("domains") + @classmethod + def _domains(cls, domains: tuple[str, ...]) -> tuple[str, ...]: + bad = [d for d in domains if not _HOST.fullmatch(d)] + if bad: + raise ValueError(f"{bad} aren't host names in lower case, '*.' only as the leftmost label") + return domains + + @field_validator("health") + @classmethod + def _health(cls, health: Optional[str]) -> Optional[str]: + if health is not None and (not _PATH.fullmatch(health) or health == "/" or reserved_path(health)): + raise ValueError(f"health is a path of its own under '/', not {health!r}") + return health + + +class ImageSource(_Section): + """A server image built from a Dockerfile, both relative to the spec.""" + + dockerfile: str + context: str = "." + + +class LiveSource(_Section): + """The real site, reached through the env: the hosts besides its domains it may call, and the credentials it + signs in with, each a ``secret:`` or ``env:`` reference, never a value.""" + + allow_hosts: tuple[str, ...] = () + credentials: dict[str, str] = {} + + @field_validator("allow_hosts") + @classmethod + def _hosts(cls, hosts: tuple[str, ...]) -> tuple[str, ...]: + return Site._domains(hosts) + + @field_validator("credentials") + @classmethod + def _credentials(cls, credentials: dict[str, str]) -> dict[str, str]: + bad = sorted(name for name, ref in credentials.items() if not _SECRET_REF.fullmatch(ref)) + if bad: + raise ValueError(f"credentials {bad} must be secret: or env: references, not values") + return credentials + + +class ReplaySource(_Section): + """A capture of the site served back: the artifact, relative to the spec, and the hosts still let through.""" + + capture: str + allow_hosts: tuple[str, ...] = () + + @field_validator("allow_hosts") + @classmethod + def _hosts(cls, hosts: tuple[str, ...]) -> tuple[str, ...]: + return Site._domains(hosts) + + +class Source(_Section): + """What serves the site, exactly one of: an ``image`` (a reference, or a table building one), an ``sdk`` + WebsiteEnvironment class as ``module:Class`` with its static ``assets`` folder, a ``static`` folder, the + ``live`` real site, or a ``replay`` of a capture.""" + + image: Union[str, ImageSource, None] = None + sdk: Optional[str] = None + static: Optional[str] = None + live: Optional[LiveSource] = None + replay: Optional[ReplaySource] = None + assets: Optional[str] = None + + @model_validator(mode="after") + def _one(self) -> Source: + given = [kind for kind in SOURCE_KINDS if getattr(self, kind) is not None] + if len(given) != 1: + raise ValueError(f"names exactly one of {', '.join(SOURCE_KINDS)}, not {given or 'none'}") + if self.sdk is not None and not _SDK_REF.fullmatch(self.sdk): + raise ValueError(f"sdk is module:Class, not {self.sdk!r}") + if self.assets is not None and self.sdk is None: + raise ValueError("assets is the static folder of an sdk source") + return self + + @property + def kind(self) -> str: + return next(kind for kind in SOURCE_KINDS if getattr(self, kind) is not None) + + +class Collection(_Section): + """One collection of the site's data: the field naming each object (``key``), the field naming the user who + owns one (``owner``), and a JSON Schema of an object, inline or a file relative to the spec.""" + + key: str = "id" + owner: Optional[str] = None + schema_: Union[str, dict, None] = Field(default=None, alias="schema") + + +class Data(_Section): + """The site's data: its collections, or one JSON Schema of the whole state, or ``models = "mcp:"`` + binding it to an MCP server's schema; and the seed files it starts with, in order.""" + + collections: dict[str, Collection] = {} + schema_: Union[str, dict, None] = Field(default=None, alias="schema") + models: Optional[str] = None + seed: tuple[str, ...] = () + + @field_validator("seed", mode="before") + @classmethod + def _seed(cls, seed: Any) -> Any: + return (seed,) if isinstance(seed, str) else seed + + @field_validator("models") + @classmethod + def _models(cls, models: Optional[str]) -> Optional[str]: + if models is not None and not re.fullmatch(r"mcp:[A-Za-z0-9_.-]+", models): + raise ValueError(f"models is mcp:, not {models!r}") + return models + + @model_validator(mode="after") + def _schema_or_models(self) -> Data: + if self.schema_ is not None and self.models is not None: + raise ValueError("data takes a schema or models, not both") + if bad := sorted(name for name in self.collections if not _NAME.fullmatch(name)): + raise ValueError(f"collection names {bad} aren't names") + return self + + +class Persona(_Section): + id: str + name: Optional[str] = None + role: Optional[str] = None + + +class Users(_Section): + """Who uses the site: its personas, the acting user a browser with no session is signed in as, and how a + browser signs in: ``cookie`` (the site's signed session cookie, as ``session_path`` hands it out), ``none``, or + ``live``, the real site's own sign-in.""" + + personas: tuple[Persona, ...] = () + acting_user: Optional[str] = None + sign_in: Literal["cookie", "none", "live"] = "cookie" + session_cookie: Optional[str] = None + session_path: Optional[str] = None + + @field_validator("personas", mode="before") + @classmethod + def _personas(cls, personas: Any) -> Any: + return [{"id": p} if isinstance(p, str) else p for p in personas] if isinstance(personas, list) else personas + + @model_validator(mode="after") + def _consistent(self) -> Users: + ids = [p.id for p in self.personas] + if twice := sorted({i for i in ids if ids.count(i) > 1}): + raise ValueError(f"personas {twice} are declared twice") + if self.acting_user is not None and ids and self.acting_user not in ids: + raise ValueError(f"acting_user {self.acting_user!r} is none of the personas {ids}") + if self.sign_in != "cookie" and (self.session_cookie or self.session_path): + raise ValueError("session_cookie and session_path are a cookie sign-in's") + return self + + +class Route(_Section): + """One named operation: its verb, its OpenAPI-style path, the roles it answers (any when empty), and, for a site + served by a live or replayed source, the ``module:Class`` whose route of the same name answers it instead.""" + + method: str + path: str + roles: tuple[str, ...] = () + served_by: Optional[str] = None + + @model_validator(mode="before") + @classmethod + def _shorthand(cls, value: Any) -> Any: + if isinstance(value, str): + method, _, path = value.strip().partition(" ") + value = {"method": method, "path": path.strip()} + if isinstance(value, dict): + value = dict(value) + if isinstance(value.get("method"), str): + value["method"] = value["method"].upper() + if isinstance(value.get("path"), str): + value["path"] = openapi_path(value["path"]) + return value + + @model_validator(mode="after") + def _route(self) -> Route: + if self.method not in METHODS: + raise ValueError(f"method is one of {', '.join(METHODS)}, not {self.method!r}") + if not _PATH.fullmatch(self.path): + raise ValueError(f"path starts with '/', not {self.path!r}") + if reserved_path(self.path): + raise ValueError(f"path {self.path!r} is the data plane's or the card's") + if self.served_by is not None and not _SDK_REF.fullmatch(self.served_by): + raise ValueError(f"served_by is module:Class, not {self.served_by!r}") + return self + + def advertised(self, name: str) -> dict: + return {"method": self.method, "path": self.path, "name": name, "roles": list(self.roles) or None} + + +class Routes(_Section): + """The site's named operations, ``name = "METHOD /path"`` or a table, and an OpenAPI document whose operations, + by ``operationId``, are routes too; the names are what triggers and records call ``op``.""" + + openapi: Optional[str] = None + ops: dict[str, Route] = {} + + @field_validator("ops") + @classmethod + def _names(cls, ops: dict[str, Route], info: ValidationInfo) -> dict[str, Route]: + if (info.context or {}).get("declared_routes"): + return ops + if bad := sorted(name for name in ops if not _NAME.fullmatch(name)): + raise ValueError(f"route names {bad} aren't names") + return ops + + +class Clock(_Section): + """How the site follows the universe's time: ``header`` (its answers carry ``X-Universe-Now`` and its pages read + it), ``date_patch`` (its pages' ``Date`` is shifted too), or ``none`` (wall time).""" + + follow: Literal["header", "date_patch", "none"] = "header" + + +class Use(_Section): + """A component: another spec, relative to this one, mounted under ``mount``, named ``as`` (its site's name when + left out), with ``data`` mapping a collection of the component's onto one of this site's.""" + + spec: str + mount: str = "/" + as_: Optional[str] = Field(default=None, alias="as") + data: dict[str, str] = {} + + @field_validator("mount") + @classmethod + def _mount(cls, mount: str) -> str: + if not _PATH.fullmatch(mount) or "{" in mount or reserved_path(mount): + raise ValueError(f"mount is a plain path under '/', not {mount!r}") + return mount.rstrip("/") or "/" + + +@dataclasses.dataclass(frozen=True) +class Component: + """A component a resolved spec mounted: its name, its path prefix, and its own resolved spec.""" + + name: str + mount: str + spec: WebsiteSpec + + +class WebsiteSpec(_Section): + """A site as one declaration. ``load`` reads and resolves one; a spec built from a dict is resolved by + ``resolve``. Paths in it are relative to ``base_dir``.""" + + extends: Optional[str] = None + site: Site = Site() + source: Optional[Source] = None + data: Data = Data() + users: Users = Users() + roles: dict[str, tuple[str, ...]] = {} + routes: Routes = Routes() + clock: Optional[Clock] = None + uses: tuple[Use, ...] = () + + _base_dir: Path = PrivateAttr(default_factory=Path.cwd) + _components: tuple[Component, ...] = PrivateAttr(default=()) + _declared: Optional[dict] = PrivateAttr(default=None) + _declared_routes: bool = PrivateAttr(default=False) + + def __eq__(self, other: object) -> bool: + """Two specs are equal when they resolve to the same site in the same folder, however each was declared.""" + if not isinstance(other, WebsiteSpec): + return NotImplemented + return (type(self), self.__dict__, self._base_dir, self._components) == ( + type(other), other.__dict__, other._base_dir, other._components) + + __hash__ = None + + @property + def base_dir(self) -> Path: + return self._base_dir + + @property + def components(self) -> tuple[Component, ...]: + return self._components + + @property + def name(self) -> Optional[str]: + return self.site.name + + @property + def health(self) -> str: + return self.site.health or DEFAULT_HEALTH_PATH + + def path(self, relative: str) -> Path: + """``relative``, a path the spec names, as a path on disk.""" + given = Path(relative) + return given if given.is_absolute() else self._base_dir / given + + def route_table(self) -> dict[str, Route]: + """Every route by name, each with the roles ``[roles]`` grants it added to its own.""" + granted: dict[str, list[str]] = {} + for role, names in self.roles.items(): + for name in names: + granted.setdefault(name, []).append(role) + return {name: route.model_copy(update={"roles": tuple(dict.fromkeys([*route.roles, *granted.get(name, [])]))}) + for name, route in self.routes.ops.items()} + + def to_dict(self) -> dict: + """The spec as a JSON-able dict, empty sections left out, which ``resolve`` reads back as the same spec.""" + return _pruned(self.model_dump(mode="json", by_alias=True, exclude_none=True)) + + def overlay(self, changes: dict, *, base_dir: Path | str | None = None) -> WebsiteSpec: + """This spec with ``changes`` merged over it as an ``extends`` overlay merges, then resolved again; paths in + ``changes`` are relative to ``base_dir``, this spec's own when left out. A spec ``resolve`` or ``load`` made + is merged into as it was declared, its ``[[uses]]`` not yet mounted, so a component of the same ``as`` + replaces the mounted one as it would in a file overlay. A spec read from a class or a card keeps the route + names its server serves, as it was read.""" + here = Path(base_dir) if base_dir is not None else self._base_dir + declared = self._declared if self._declared is not None else self.to_dict() + merged = _merge(declared, _expanded(_rebased(changes, here, self._base_dir))) + return resolve(merged, self._base_dir, declared_routes=self._declared_routes) + + def to_card_extension(self) -> dict: + """The ``urn:agentenv:website/v1`` params a server of this spec advertises: its domains, health path and + routes by name, its session cookie and path for a cookie sign-in, and how its clock is followed, when + declared.""" + params: dict = {"domains": list(self.site.domains), "health": self.health, + "routes": [route.advertised(name) for name, route in self.route_table().items()]} + if self.users.sign_in == "cookie": + params["session_cookie"] = self.users.session_cookie or session_cookie_name(self.site.name) + params["session_path"] = self.users.session_path or SESSION_PATH + if self.clock is not None: + params["clock"] = {"follow": self.clock.follow} + return params + + +def load(path: str | Path) -> WebsiteSpec: + """The spec in the TOML file at ``path``, resolved: its ``extends`` merged, its ``uses`` composed, checked.""" + return _load(Path(path).resolve(), ()) + + +def resolve(raw: dict, base_dir: str | Path, *, origin: str = "spec", declared_routes: bool = False) -> WebsiteSpec: + """The spec ``raw`` declares, paths in it relative to ``base_dir``, resolved as ``load`` resolves a file's. With + ``declared_routes``, its routes are the ones a class's ``@api`` decorators declare, named as they take any name.""" + return _resolve(copy.deepcopy(raw), Path(base_dir).resolve(), origin, (), declared_routes=declared_routes) + + +def _load(path: Path, stack: tuple[Path, ...]) -> WebsiteSpec: + return _resolve(_read(path), path.parent, str(path), stack + (path,)) + + +def _read(path: Path) -> dict: + if tomllib is None: + raise SpecError([f"{path}: reading a TOML spec needs Python 3.11's tomllib, or tomli before it"]) + try: + return tomllib.loads(path.read_text(encoding="utf-8")) + except OSError as e: + raise SpecError([f"{path}: can't be read: {e.strerror or e}"]) from None + except tomllib.TOMLDecodeError as e: + raise SpecError([f"{path}: isn't TOML: {e}"]) from None + + +def _resolve(raw: dict, base_dir: Path, origin: str, stack: tuple[Path, ...], *, + declared_routes: bool = False) -> WebsiteSpec: + """The spec ``raw`` declares, resolved. A spec that uses components has its acting user checked only once their + personas are merged into its own, since the acting user may be one only a component declares.""" + raw = _extended(raw, base_dir, origin, stack) + declared = _expanded(copy.deepcopy(raw)) + removed = _removals(raw) + problems: list[str] = [] + imported = _imported(raw, base_dir, removed, problems) + users = raw.get("users") + spec = _validated({**raw, "users": {k: v for k, v in users.items() if k != "acting_user"}} + if raw.get("uses") and isinstance(users, dict) else raw, origin, declared_routes=declared_routes) + spec._base_dir = base_dir + ops = dict(imported) + for name, route in spec.routes.ops.items(): + if name in ops and (ops[name].method, ops[name].path) != (route.method, route.path): + problems.append(f"routes.ops.{name}: {route.method} {route.path} isn't the openapi document's " + f"{ops[name].method} {ops[name].path}") + ops[name] = route + raw_routes = raw.setdefault("routes", {}) + raw_routes["ops"] = {name: route.model_dump(mode="json", exclude_none=True, exclude_defaults=True) | { + "method": route.method, "path": route.path} for name, route in ops.items()} + raw_routes.pop("openapi", None) + components = [] + for index, use in enumerate(spec.uses): + component = _component(use, index, spec, stack, problems) + if component is not None: + components.append(component) + _mount(raw, component, use, base_dir, f"uses[{index}]", problems) + raw.pop("uses", None) + if problems: + raise SpecError(f"{origin}: {p}" for p in problems) + resolved = _validated(raw, origin, declared_routes=declared_routes) + resolved._base_dir, resolved._components, resolved._declared = base_dir, tuple(components), declared + resolved._declared_routes = declared_routes + problems = _check(resolved) + if problems: + raise SpecError(f"{origin}: {p}" for p in problems) + return resolved + + +def _extended(raw: dict, base_dir: Path, origin: str, stack: tuple[Path, ...]) -> dict: + """``raw`` merged over the spec its ``extends`` names, recursively, that spec's paths made relative to + ``base_dir``.""" + parent = raw.pop("extends", None) + if parent is None: + return raw + if not isinstance(parent, str): + raise SpecError([f"{origin}: extends is the path of a spec, not {parent!r}"]) + path = (base_dir / parent).resolve() + if path in stack: + raise SpecError([f"{origin}: extends {parent} is a cycle: {' -> '.join(str(p) for p in (*stack, path))}"]) + base = _extended(_read(path), path.parent, str(path), stack + (path,)) + return _merge(_expanded(_rebased(base, path.parent, base_dir)), _expanded(raw)) + + +def _expanded(raw: dict) -> dict: + """``raw`` with each ``"METHOD /path"`` route written as its table, so an overlay can change one key of it.""" + ops = (raw.get("routes") or {}).get("ops") if isinstance(raw.get("routes"), dict) else None + if isinstance(ops, dict): + for name, route in ops.items(): + if isinstance(route, str): + method, _, path = route.strip().partition(" ") + ops[name] = {"method": method, "path": path.strip()} + return raw + + +def _removals(raw: dict) -> set[str]: + """The names of the routes ``raw``'s merged overlays removed, each a ``false`` left in ``routes.ops`` so it + removes an operation of the OpenAPI document too, taken out of ``raw``.""" + routes = raw.get("routes") + ops = routes.get("ops") if isinstance(routes, dict) else None + if not isinstance(ops, dict): + return set() + removed = {name for name, route in ops.items() if route is False} + for name in removed: + del ops[name] + return removed + + +def _validated(raw: dict, origin: str, *, declared_routes: bool = False) -> WebsiteSpec: + try: + return WebsiteSpec.model_validate(raw, context={"declared_routes": declared_routes}) + except ValidationError as e: + raise SpecError(f"{origin}: {_where(err['loc'])}: {_why(err)}" for err in e.errors()) from None + + +def _where(loc: tuple) -> str: + where = "" + for part in loc: + if isinstance(part, int): + where += f"[{part}]" + elif part not in ("str", "dict", "ImageSource") and "[" not in part: + where += f".{part}" if where else part + return where or "spec" + + +def _why(err: dict) -> str: + message = err["msg"] + return message.removeprefix("Value error, ") if err["type"] != "extra_forbidden" else "isn't a key the spec takes" + + +def _merge(base: Any, over: Any, at: tuple[str, ...] = ()) -> Any: + """``over`` merged over ``base``: tables key by key, ``[source]`` whole, ``false`` removing an entry of + routes, roles or collections, ``[[uses]]`` added by ``as``, and anything else replaced. A removed route stays a + ``false``, so that resolving the merge removes it from the OpenAPI document's operations as well.""" + if not (isinstance(base, dict) and isinstance(over, dict)) or at == ("source",): + return copy.deepcopy(over) + merged = copy.deepcopy(base) + for key, value in over.items(): + if value is False and at == ("routes", "ops"): + merged[key] = False + elif value is False and at in _REMOVABLE: + merged.pop(key, None) + elif key == "uses" and not at and isinstance(value, list) and isinstance(merged.get("uses", []), list): + kept = {_use_id(u): u for u in merged.get("uses", [])} + kept.update({_use_id(u): copy.deepcopy(u) for u in value}) + merged["uses"] = list(kept.values()) + elif key in merged: + merged[key] = _merge(merged[key], value, (*at, key)) + else: + merged[key] = copy.deepcopy(value) + return merged + + +def _use_id(use: Any) -> str: + return str(use.get("as") or use.get("spec")) if isinstance(use, dict) else repr(use) + + +def _rebased(raw: dict, from_dir: Path, to_dir: Path) -> dict: + """``raw`` with each relative path it names, relative to ``from_dir``, made relative to ``to_dir``; an image + source's context is moved even when left to its default, so it stays the folder its own spec meant.""" + if from_dir == to_dir: + return copy.deepcopy(raw) + raw = copy.deepcopy(raw) + + def moved(value: Any) -> Any: + return _moved(value, from_dir, to_dir) + + image = raw.get("source", {}).get("image") if isinstance(raw.get("source"), dict) else None + if isinstance(image, dict): + image.setdefault("context", ImageSource.model_fields["context"].default) + for keys in _PATH_FIELDS: + holder = raw + for key in keys[:-1]: + holder = holder.get(key) if isinstance(holder, dict) else None + if isinstance(holder, dict) and keys[-1] in holder: + holder[keys[-1]] = moved(holder[keys[-1]]) + data = raw.get("data") if isinstance(raw.get("data"), dict) else {} + if isinstance(data.get("seed"), (str, list)): + data["seed"] = moved(data["seed"]) if isinstance(data["seed"], str) else [moved(s) for s in data["seed"]] + collections = data.get("collections") + for collection in collections.values() if isinstance(collections, dict) else (): + if isinstance(collection, dict) and "schema" in collection: + collection["schema"] = moved(collection["schema"]) + uses = raw.get("uses") + for use in uses if isinstance(uses, list) else (): + if isinstance(use, dict) and "spec" in use: + use["spec"] = moved(use["spec"]) + return raw + + +def _moved(value: Any, from_dir: Path, to_dir: Path) -> Any: + """``value``, a path relative to ``from_dir``, made relative to ``to_dir``; anything else as it is.""" + if not isinstance(value, str) or Path(value).is_absolute(): + return value + return os.path.relpath(from_dir / value, to_dir) + + +def _imported(raw: dict, base_dir: Path, removed: set[str], problems: list[str]) -> dict[str, Route]: + """The routes of ``raw``'s OpenAPI document that no overlay removed, each of ``raw``'s ``routes.ops`` tables + naming one of them written over it, so an overlay can change one key of an imported operation as it can of a + declared one.""" + routes = raw.get("routes") if isinstance(raw.get("routes"), dict) else {} + document = routes.get("openapi") + if not isinstance(document, str): + return {} + imported = {name: route for name, route in _openapi_routes(document, base_dir, problems).items() + if name not in removed} + ops = routes.get("ops") + for name, route in (ops.items() if isinstance(ops, dict) else ()): + if isinstance(route, dict) and name in imported: + ops[name] = imported[name].model_dump(mode="json", exclude_none=True) | route + return imported + + +def _openapi_routes(document: str, base_dir: Path, problems: list[str]) -> dict[str, Route]: + """The routes of the OpenAPI document at ``document``, relative to ``base_dir``, by operationId, roles from + ``x-agentenv-roles``; a document of another shape, an operation without a name and a name two operations share + are problems.""" + try: + parsed = json.loads((base_dir / document).read_text(encoding="utf-8")) + except (OSError, ValueError) as e: + problems.append(f"routes.openapi: {document} isn't a readable JSON document: {e}") + return {} + paths = (parsed.get("paths") or {}) if isinstance(parsed, dict) else None + if not isinstance(paths, dict): + problems.append(f"routes.openapi: {document} isn't an OpenAPI document, an object whose paths " + "are an object") + return {} + routes: dict[str, Route] = {} + for path, operations in paths.items(): + if not isinstance(operations, dict): + problems.append(f"routes.openapi: path {path} isn't an object of operations") + continue + for method, operation in operations.items(): + if method.upper() not in METHODS or not isinstance(operation, dict): + continue + name = operation.get("operationId") + if not name or not isinstance(name, str): + problems.append(f"routes.openapi: {method.upper()} {path} has no operationId to name it by") + continue + if name in routes: + problems.append(f"routes.openapi: {name} names both {routes[name].method} {routes[name].path} and " + f"{method.upper()} {path}") + continue + try: + routes[name] = Route(method=method, path=path, roles=operation.get("x-agentenv-roles") or ()) + except ValidationError as e: + problems.append(f"routes.openapi: {name}: {_why(e.errors()[0])}") + return routes + + +def _component(use: Use, index: int, host: WebsiteSpec, stack: tuple[Path, ...], + problems: list[str]) -> Optional[Component]: + path = host.path(use.spec).resolve() + if path in stack: + problems.append(f"uses[{index}]: {use.spec} is a cycle: {' -> '.join(str(p) for p in (*stack, path))}") + return None + try: + spec = _load(path, stack) + except SpecError as e: + problems.extend(f"uses[{index}]: {p}" for p in e.problems) + return None + name = use.as_ or spec.site.name + if name is None: + problems.append(f"uses[{index}]: {use.spec} has no site.name; name it with as") + return None + return Component(name, use.mount, spec) + + +def _mount(raw: dict, component: Component, use: Use, base_dir: Path, where: str, problems: list[str]) -> None: + """Merge ``component`` into ``raw``, whose paths are relative to ``base_dir``: its routes prefixed and + namespaced, its collections namespaced or mapped, their schema files made relative to ``base_dir``, its personas + added; every clash is a problem.""" + ns, spec = component.name, component.spec + ops = raw["routes"]["ops"] + taken = {(r["method"], r["path"]): name for name, r in ops.items()} + if use.mount != "/": + for name, r in ops.items(): + if r["path"] == use.mount or r["path"].startswith(use.mount + "/"): + problems.append(f"{where}: route {name} ({r['method']} {r['path']}) is under {use.mount}, where " + f"{ns} is mounted") + for name, route in spec.route_table().items(): + full, path = f"{ns}.{name}", _prefixed(use.mount, route.path) + if full in ops: + problems.append(f"{where}: route {full} is declared by the site and by component {ns}") + elif (route.method, path) in taken: + problems.append(f"{where}: {ns}'s route {name} is {route.method} {path}, which {taken[(route.method, path)]} " + "already serves") + else: + ops[full] = {**route.model_dump(mode="json", exclude_none=True, exclude_defaults=True), + "method": route.method, "path": path} + taken[(route.method, path)] = full + data = raw.setdefault("data", {}) + collections = data.setdefault("collections", {}) + if missing := sorted(set(use.data) - set(spec.data.collections)): + problems.append(f"{where}: data maps {missing}, which {ns} doesn't declare") + for name, collection in spec.data.collections.items(): + target = use.data.get(name, f"{ns}.{name}") + theirs = collection.model_dump(mode="json", by_alias=True, exclude_none=True) + if "schema" in theirs: + theirs["schema"] = _moved(theirs["schema"], spec.base_dir, base_dir) + ours = collections.get(target) + if ours is None: + collections[target] = theirs + continue + merged = {**theirs, **ours} + clash = [k for k in ("key", "owner", "schema") if k in ours and k in theirs and ours[k] != theirs[k]] + clash += ["key"] if "key" not in ours and theirs.get("key", "id") != "id" else [] + if clash: + problems.append(f"{where}: {ns}'s collection {name} maps onto {target}, whose {', '.join(clash)} differ") + collections[target] = merged + users = raw.setdefault("users", {}) + personas = {(p["id"] if isinstance(p, dict) else p): ({"id": p} if isinstance(p, str) else p) + for p in users.get("personas", [])} + for persona in spec.users.personas: + theirs, ours = persona.model_dump(mode="json", exclude_none=True), personas.get(persona.id, {}) + if any(key in ours and ours[key] != value for key, value in theirs.items()): + problems.append(f"{where}: persona {persona.id} is declared otherwise by {ns}") + personas[persona.id] = {**theirs, **ours} + if personas: + users["personas"] = list(personas.values()) + sign_in = users.get("sign_in", "cookie") + if spec.users.sign_in not in ("none", sign_in): + problems.append(f"{where}: {ns} signs in by {spec.users.sign_in}, the site by {sign_in}") + + +def _prefixed(mount: str, path: str) -> str: + return path if mount == "/" else mount + path + + +def _check(spec: WebsiteSpec) -> list[str]: + """What doesn't hold together across sections of a resolved spec.""" + problems = [] + table = spec.route_table() + seen: dict[tuple[str, str], str] = {} + for name, route in table.items(): + if (route.method, route.path) in seen: + problems.append(f"routes.ops: {name} and {seen[route.method, route.path]} are both {route.method} {route.path}") + seen.setdefault((route.method, route.path), name) + if ("GET", spec.health) in seen: + problems.append(f"site.health: {spec.health} is served by route {seen['GET', spec.health]}") + for role, names in spec.roles.items(): + if unknown := [n for n in names if n not in table]: + problems.append(f"roles.{role}: names no route {unknown}") + kind = spec.source.kind if spec.source is not None else None + if spec.users.sign_in == "live" and kind not in ("live", "replay"): + problems.append("users.sign_in: live is the sign-in of a live or replayed source") + if kind == "sdk" and spec.clock is not None and spec.clock.follow == "none": + problems.append("clock.follow: an sdk site follows the universe's clock; none is a live site's") + if kind == "static" and table: + problems.append("routes: a static site serves no routes") + if served := [n for n, r in table.items() if r.served_by is not None and kind not in ("live", "replay")]: + problems.append(f"routes.ops: served_by swaps a route of a live or replayed site, not of {kind or 'no'} " + f"source ({served})") + for name, collection in spec.data.collections.items(): + if collection.owner is not None and spec.users.sign_in == "none": + problems.append(f"data.collections.{name}: owner names a user, and the site signs nobody in") + return problems + + +def _pruned(value: Any, at: tuple[str, ...] = ()) -> Any: + """``value`` without its empty tables and arrays, but for a JSON Schema, which is kept as it is.""" + if _is_schema(at): + return value + if isinstance(value, dict): + pruned = {k: _pruned(v, (*at, k)) for k, v in value.items()} + return {k: v for k, v in pruned.items() if _is_schema((*at, k)) or v not in ({}, [], ())} + if isinstance(value, list): + return [_pruned(v, at) for v in value] + return value + + +def _is_schema(at: tuple[str, ...]) -> bool: + return at == ("data", "schema") or (len(at) == 4 and at[:2] == ("data", "collections") and at[3] == "schema") + + +def from_card(card: dict) -> WebsiteSpec: + """The spec a running site's card declares: its name, domains, health path, routes and sign-in, and its clock + when the card says it. A card says nothing of a site's source, data or personas.""" + extension = find_extension(card, WEBSITE_EXTENSION_URI) + if extension is None: + raise SpecError([f"the card declares no {WEBSITE_EXTENSION_URI}"]) + params = extension.get("params") or {} + raw: dict = {"site": {"name": card.get("name"), "domains": [d.lower() if isinstance(d, str) else d + for d in params.get("domains") or []], + "health": params.get("health")}, + "routes": {"ops": {r["name"]: {"method": r["method"], "path": r["path"], "roles": r.get("roles") or []} + for r in params.get("routes") or []}}} + if params.get("session_cookie"): + raw["users"] = {"sign_in": "cookie"} + if params["session_cookie"] != session_cookie_name(card.get("name")): + raw["users"]["session_cookie"] = params["session_cookie"] + if params.get("session_path") not in (None, SESSION_PATH): + raw["users"]["session_path"] = params["session_path"] + else: + raw["users"] = {"sign_in": "none"} + if isinstance(params.get("clock"), dict): + raw["clock"] = params["clock"] + return resolve(_pruned(raw), Path.cwd(), origin=f"card {card.get('name')!r}", declared_routes=True) + + +def from_environment(cls: type) -> WebsiteSpec: + """The spec a WebsiteEnvironment class's decorators declare: ``@environment_card``'s name, ``@website``'s + domains, health path, static folder, seed and clock patch, its ``@api`` routes and a cookie sign-in.""" + declaration = cls.declaration() + if declaration.spec is not None: + base = declaration.spec + raw = base.to_dict() + else: + base, raw = None, {} + card_name = (getattr(cls, _CARD_CONFIG_ATTR, None) or {}).get("name") + routes = cls.routes() + site_file = Path(inspect.getfile(cls)).resolve().parent + raw["site"] = {**raw.get("site", {}), "domains": [d.lower() for d in declaration.domains], + "health": declaration.health} + if card_name or raw["site"].get("name") is None: + raw["site"]["name"] = card_name or cls.__name__ + raw["source"] = {"sdk": f"{cls.__module__}:{cls.__qualname__}", + **({"assets": str(cls._site_path(declaration.static))} if declaration.static else {})} + seeds = [str(path) for path in cls._seed_files()] + if seeds: + raw.setdefault("data", {})["seed"] = seeds + acting = _seed_user(seeds) + if acting is not None and "acting_user" not in raw.get("users", {}): + raw.setdefault("users", {})["acting_user"] = acting + raw.setdefault("users", {})["sign_in"] = "cookie" + raw["routes"] = {"ops": {r.name: {"method": r.method, "path": openapi_path(r.path), "roles": list(r.roles)} + for r in routes}} + raw.pop("roles", None) + raw["clock"] = {"follow": "date_patch" if declaration.clock_patch else "header"} + return resolve(raw, base.base_dir if base is not None else site_file, origin=cls.__qualname__, declared_routes=True) + + +def _seed_user(paths: Iterable[str | Path]) -> Optional[str]: + """The ``current_user`` the store is left with once the seed files at ``paths`` are loaded in order, each that + has one taking its place.""" + user = None + for path in paths: + try: + data = json.loads(Path(path).read_text(encoding="utf-8")) + except (OSError, ValueError): + continue + if isinstance(data, dict) and "current_user" in data: + user = data["current_user"] if isinstance(data["current_user"], str) else None + return user + + +def problems_with_environment(spec: WebsiteSpec, cls: type) -> list[str]: + """Where a WebsiteEnvironment class doesn't serve what ``spec`` declares: its source, routes by name, verb, + path and roles, sign-in and clock.""" + problems = [] + if spec.source is not None and spec.source.kind != "sdk": + problems.append(f"source: a WebsiteEnvironment is an sdk source, and the spec names {spec.source.kind}") + elif spec.source is not None and spec.source.sdk.rsplit(":", 1)[1] not in (cls.__qualname__, cls.__name__): + problems.append(f"source.sdk: {spec.source.sdk} names another class than {cls.__name__}") + derived = from_environment(cls) + ours = {name: (r.method, r.path, tuple(sorted(r.roles))) for name, r in spec.route_table().items()} + theirs = {name: (r.method, r.path, tuple(sorted(r.roles))) for name, r in derived.route_table().items()} + if spec.routes.ops: + for name in sorted(set(ours) | set(theirs)): + if name not in theirs: + problems.append(f"routes.ops.{name}: {' '.join(ours[name][:2])} isn't an @api route of {cls.__name__}") + elif name not in ours: + problems.append(f"routes.ops: {cls.__name__} serves {name} ({' '.join(theirs[name][:2])}), " + "which the spec doesn't declare") + elif ours[name] != theirs[name]: + problems.append(f"routes.ops.{name}: the spec says {_shown(ours[name])}, the class serves " + f"{_shown(theirs[name])}") + if spec.users.sign_in != "cookie": + problems.append(f"users.sign_in: a WebsiteEnvironment signs in by cookie, not {spec.users.sign_in}") + cookie = session_cookie_name(spec.site.name) + if spec.users.session_cookie not in (None, cookie) or spec.users.session_path not in (None, SESSION_PATH): + problems.append(f"users: a WebsiteEnvironment's session is {cookie} at {SESSION_PATH}") + acting = _seed_user(cls._seed_files()) + if spec.users.acting_user is not None and acting is not None and spec.users.acting_user != acting: + problems.append(f"users.acting_user: the spec says {spec.users.acting_user}, the seed's current_user {acting}") + return problems + + +def _shown(route: tuple) -> str: + method, path, roles = route + return f"{method} {path}" + (f" for {list(roles)}" if roles else "") + + +@dataclasses.dataclass(frozen=True) +class NameEntry: + """One host name pattern of a multi env's name table: the site it routes to and how that site is reached.""" + + pattern: str + site: str + kind: NameKind + + def matches(self, host: str) -> bool: + host = host.lower().rstrip(".") + if self.pattern.startswith("*."): + return host.endswith(self.pattern[1:]) + return host == self.pattern + + @property + def specificity(self) -> tuple[int, int]: + return (0, len(self.pattern)) if self.pattern.startswith("*.") else (1, len(self.pattern)) + + +@dataclasses.dataclass(frozen=True) +class NameTable: + """A multi env's host names: each pattern's winning entry, and each entry a higher kind shadowed, beside the + entry it lost to.""" + + entries: tuple[NameEntry, ...] + shadowed: tuple[tuple[NameEntry, NameEntry], ...] = () + + def resolve(self, host: str) -> Optional[NameEntry]: + """The entry ``host`` routes to: of those matching it, the highest kind's (synthetic, live, allowlist), + and among those the most specific pattern (an exact name, then the longest wildcard).""" + matching = [e for e in self.entries if e.matches(host)] + if not matching: + return None + return min(matching, key=lambda e: (_PRECEDENCE[e.kind], tuple(-n for n in e.specificity))) + + def as_dict(self) -> dict[str, dict]: + return {e.pattern: {"site": e.site, "kind": e.kind} for e in self.entries} + + +def name_table(specs: Iterable[WebsiteSpec]) -> NameTable: + """The one table of host names of a multi env's sites: each site's domains, ``synthetic`` for one the env + serves (image, sdk, static, replay) and ``live`` for a live source, and its source's ``allow_hosts`` as + ``allowlist``. A pattern two kinds claim goes to the higher, synthetic before live before allowlist; two sites + claiming one pattern at one kind is a conflict, but for an allowlist, which only lets traffic through. Raises + SpecError listing every conflict.""" + claims: dict[str, list[NameEntry]] = {} + problems = [] + names: set[str] = set() + for spec in specs: + if spec.site.name is None: + problems.append(f"a site with domains {list(spec.site.domains)} has no site.name") + continue + if spec.site.name in names: + problems.append(f"two sites are named {spec.site.name}") + names.add(spec.site.name) + source = spec.source + kind: NameKind = "live" if source is not None and source.kind == "live" else "synthetic" + allowed = (source.live.allow_hosts if source is not None and source.live is not None + else source.replay.allow_hosts if source is not None and source.replay is not None else ()) + for pattern in spec.site.domains: + claims.setdefault(pattern, []).append(NameEntry(pattern, spec.site.name, kind)) + for pattern in allowed: + claims.setdefault(pattern, []).append(NameEntry(pattern, spec.site.name, "allowlist")) + entries, shadowed = [], [] + for pattern, claimed in claims.items(): + clashes = [(kind, sorted({e.site for e in claimed if e.kind == kind})) for kind in ("synthetic", "live")] + clashes = [(kind, sites) for kind, sites in clashes if len(sites) > 1] + problems.extend(f"{pattern}: claimed as {kind} by {', '.join(sites)}" for kind, sites in clashes) + if clashes: + continue + best = min(_PRECEDENCE[e.kind] for e in claimed) + top = list(dict.fromkeys(e for e in claimed if _PRECEDENCE[e.kind] == best)) + winner = top[0] + entries.append(winner) + shadowed.extend((e, winner) for e in dict.fromkeys(claimed) if e != winner and e.site != winner.site) + if problems: + raise SpecError(problems) + return NameTable(tuple(entries), tuple(shadowed)) + + +def json_schema() -> dict: + """The JSON Schema of a spec file, with the string shorthands a route and a persona take, the partial route + table an overlay changes an inherited route's keys with, and the ``false`` that removes an inherited route, role + or collection.""" + schema = WebsiteSpec.model_json_schema(by_alias=True) + defs = schema.get("$defs", {}) + route_ref = {"$ref": "#/$defs/Route"} + either_case = "|".join("".join(f"[{c}{c.lower()}]" for c in method) for method in METHODS) + shorthand = {"type": "string", "pattern": rf"^\s*({either_case}) +/", "description": "METHOD /path"} + removed = {"const": False, "description": "removes the inherited entry"} + partial = {**{k: v for k, v in defs["Route"].items() if k != "required"}, + "title": "RouteChange", "description": "keys of an inherited route an overlay changes"} + defs["Routes"]["properties"]["ops"]["additionalProperties"] = {"anyOf": [route_ref, shorthand, removed, partial]} + for holder in (schema["properties"]["roles"], defs["Data"]["properties"]["collections"]): + holder["additionalProperties"] = {"anyOf": [holder["additionalProperties"], removed]} + defs["Users"]["properties"]["personas"]["items"] = {"anyOf": [{"$ref": "#/$defs/Persona"}, {"type": "string"}]} + defs["Data"]["properties"]["seed"] = {"anyOf": [{"type": "string"}, {"type": "array", "items": {"type": "string"}}], + "default": []} + return {"$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "urn:agentenv:website/v1:spec", + **schema} + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="python -m agentenv_protocol.website.spec", + description="Validate, show or describe a website/v1 spec.") + commands = parser.add_subparsers(dest="command", required=True) + commands.add_parser("validate", help="check a spec and everything it extends and uses").add_argument("path") + show = commands.add_parser("show", help="print the resolved spec as JSON") + show.add_argument("path") + show.add_argument("--card", action="store_true", help="print the website/v1 card params it advertises instead") + commands.add_parser("schema", help="print the JSON Schema of a spec file") + args = parser.parse_args(argv) + if args.command == "schema": + print(json.dumps(json_schema(), indent=2)) + return 0 + try: + spec = load(args.path) + except SpecError as e: + print("\n".join(e.problems), file=sys.stderr) + return 1 + if args.command == "show": + print(json.dumps(spec.to_card_extension() if args.card else spec.to_dict(), indent=2)) + return 0 + source = spec.source.kind if spec.source is not None else "no source" + mounted = f", {len(spec.components)} component(s)" if spec.components else "" + print(f"{args.path}: ok: {spec.site.name or 'unnamed site'}, {source}, {len(spec.routes.ops)} route(s), " + f"{len(spec.data.collections)} collection(s){mounted}") + return 0 diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/spec/__main__.py b/packages/agentenv-protocol/src/agentenv_protocol/website/spec/__main__.py new file mode 100644 index 000000000..15170e59a --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/spec/__main__.py @@ -0,0 +1,5 @@ +import sys + +from . import main + +sys.exit(main()) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/store.py b/packages/agentenv-protocol/src/agentenv_protocol/website/store.py new file mode 100644 index 000000000..bf767efc3 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/store.py @@ -0,0 +1,100 @@ +"""The default data store of a simple site: named collections of JSON objects, each with an ``id``. Its data is a JSON +object of collections by name, beside which a string ``current_user`` names the user a browser starts signed in as; +the data plane loads, exports and resets it.""" +from __future__ import annotations + +import base64 +import copy +import json +from pathlib import Path +from typing import Any, Iterable +from urllib.parse import unquote, urlparse + +from ..types import DataPart, FilePart, Part, TextPart + +CURRENT_USER = "current_user" + + +class Collections: + """Named lists of JSON objects; ``store["orders"]`` is the list itself, created empty on first use.""" + + def __init__(self, data: dict | None = None) -> None: + self._collections: dict[str, list[dict]] = {} + self.current_user: str | None = None + if data: + self.load(data) + + def __getitem__(self, name: str) -> list[dict]: + return self._collections.setdefault(name, []) + + def names(self) -> list[str]: + return list(self._collections) + + def get(self, name: str, id: Any) -> dict | None: + return next((doc for doc in self[name] if doc.get("id") == id), None) + + def find(self, name: str, **match: Any) -> list[dict]: + return [doc for doc in self[name] if all(doc.get(k) == v for k, v in match.items())] + + def insert(self, name: str, doc: dict) -> dict: + """Add ``doc``, giving it the next integer ``id`` of its collection when it has none.""" + doc = dict(doc) + if doc.get("id") is None: + doc["id"] = 1 + max((d["id"] for d in self[name] if isinstance(d.get("id"), int)), default=0) + self[name].append(doc) + return doc + + def update(self, name: str, id: Any, **fields: Any) -> dict | None: + doc = self.get(name, id) + if doc is not None: + doc.update(fields) + return doc + + def delete(self, name: str, id: Any) -> bool: + doc = self.get(name, id) + if doc is not None: + self[name].remove(doc) + return doc is not None + + def load(self, data: dict) -> None: + """Add ``data``'s collections to the store's, and take its ``current_user`` when it names one.""" + if not isinstance(data, dict): + raise ValueError("a store's data is a JSON object of collections by name") + for name, docs in data.items(): + if name == CURRENT_USER: + self.current_user = docs + elif isinstance(docs, list) and all(isinstance(d, dict) for d in docs): + self[name].extend(copy.deepcopy(docs)) + else: + raise ValueError(f"collection {name!r} is not a list of JSON objects") + + def load_parts(self, parts: Iterable[Part]) -> None: + for part in parts: + self.load(part_data(part)) + + def reset(self) -> None: + self._collections.clear() + self.current_user = None + + def export(self) -> dict: + data: dict = copy.deepcopy(self._collections) + if self.current_user is not None: + data[CURRENT_USER] = self.current_user + return data + + +def part_data(part: Part) -> dict: + """A part's JSON: a data part's data, a text part's text, or a file part's bytes or ``file://`` file.""" + if isinstance(part, DataPart): + return part.data + if isinstance(part, TextPart): + return json.loads(part.text) + if isinstance(part, FilePart): + file = part.file + if getattr(file, "bytes", None) is not None: + return json.loads(base64.b64decode(file.bytes)) + uri = urlparse(file.uri) + if uri.scheme == "file": + return json.loads(Path(unquote(uri.path)).read_text(encoding="utf-8")) + raise ValueError(f"the store reads file parts by their bytes or a file:// URI, not {file.uri!r}") + raise ValueError(f"the store can't read a {type(part).__name__}") diff --git a/packages/agentenv-protocol/tests/test_client.py b/packages/agentenv-protocol/tests/test_client.py index edcd38798..bfc65e9a7 100644 --- a/packages/agentenv-protocol/tests/test_client.py +++ b/packages/agentenv-protocol/tests/test_client.py @@ -190,6 +190,58 @@ def test_find_child_tolerates_missing_children(): assert protocol_v1.find_child({"children_environments": None}, "slack") is None +_SHARED_NAME = {"name": "gw", "children_environments": [ + {"name": "mail", "url": "/svc/mcp-mail/agentenv", "capabilities": {"operations": ["data/get"], "extensions": [ + {"uri": "urn:agentenv:clock/v1"}]}}, + {"name": "mail", "url": "/svc/mail/agentenv", "capabilities": {"operations": []}}, + {"name": "docs", "capabilities": {}}, + "not a card", +]} + + +def test_children_are_each_nested_card_by_its_name_url_and_card(): + found = protocol_v1.children(_SHARED_NAME) + + assert [(c.name, c.url) for c in found] == [("mail", "/svc/mcp-mail/agentenv"), ("mail", "/svc/mail/agentenv"), ("docs", None)] + assert found[0].card is _SHARED_NAME["children_environments"][0] + assert [c.extensions for c in found] == [[{"uri": "urn:agentenv:clock/v1"}], [], []] + assert [c.operations for c in found] == [["data/get"], [], None] + assert [c.serves("data/get") for c in found] == [True, False, None] + assert found[0].find_extension("urn:agentenv:clock/v1") is not None and found[1].find_extension("urn:agentenv:clock/v1") is None + assert protocol_v1.children({"name": "leaf"}) == [] and protocol_v1.children({"children_environments": None}) == [] + + +def test_a_childs_base_url_is_its_url_without_the_rpc_path_against_the_composed_cards_address(): + mcp, website, docs = protocol_v1.children(_SHARED_NAME) + + assert mcp.base_url("https://env.example/") == "https://env.example/svc/mcp-mail" + assert website.base_url("https://env.example") == "https://env.example/svc/mail" + assert docs.base_url("https://env.example") == "https://env.example" + assert protocol_v1.ChildEnv("x", "https://x.example/agentenv", {}).base_url("https://env.example") == "https://x.example" + + +def test_child_at_finds_a_child_by_its_url_or_data_plane_route_where_two_share_a_name(): + assert protocol_v1.child_at(_SHARED_NAME, "/svc/mail").url == "/svc/mail/agentenv" + assert protocol_v1.child_at(_SHARED_NAME, "/svc/mcp-mail/").url == "/svc/mcp-mail/agentenv" + assert protocol_v1.child_at(_SHARED_NAME, "/svc/mail/agentenv").url == "/svc/mail/agentenv" + assert protocol_v1.child_at(_SHARED_NAME, "/svc/docs") is None + assert protocol_v1.child_at({"name": "leaf"}, "/svc/mail") is None + + +def test_child_named_refuses_a_name_two_children_share(): + assert protocol_v1.child_named(_SHARED_NAME, "docs").name == "docs" + assert protocol_v1.child_named(_SHARED_NAME, "gmail") is None + with pytest.raises(protocol_v1.AmbiguousChildError, match="2 child envs are named 'mail'"): + protocol_v1.child_named(_SHARED_NAME, "mail") + assert protocol_v1.find_child(_SHARED_NAME, "mail")["url"] == "/svc/mcp-mail/agentenv" + + +def test_serves_is_what_a_card_declares_of_a_method_and_none_when_it_declares_no_operations(): + assert protocol_v1.serves({"capabilities": {"operations": ["data/get"]}}, "data/get") is True + assert protocol_v1.serves({"capabilities": {"operations": ["data/add"]}}, "data/get") is False + assert protocol_v1.serves({"capabilities": {}}, "data/get") is None + + def test_find_extension_ignores_children(): # set-errors/v1 is advertised only on the children, never at the top level assert protocol_v1.find_extension(_CARD_WITH_CHILDREN, "urn:agentenv:set-errors/v1") is None diff --git a/packages/agentenv-protocol/tests/test_website_api.py b/packages/agentenv-protocol/tests/test_website_api.py new file mode 100644 index 000000000..e461ce097 --- /dev/null +++ b/packages/agentenv-protocol/tests/test_website_api.py @@ -0,0 +1,428 @@ +"""The website SDK's @api routes: binding from type hints, JSON answers, errors, roles, and what a class may not +declare.""" + +from __future__ import annotations + +import datetime +import enum +from typing import Optional, Union + +import pytest +from pydantic import BaseModel, computed_field +from starlette.responses import PlainTextResponse +from starlette.testclient import TestClient + +from agentenv_protocol.website import ( + ROLE_KEY_ENV, + ROLE_KEY_HEADER, + ApiContext, + BadRequest, + Conflict, + NotFound, + WebsiteEnvironment, + api, + reserved_path, + website, +) + + +class Color(enum.Enum): + RED = "red" + BLUE = "blue" + + +class NewItem(BaseModel): + title: str + price: float = 0 + + +@website(domains=["shop.example"]) +class Shop(WebsiteEnvironment): + def __init__(self) -> None: + super().__init__() + self.items: dict[int, dict] = {1: {"id": 1, "title": "mug", "price": 3.5}} + + @api.get("/api/items/{item_id}", name="get_item") + async def get_item(self, item_id: int) -> dict: + """Read one item.""" + if item_id not in self.items: + raise NotFound(f"no item {item_id}") + return self.items[item_id] + + @api.get("/api/items") + def list_items(self, limit: int = 10, cheap: bool = False, color: Optional[Color] = None, + since: Optional[datetime.date] = None, tags: list[str] = []) -> dict: + return {"limit": limit, "cheap": cheap, "color": color.value if color else None, + "since": since.isoformat() if since else None, "tags": tags} + + @api.post("/api/items", name="create_item", status=201) + async def create_item(self, item: NewItem) -> NewItem: + if any(i["title"] == item.title for i in self.items.values()): + raise Conflict(f"{item.title} exists") + self.items[len(self.items) + 1] = {"id": len(self.items) + 1, **item.model_dump()} + return item + + @api.patch("/api/items/{item_id}") + async def patch_item(self, item_id: int, changes: dict) -> dict: + self.items[item_id].update(changes) + return self.items[item_id] + + @api.delete("/api/items/{item_id}", roles=["admin"]) + async def delete_item(self, item_id: int) -> None: + self.items.pop(item_id, None) + + @api.put("/api/teapot") + async def teapot(self) -> None: + raise BadRequest("short and stout", code="teapot", status=418) + + @api.get("/api/whoami") + async def whoami(self, ctx: ApiContext) -> dict: + return {"user": ctx.user, "role": ctx.role, "base_url": ctx.base_url, "q": ctx.query.get("q"), + "header": ctx.headers.get("x-probe"), "cookie": ctx.cookies.get("c")} + + @api.get("/api/plain") + async def plain(self) -> PlainTextResponse: + return PlainTextResponse("as is") + + @api.get("/api/boom") + async def boom(self) -> dict: + raise RuntimeError("secret detail") + + @api.get("/api/picked") + def picked(self, ids: Optional[list[int]] = None) -> dict: + return {"ids": ids} + + @api.delete("/api/cart", status=204) + async def empty_cart(self) -> None: + return None + + +@pytest.fixture +def client() -> TestClient: + return TestClient(Shop().create_app()) + + +def test_path_query_and_body_parameters_are_bound_from_type_hints(client: TestClient) -> None: + assert client.get("/api/items/1").json() == {"id": 1, "title": "mug", "price": 3.5} + assert client.get("/api/items").json() == {"limit": 10, "cheap": False, "color": None, "since": None, "tags": []} + listed = client.get("/api/items?limit=3&cheap=yes&color=blue&since=2030-01-02&tags=a&tags=b").json() + assert listed == {"limit": 3, "cheap": True, "color": "blue", "since": "2030-01-02", "tags": ["a", "b"]} + created = client.post("/api/items", json={"title": "pen", "price": 1}) + assert created.status_code == 201 and created.json() == {"title": "pen", "price": 1.0} + assert client.patch("/api/items/1", json={"price": 4}).json()["price"] == 4 + + +def test_a_request_its_parameters_cannot_be_bound_from_answers_400(client: TestClient) -> None: + for response in (client.get("/api/items/abc"), client.get("/api/items?limit=many"), + client.get("/api/items?color=green"), client.post("/api/items", json={"price": 2}), + client.post("/api/items", content=b"not json"), client.post("/api/items"), + client.patch("/api/items/1", json=[1, 2])): + assert response.status_code == 400, response.text + assert response.json()["code"] == "invalid_params" + assert "title" in client.post("/api/items", json={"price": 2}).json()["error"] + + +class Pin(BaseModel): + lat: float + + +class Place(BaseModel): + name: str + pin: Pin + + +class Note(BaseModel): + text: str + + +@website(domains=["map.example"]) +class Map(WebsiteEnvironment): + @api.post("/api/places") + def add_place(self, place: Place) -> list[Place]: + return [place] + + @api.post("/api/marks") + def mark(self, mark: Union[Note, Pin]) -> dict: + return {"kind": type(mark).__name__} + + @api.put("/api/settings") + def settings(self, settings: Optional[dict]) -> dict: + return {"settings": settings} + + +def test_a_body_is_validated_against_its_whole_annotation() -> None: + client = TestClient(Map().create_app()) + assert client.put("/api/settings", content=b"null", headers={"content-type": "application/json"}).json() == { + "settings": None} + assert client.post("/api/marks", json={"lat": 1.5}).json() == {"kind": "Pin"} + assert client.post("/api/marks", json={"text": "hi"}).json() == {"kind": "Note"} + assert client.post("/api/marks", json={"other": 1}).status_code == 400 + + +def test_nested_models_are_the_openapi_document_s_components_its_schemas_reference() -> None: + document = TestClient(Map().create_app()).get("/agentenv/ext/website/openapi").json() + schemas = document["components"]["schemas"] + assert set(schemas) == {"Pin", "Place", "Note"} + assert schemas["Place"]["properties"]["pin"] == {"$ref": "#/components/schemas/Pin"} + body = document["paths"]["/api/places"]["post"]["requestBody"]["content"]["application/json"]["schema"] + answer = document["paths"]["/api/places"]["post"]["responses"]["200"]["content"]["application/json"]["schema"] + assert body == {"$ref": "#/components/schemas/Place"} + assert answer == {"type": "array", "items": {"$ref": "#/components/schemas/Place"}} + refs = [v for v in _walk(document) if isinstance(v, str) and v.startswith("#/")] + assert refs and all(ref.split("/")[-1] in schemas and ref.startswith("#/components/schemas/") for ref in refs) + + +class Cart(BaseModel): + prices: list[float] + + @computed_field + @property + def total(self) -> float: + return sum(self.prices) + + +@website(domains=["till.example"]) +class Till(WebsiteEnvironment): + @api.post("/api/carts") + def price(self, cart: Cart) -> Cart: + return cart + + +def test_an_answer_s_schema_describes_it_as_serialized_and_a_body_s_as_validated() -> None: + client = TestClient(Till().create_app()) + assert client.post("/api/carts", json={"prices": [1, 2]}).json() == {"prices": [1, 2], "total": 3} + document = client.get("/agentenv/ext/website/openapi").json() + schemas = document["components"]["schemas"] + operation = document["paths"]["/api/carts"]["post"] + body = operation["requestBody"]["content"]["application/json"]["schema"] + answer = operation["responses"]["200"]["content"]["application/json"]["schema"] + assert body == {"$ref": "#/components/schemas/Cart-Input"} + assert answer == {"$ref": "#/components/schemas/Cart-Output"} + assert "total" not in schemas["Cart-Input"]["properties"] and "total" in schemas["Cart-Output"]["properties"] + + +def _walk(value: object): + if isinstance(value, dict): + for v in value.values(): + yield from _walk(v) + elif isinstance(value, list): + for v in value: + yield from _walk(v) + else: + yield value + + +def test_api_errors_answer_with_their_status_and_code(client: TestClient) -> None: + missing = client.get("/api/items/9") + assert (missing.status_code, missing.json()) == (404, {"error": "no item 9", "code": "not_found"}) + assert client.post("/api/items", json={"title": "mug"}).json() == {"error": "mug exists", "code": "conflict"} + teapot = client.put("/api/teapot") + assert (teapot.status_code, teapot.json()["code"]) == (418, "teapot") + boom = client.get("/api/boom") + assert (boom.status_code, boom.json()) == (500, {"error": "internal error", "code": "internal"}) + + +def test_responses_pass_through_and_none_is_json_null(client: TestClient, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv(ROLE_KEY_ENV, "k3y") + assert client.get("/api/plain").text == "as is" + deleted = client.delete("/api/items/1", headers={"AgentEnv-Role": "admin", ROLE_KEY_HEADER: "k3y"}) + assert (deleted.status_code, deleted.json()) == (200, None) + + +def test_an_optional_list_query_parameter_is_a_list_of_its_coerced_values(client: TestClient) -> None: + assert client.get("/api/picked?ids=1&ids=2").json() == {"ids": [1, 2]} + assert client.get("/api/picked").json() == {"ids": None} + assert client.get("/api/picked?ids=x").status_code == 400 + [ids] = client.get("/agentenv/ext/website/openapi").json()["paths"]["/api/picked"]["get"]["parameters"] + assert ids["schema"] == {"type": "array", "items": {"type": "integer"}, "default": None} + + +def test_a_route_declared_204_answers_with_no_body(client: TestClient) -> None: + emptied = client.delete("/api/cart") + assert (emptied.status_code, emptied.content) == (204, b"") + assert "content-length" not in emptied.headers and "content-type" not in emptied.headers + + +def test_a_204_answer_keeps_a_real_server_connection_usable() -> None: + import socket + import threading + import time + + uvicorn = pytest.importorskip("uvicorn") + server = uvicorn.Server(uvicorn.Config(Shop().create_app(), host="127.0.0.1", port=0, log_level="error", + ws="none")) + thread = threading.Thread(target=server.run, daemon=True) + thread.start() + while not server.started: + time.sleep(0.01) + port = server.servers[0].sockets[0].getsockname()[1] + try: + with socket.create_connection(("127.0.0.1", port), timeout=5) as conn: + conn.sendall(b"DELETE /api/cart HTTP/1.1\r\nHost: x\r\n\r\nGET /api/items/1 HTTP/1.1\r\nHost: x\r\n\r\n") + received = b"" + while b"mug" not in received: + chunk = conn.recv(65536) + if not chunk: + break + received += chunk + assert received.startswith(b"HTTP/1.1 204") and b"HTTP/1.1 200" in received and b"mug" in received + finally: + server.should_exit = True + thread.join(5) + + +def test_a_session_cookie_that_is_not_ascii_is_no_session(client: TestClient) -> None: + for path in ("/api/whoami", "/api/session"): + response = client.get(path, headers={"Cookie": "agentenv_session=abc.\xe9".encode("latin-1")}) + assert response.status_code == 200, response.text + assert response.json()["user"] is None + + +def test_roles_refuse_other_callers_with_403(client: TestClient, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv(ROLE_KEY_ENV, "k3y") + for headers in ({}, {"AgentEnv-Role": "default"}, {"AgentEnv-Role": "agent", ROLE_KEY_HEADER: "k3y"}): + refused = client.delete("/api/items/1", headers=headers) + assert refused.status_code == 403 and refused.json()["code"] == "forbidden" + assert client.get("/api/items/1").status_code == 200 + assert client.delete("/api/items/1", headers={"AgentEnv-Role": "admin", ROLE_KEY_HEADER: "k3y"}).status_code == 200 + + +def test_a_role_is_believed_only_with_the_site_s_role_key(client: TestClient, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv(ROLE_KEY_ENV, raising=False) + claim = {"AgentEnv-Role": "admin"} + assert client.delete("/api/items/1", headers=claim).status_code == 403 + assert client.delete("/api/items/1", headers={**claim, ROLE_KEY_HEADER: ""}).status_code == 403 + monkeypatch.setenv(ROLE_KEY_ENV, "k3y") + for key in (None, "", "wrong", "k3y "): + headers = claim if key is None else {**claim, ROLE_KEY_HEADER: key} + assert client.delete("/api/items/1", headers=headers).status_code == 403 + assert client.get("/api/whoami", headers=headers).json()["role"] == "default" + assert client.get("/api/whoami", headers={**claim, ROLE_KEY_HEADER: "k3y"}).json()["role"] == "admin" + + +def test_the_context_carries_the_request_role_and_forwarded_address(client: TestClient, + monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv(ROLE_KEY_ENV, "k3y") + client.cookies.set("c", "crumb") + seen = client.get("/api/whoami?q=1", headers={"x-probe": "p", "AgentEnv-Role": "agent", ROLE_KEY_HEADER: "k3y"}).json() + assert seen == {"user": None, "role": "agent", "base_url": "http://testserver", "q": "1", "header": "p", + "cookie": "crumb"} + forwarded = client.get("/api/whoami", headers={"X-Forwarded-Host": "shop.example, proxy", + "X-Forwarded-Proto": "https"}).json() + assert (forwarded["base_url"], forwarded["role"]) == ("https://shop.example", "default") + + +def test_a_redirect_names_the_forwarded_address(client: TestClient) -> None: + redirected = client.get("/api/items/", follow_redirects=False, + headers={"Host": "internal-x.invalid", "X-Forwarded-Host": "shop.example", + "X-Forwarded-Proto": "https"}) + assert (redirected.status_code, redirected.headers["location"]) == (307, "https://shop.example/api/items") + assert client.get("/api/items/", follow_redirects=False).headers["location"] == "http://testserver/api/items" + + +def test_a_site_behind_a_forwarded_prefix_names_it_in_its_address_and_redirects_and_routes_its_own_paths( + client: TestClient) -> None: + behind = {"Host": "internal-x.invalid", "X-Forwarded-Host": "gw.example", "X-Forwarded-Proto": "https", + "X-Forwarded-Prefix": "/website/shop/"} + assert client.get("/api/whoami", headers=behind).json()["base_url"] == "https://gw.example/website/shop" + assert client.get("/api/items/1", headers=behind).json()["title"] == "mug" + redirected = client.get("/api/items/", follow_redirects=False, headers=behind) + assert (redirected.status_code, redirected.headers["location"]) == (307, "https://gw.example/website/shop/api/items") + for unusable in ("//evil.example", "website", ""): + seen = client.get("/api/whoami", headers={**behind, "X-Forwarded-Prefix": unusable}).json() + assert seen["base_url"] == "https://gw.example", unusable + + +def test_parameterized_routes_never_take_the_data_plane_or_card() -> None: + @website(domains=["wiki.example"]) + class Wiki(WebsiteEnvironment): + @api.get("/{slug}") + def page(self, slug: str) -> dict: + return {"slug": slug} + + @api.post("/{collection}") + def add(self, collection: str) -> dict: + return {"captured": collection} + + @api.get("/.well-known/{name}") + def well_known(self, name: str) -> dict: + return {"captured": name} + + @api.get("/files/{rest:path}") + def files(self, rest: str) -> dict: + return {"rest": rest} + + client = TestClient(Wiki().create_app()) + assert client.get("/hello").json() == {"slug": "hello"} + assert client.post("/notes").json() == {"captured": "notes"} + assert client.get("/.well-known/other.json").json() == {"captured": "other.json"} + assert client.get("/files/a/b.txt").json() == {"rest": "a/b.txt"} + reset = client.post("/agentenv", json={"jsonrpc": "2.0", "id": 1, "method": "data/get", "params": {}}).json() + assert reset["id"] == 1 and "result" in reset + assert client.get("/.well-known/agent-env.json").json()["name"] == "Wiki" + assert client.get("/api/health").json() == {"status": "ok"} + + +@pytest.mark.parametrize("path", ["/agentenv", "/agentenv/ext/x", "/.well-known/agent-env.json", + "/api/../agentenv/x", "//agentenv"]) +def test_api_paths_under_the_reserved_ones_are_refused(path: str) -> None: + assert reserved_path(path) + with pytest.raises(ValueError, match="data plane"): + api.get(path) + + +def test_a_class_serving_a_name_or_route_twice_is_refused() -> None: + with pytest.raises(ValueError, match="name"): + class Twice(WebsiteEnvironment): + @api.get("/a", name="same") + def a(self) -> None: ... + + @api.get("/b", name="same") + def b(self) -> None: ... + + with pytest.raises(ValueError, match="route"): + class Again(WebsiteEnvironment): + @api.post("/a") + def a(self) -> None: ... + + @api.post("/a") + def b(self) -> None: ... + + +def test_a_route_must_take_its_path_parameters_and_at_most_one_body() -> None: + with pytest.raises(TypeError, match="item_id"): + api.get("/api/{item_id}")(lambda self: None) + + class Bodies(WebsiteEnvironment): + @api.post("/api/x") + def x(self, a: dict, b: NewItem) -> None: ... + + with pytest.raises(TypeError, match="one JSON body"): + Bodies().create_app() + + +def test_the_health_path_is_not_an_api_route() -> None: + with pytest.raises(ValueError, match="ready_check"): + @website(health="/api/up") + class Clash(WebsiteEnvironment): + @api.get("/api/up") + def up(self) -> dict: + return {} + + with pytest.raises(ValueError, match="health"): + website(health="/agentenv/health") + + +@website(domains=["shelf.example"]) +class Shelf(WebsiteEnvironment): + @api.delete("/api/books/{book_id}", status=204) + def drop(self, book_id: int) -> Pin: + return Pin(lat=1.0) + + +def test_a_bodyless_status_publishes_no_body_and_answers_none() -> None: + client = TestClient(Shelf().create_app()) + answered = client.delete("/api/books/1") + assert answered.status_code == 204 and answered.content == b"" + responses = client.get("/agentenv/ext/website/openapi").json()["paths"]["/api/books/{book_id}"]["delete"]["responses"] + assert "content" not in responses["204"] diff --git a/packages/agentenv-protocol/tests/test_website_conformance.py b/packages/agentenv-protocol/tests/test_website_conformance.py new file mode 100644 index 000000000..9140ab39d --- /dev/null +++ b/packages/agentenv-protocol/tests/test_website_conformance.py @@ -0,0 +1,321 @@ +"""The website/v1 conformance checker against an SDK site, a hand-written server that meets the contract, and servers +that don't.""" + +from __future__ import annotations + +import datetime +from pathlib import Path + +import httpx +import pytest +from starlette.applications import Starlette +from starlette.requests import Request +from starlette.responses import HTMLResponse, JSONResponse, Response +from starlette.routing import Route +from starlette.testclient import TestClient + +from agentenv_protocol.website import WebsiteEnvironment, api, website +from agentenv_protocol.website import conformance +from agentenv_protocol.website.clock import UNIVERSE_NOW_HEADER, iso + +CHECKS = ["card", "any_host", "health", "universe_now", "reserved_paths", "data_plane", "clock", "forwarded", "cookies"] + + +def statuses(results: list[conformance.Result]) -> dict[str, str]: + return {r.name: r.status for r in results} + + +@pytest.fixture +def notes(tmp_path: Path) -> WebsiteEnvironment: + (tmp_path / "index.html").write_text("hi") + + @website(domains=["notes.example"], static=str(tmp_path)) + class Notes(WebsiteEnvironment): + @api.get("/api/notes") + def notes(self) -> list: + return self.store["notes"] + + env = Notes() + env.store.load({"current_user": "ann", "notes": [{"id": 1, "text": "hello"}]}) + return env + + +@pytest.fixture +def sdk_site(notes: WebsiteEnvironment) -> TestClient: + return TestClient(notes.create_app()) + + +def test_an_sdk_site_passes_every_check(sdk_site: TestClient) -> None: + results = conformance.check("http://testserver", client=sdk_site) + assert [r.name for r in results] == CHECKS + assert all(r.status == "passed" for r in results), results + assert sdk_site.post("/agentenv", json={"jsonrpc": "2.0", "id": 1, "method": "data/get"}).json()["result"][ + "parts"][0]["data"] == {"current_user": "ann", "notes": [{"id": 1, "text": "hello"}]} + + +def _handwritten(*, leak_host: bool = False, domain_cookie: bool = False, clock: bool = True, + clock_ops_first: tuple[str, ...] = (), + leak_redirect: bool = False, stamp_rpc: bool = True, reset_keeps: bool = False, + get_fails_after_reset: bool = False, rpc_reply: tuple[int, dict] | None = None, + ready_answer: object = None, ready_status: int = 200, rpc_path: str | None = None) -> Starlette: + """A server written without the SDK: the contract by hand, optionally broken. Its state is its ``state.data``; with + ``ready_answer`` its card advertises ``website/v1``'s ``ready``, which answers it with ``ready_status``. With ``rpc_path`` its card's + ``url`` names its data plane there, not at ``/agentenv``.""" + state = {"items": ["a"], "url": None, "at": None, "reset": False} + + def now() -> datetime.datetime: + return state["at"] or datetime.datetime.now(datetime.timezone.utc) + + def stamp(response: Response) -> Response: + response.headers[UNIVERSE_NOW_HEADER] = iso(now()) + return response + + async def page(request: Request) -> Response: + host = request.headers["host"] if leak_host else request.headers.get("x-forwarded-host", request.headers["host"]) + response = stamp(HTMLResponse(f'x')) + response.set_cookie("s", "1", domain="example.com" if domain_cookie else None) + return response + + async def health(request: Request) -> Response: + return stamp(JSONResponse({"ok": True})) + + async def ready(request: Request) -> Response: + return stamp(JSONResponse(ready_answer, status_code=ready_status)) + + async def items(request: Request) -> Response: + return stamp(JSONResponse(state["items"])) + + async def items_slash(request: Request) -> Response: + host = request.headers["host"] if leak_redirect else request.headers.get("x-forwarded-host", request.headers["host"]) + return stamp(Response(status_code=307, headers={"Location": f"https://{host}/items"})) + + async def card(request: Request) -> Response: + extensions = [{"uri": "urn:agentenv:website/v1", + "params": {"domains": ["h.example"], "health": "/healthz", + "routes": [{"method": "GET", "path": "/items", "name": "items"}]}}] + if ready_answer is not None: + extensions[0]["params"]["methods"] = {"ready": {"method": "GET", "endpoint": "/ready"}} + if clock: + extensions.append({"uri": "urn:agentenv:clock/v1", "params": { + "endpoint": "/agentenv/ext/sync_time", "methods": { + **{op: {"method": "GET", "endpoint": f"/agentenv/ext/{op}"} for op in clock_ops_first}, + "sync_time": {"method": "POST"}}}}) + return stamp(JSONResponse({"name": "hand", **({"url": rpc_path} if rpc_path else {}), + "capabilities": {"extensions": extensions, + "operations": ["data/reset", "data/add", "data/get"]}})) + + async def rpc(request: Request) -> Response: + body = await request.json() + method = body["method"] + if rpc_reply is not None: + return stamp(JSONResponse(rpc_reply[1], status_code=rpc_reply[0])) + if method == "data/reset": + state["reset"] = True + if not reset_keeps: + state["items"] = [] + elif method == "data/add": + state["reset"] = False + added = [item for part in body["params"]["parts"] for item in part["data"]["items"]] + state["items"] = added if reset_keeps else state["items"] + added + if method == "data/get" and get_fails_after_reset and state["reset"]: + answer = {"jsonrpc": "2.0", "id": body["id"], "error": {"code": -32603, "message": "down"}} + else: + result = {"parts": [{"kind": "data", "data": {"items": state["items"]}}]} if method == "data/get" else {} + answer = {"jsonrpc": "2.0", "id": body["id"], "result": result} + return stamp(JSONResponse(answer)) if stamp_rpc else JSONResponse(answer) + + async def sync_time(request: Request) -> Response: + import httpx + + url = (await request.json())["env_get_time_url"] + async with httpx.AsyncClient() as client: + virtual = (await client.get(url)).json()["virtual_time"] + state["at"] = datetime.datetime.fromisoformat(virtual.replace("Z", "+00:00")) + return stamp(JSONResponse({"gateway_virtual_time": virtual})) + + async def unrouted(request: Request, exc: Exception) -> Response: + return stamp(JSONResponse({"error": "not found"}, status_code=getattr(exc, "status_code", 404))) + + app = Starlette(routes=[Route("/", page), Route("/healthz", health), Route("/ready", ready), Route("/items", items), + Route("/items/", items_slash), + Route("/.well-known/agent-env.json", card), Route(rpc_path or "/agentenv", rpc, methods=["POST"]), + Route("/agentenv/ext/sync_time", sync_time, methods=["POST"])], + exception_handlers={404: unrouted, 405: unrouted}) + app.state.data = state + return app + + +def test_a_handwritten_server_meeting_the_contract_passes() -> None: + results = conformance.check("http://testserver", client=TestClient(_handwritten())) + assert statuses(results) == dict.fromkeys(CHECKS, "passed"), results + + +@pytest.mark.parametrize(("answer", "status", "detail"), [ + ({"ready": True, "problems": [], "since": "boot"}, "passed", "website/v1 ready says it is ready"), + ({"ready": False, "problems": ["seeded", "indexed"]}, "failed", "website/v1 ready says it is not ready: seeded, indexed"), + ({"ready": False}, "failed", "website/v1 ready says it is not ready: no problem named"), +]) +def test_health_reads_the_cards_ready_as_a_website_readiness(answer: dict, status: str, detail: str) -> None: + results = {r.name: r for r in conformance.check("http://testserver", client=TestClient(_handwritten(ready_answer=answer)))} + assert (results["health"].status, results["health"].detail) == (status, detail) + + +@pytest.mark.parametrize(("answer", "detail"), [ + ({"ready": True, "problems": []}, "website/v1 ready answered 503: no problem named"), + ({"ready": False, "problems": ["seeding"]}, "website/v1 ready answered 503: seeding"), +]) +def test_ready_answering_an_http_error_fails_health_whatever_its_body_says(answer: dict, detail: str) -> None: + client = TestClient(_handwritten(ready_answer=answer, ready_status=503)) + health = next(r for r in conformance.check("http://testserver", client=client) if r.name == "health") + assert (health.status, health.detail) == ("failed", detail) + + +@pytest.mark.parametrize(("answer", "why"), [ + ({"ok": True}, "ready: Field required"), + ({"ready": "soon", "problems": []}, "ready: Input should be a valid boolean"), + ({"ready": False, "problems": "seeded"}, "problems: Input should be a valid list"), + (["ready"], "the answer: Input should be a valid dictionary or instance of WebsiteReadiness"), +]) +def test_an_answer_to_ready_that_isnt_a_website_readiness_fails_health_saying_why(answer: object, why: str) -> None: + health = next(r for r in conformance.check("http://testserver", client=TestClient(_handwritten(ready_answer=answer))) + if r.name == "health") + assert health.status == "failed" + assert health.detail.startswith(f"website/v1 ready answered {answer!r}, not a WebsiteReadiness: ") and why in health.detail + + +def test_a_server_breaking_the_contract_fails_those_checks() -> None: + results = statuses(conformance.check("http://testserver", + client=TestClient(_handwritten(leak_host=True, domain_cookie=True, clock=False)))) + assert results["forwarded"] == results["cookies"] == results["clock"] == "warning" + assert results["card"] == results["health"] == results["data_plane"] == "passed" + + +def test_the_forwarded_check_follows_no_redirect_and_reads_its_location(sdk_site: TestClient) -> None: + forwarded = {r.name: r for r in conformance.check("http://testserver", client=sdk_site)}["forwarded"] + assert forwarded.status == "passed" and "/api/notes/" in forwarded.detail + leaked = {r.name: r for r in conformance.check("http://testserver", + client=TestClient(_handwritten(leak_redirect=True)))}["forwarded"] + assert leaked.status == "warning" and "Location of /items/" in leaked.detail + + +def test_universe_now_is_checked_on_the_answers_of_every_check() -> None: + result = {r.name: r for r in conformance.check("http://testserver", + client=TestClient(_handwritten(stamp_rpc=False)))}["universe_now"] + assert result.status == "warning" and "/agentenv" in result.detail + + +def test_a_reset_that_clears_nothing_fails_the_data_plane_and_leaves_the_data() -> None: + app = _handwritten(reset_keeps=True) + result = {r.name: r for r in conformance.check("http://testserver", client=TestClient(app))}["data_plane"] + assert result.status == "failed" and "data/reset" in result.detail + assert app.state.data["items"] == ["a"] + + +def test_the_data_is_added_back_when_a_step_after_the_reset_fails() -> None: + app = _handwritten(get_fails_after_reset=True) + result = {r.name: r for r in conformance.check("http://testserver", client=TestClient(app))}["data_plane"] + assert result.status == "failed" and "down" in result.detail + assert app.state.data["items"] == ["a"] + + +@pytest.mark.parametrize("reply", [(500, {}), (200, {}), (500, {"jsonrpc": "2.0", "id": "x", "result": {}})]) +def test_a_data_request_without_a_2xx_json_rpc_result_fails_the_data_plane(reply: tuple[int, dict]) -> None: + app = _handwritten(rpc_reply=reply) + result = {r.name: r for r in conformance.check("http://testserver", client=TestClient(app))}["data_plane"] + assert result.status == "failed" and "no JSON-RPC result" in result.detail + + +def test_an_sdk_sites_data_round_trips_and_is_left_as_the_check_found_it_changed_or_not( + notes: WebsiteEnvironment, sdk_site: TestClient) -> None: + for _ in range(2): + before = notes.store.export() + result = {r.name: r for r in conformance.check("http://testserver", client=sdk_site)}["data_plane"] + assert result.status == "passed", result + assert notes.store.export() == before + notes.store.insert("notes", {"text": "new"}) + + +def test_the_data_plane_is_reached_at_the_cards_url() -> None: + app = _handwritten(rpc_path="/rpc") + result = {r.name: r for r in conformance.check("http://testserver", client=TestClient(app))}["data_plane"] + assert result.status == "passed", result + assert app.state.data["items"] == ["a"] + + +def test_the_clock_check_calls_sync_time_whichever_operation_the_card_lists_first() -> None: + results = {r.name: r.status for r in conformance.check( + "http://testserver", client=TestClient(_handwritten(clock_ops_first=("time",))))} + assert results["clock"] == "passed", results + + +def test_a_server_with_no_card_health_or_header_fails_the_contract_and_warns_of_the_rest() -> None: + bare = Starlette(routes=[Route("/", lambda request: JSONResponse({"not": "html"}))]) + results = conformance.check("http://testserver", client=TestClient(bare)) + assert statuses(results) == {"card": "warning", "any_host": "failed", "health": "failed", "universe_now": "warning", + "reserved_paths": "passed", "data_plane": "skipped", "clock": "warning", + "forwarded": "passed", "cookies": "skipped"} + assert [r.name for r in results if r.required] == ["any_host", "health", "reserved_paths", "data_plane"] + + +def test_a_bare_server_meeting_only_the_contract_passes_with_warnings() -> None: + async def page(request: Request) -> Response: + return HTMLResponse("hi") + + bare = Starlette(routes=[Route("/", page), Route("/api/health", lambda request: JSONResponse({"ok": True}))]) + results = conformance.check("http://testserver", client=TestClient(bare)) + assert not any(r.fails() for r in results) and any(r.fails(strict=True) for r in results) + assert {r.name for r in results if r.status == "warning"} == {"card", "universe_now", "clock"} + + +def test_a_spa_fallback_that_serves_the_reserved_paths_fails() -> None: + async def page(request: Request) -> Response: + return HTMLResponse("app") + + spa = Starlette(routes=[Route("/{path:path}", page)]) + reserved = {r.name: r for r in conformance.check("http://testserver", client=TestClient(spa))}["reserved_paths"] + assert reserved.status == "failed" and reserved.detail.startswith("/agentenv answered a page") + + +@pytest.mark.parametrize("exact", ["/agentenv", "/.well-known/agent-env.json"]) +def test_a_page_at_a_reserved_path_itself_fails(exact: str) -> None: + async def page(request: Request) -> Response: + return HTMLResponse("app") + + server = Starlette(routes=[Route("/", page), Route(exact, page)]) + reserved = {r.name: r for r in conformance.check("http://testserver", client=TestClient(server))}["reserved_paths"] + assert reserved.status == "failed" and reserved.detail.startswith(f"{exact} answered a page") + + +def test_main_prints_each_check_and_exits_by_outcome(monkeypatch, capsys) -> None: + results = [conformance.Result("health", "passed"), conformance.Result("clock", "warning", "late", required=False)] + monkeypatch.setattr(conformance, "check", lambda url, **options: results) + assert conformance.main(["http://site"]) == 0 + printed = capsys.readouterr().out + assert "PASS health (required)" in printed and "WARN clock (recommended): late" in printed + assert "1 passed, 0 failed, 1 warning(s), 0 skipped" in printed + assert conformance.main(["http://site", "--strict"]) == 1 + results.append(conformance.Result("any_host", "failed", "json")) + assert conformance.main(["http://site"]) == 1 + assert "FAIL any_host (required): json" in capsys.readouterr().out + + +def test_main_exits_2_when_the_server_cannot_be_reached(capsys) -> None: + assert conformance.main(["http://127.0.0.1:9", "--timeout", "2"]) == 2 + assert "couldn't reach" in capsys.readouterr().err + + +def test_the_cookie_check_holds_the_session_path_to_the_cookie_the_card_names(sdk_site: TestClient) -> None: + card = sdk_site.get("/.well-known/agent-env.json").json() + for extension in card["capabilities"]["extensions"]: + if extension["uri"] == "urn:agentenv:website/v1": + extension["params"]["session_cookie"] = "agentenv_session_other" + real = sdk_site.get + + def get(url, **kwargs): + if url.endswith("/.well-known/agent-env.json"): + return httpx.Response(200, json=card, request=httpx.Request("GET", url)) + return real(url, **kwargs) + + sdk_site.get = get + cookies = {r.name: r for r in conformance.check("http://testserver", client=sdk_site)}["cookies"] + assert cookies.status == "warning" and "not the card's agentenv_session_other" in cookies.detail diff --git a/packages/agentenv-protocol/tests/test_website_database.py b/packages/agentenv-protocol/tests/test_website_database.py new file mode 100644 index 000000000..1c623a482 --- /dev/null +++ b/packages/agentenv-protocol/tests/test_website_database.py @@ -0,0 +1,304 @@ +"""A site of ``@website(models=...)``: its state is a database's, SQLAlchemy models' tables on DATABASE_URL (sqlite +here) or an object's own load/export/reset, which its data plane reads and writes, and its routes reach through +``self.db``.""" + +from __future__ import annotations + +import datetime +import enum +import sys +import types +from pathlib import Path + +import pytest +from starlette.testclient import TestClient + +from agentenv_protocol.website import ApiContext, WebsiteEnvironment, api, conformance, website + +sqlalchemy = pytest.importorskip("sqlalchemy") +from sqlalchemy import DateTime, ForeignKey, Integer, String, select # noqa: E402 +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column # noqa: E402 + +RPC = "/agentenv" + + +class Base(DeclarativeBase): + pass + + +class User(Base): + __tablename__ = "users" + id: Mapped[str] = mapped_column(String, primary_key=True) + name: Mapped[str] = mapped_column(String) + + +class Ticket(Base): + __tablename__ = "tickets" + id: Mapped[int] = mapped_column(Integer, primary_key=True) + owner: Mapped[str] = mapped_column(ForeignKey("users.id")) + title: Mapped[str] = mapped_column(String) + opened_at: Mapped[datetime.datetime] = mapped_column(DateTime) + + +class State(enum.Enum): + OPEN = "open" + CLOSED = "closed" + + +SEED = {"current_user": "ann", "users": [{"id": "ann", "name": "Ann"}], + "tickets": [{"id": 1, "owner": "ann", "title": "printer", "opened_at": "2031-01-02T03:04:05"}]} + + +def rpc(client: TestClient, method: str, params: dict | None = None) -> dict: + body = client.post(RPC, json={"jsonrpc": "2.0", "id": 1, "method": method, "params": params or {}}).json() + assert "error" not in body, body + return body["result"] + + +@pytest.fixture +def dist(tmp_path: Path) -> Path: + (tmp_path / "index.html").write_text("desk") + return tmp_path + + +@pytest.fixture +def database(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> str: + url = f"sqlite:///{tmp_path / 'service.db'}" + monkeypatch.setenv("DATABASE_URL", url) + return url + + +def helpdesk(dist: Path, models: object) -> type[WebsiteEnvironment]: + @website(domains=["desk.example"], static=str(dist), models=models) + class Helpdesk(WebsiteEnvironment): + @api.get("/api/tickets") + def tickets(self, ctx: ApiContext) -> list: + with self.db() as session: + return [{"id": t.id, "title": t.title} for t in session.scalars( + select(Ticket).where(Ticket.owner == ctx.user).order_by(Ticket.id))] + + @api.post("/api/tickets", status=201) + def open_ticket(self, body: dict, ctx: ApiContext) -> dict: + with self.db.begin() as session: + ticket = Ticket(owner=ctx.user, title=body["title"], opened_at=self.now().replace(tzinfo=None)) + session.add(ticket) + session.flush() + return {"id": ticket.id} + + return Helpdesk + + +def test_the_data_plane_loads_exports_and_resets_the_models_tables(dist: Path, database: str) -> None: + env = helpdesk(dist, Base)() + client = TestClient(env.create_app()) + rpc(client, "data/add", {"parts": [{"kind": "data", "data": SEED}]}) + assert rpc(client, "data/get")["parts"][0]["data"] == SEED + assert client.get("/api/session").json()["user"] == "ann" + assert client.get("/api/tickets").json() == [{"id": 1, "title": "printer"}] + rpc(client, "data/reset") + assert rpc(client, "data/get")["parts"][0]["data"] == {"users": [], "tickets": []} + + +def test_rows_that_leave_an_optional_column_out_load_as_given_in_either_order(dist: Path, database: str) -> None: + class Threads(DeclarativeBase): + pass + + class Note(Threads): + __tablename__ = "notes" + id: Mapped[int] = mapped_column(Integer, primary_key=True) + parent_id: Mapped[int | None] = mapped_column(ForeignKey("notes.id"), nullable=True) + + client = TestClient(helpdesk(dist, Threads)().create_app()) + for seed in ([{"id": 1}, {"id": 2, "parent_id": 1}, {"id": 3, "parent_id": 1}, {"id": 4}], + [{"id": 1, "parent_id": None}, {"id": 2}, {"id": 3, "parent_id": 2}]): + rpc(client, "data/reset") + rpc(client, "data/add", {"parts": [{"kind": "data", "data": {"notes": seed}}]}) + assert rpc(client, "data/get")["parts"][0]["data"]["notes"] == [{"parent_id": None, **row} for row in seed] + + +def test_an_export_with_a_computed_column_loads_back_as_the_database_computes_it(dist: Path, database: str) -> None: + class Priced(DeclarativeBase): + pass + + class Item(Priced): + __tablename__ = "items" + id: Mapped[int] = mapped_column(Integer, primary_key=True) + price: Mapped[int] = mapped_column(Integer) + doubled: Mapped[int] = mapped_column(Integer, sqlalchemy.Computed("price * 2")) + + client = TestClient(helpdesk(dist, Priced)().create_app()) + rpc(client, "data/add", {"parts": [{"kind": "data", "data": {"items": [{"id": 1, "price": 3}]}}]}) + exported = rpc(client, "data/get")["parts"][0]["data"] + assert exported == {"items": [{"id": 1, "price": 3, "doubled": 6}]} + rpc(client, "data/reset") + rpc(client, "data/add", {"parts": [{"kind": "data", "data": exported}]}) + assert rpc(client, "data/get")["parts"][0]["data"] == exported + + +def test_an_export_whose_child_row_sorts_before_its_parent_loads_back_with_foreign_keys_enforced( + dist: Path, database: str) -> None: + class Threads(DeclarativeBase): + pass + + class Note(Threads): + __tablename__ = "notes" + id: Mapped[int] = mapped_column(Integer, primary_key=True) + parent_id: Mapped[int | None] = mapped_column(ForeignKey("notes.id"), nullable=True) + + def enforce(connection, record) -> None: + connection.execute("PRAGMA foreign_keys=ON") + + sqlalchemy.event.listen(sqlalchemy.engine.Engine, "connect", enforce) + try: + client = TestClient(helpdesk(dist, Threads)().create_app()) + rpc(client, "data/add", {"parts": [{"kind": "data", "data": {"notes": [ + {"id": 3, "parent_id": None}, {"id": 2, "parent_id": 3}, {"id": 1, "parent_id": 2}]}}]}) + exported = rpc(client, "data/get")["parts"][0]["data"] + assert [row["id"] for row in exported["notes"]] == [1, 2, 3] + rpc(client, "data/reset") + rpc(client, "data/add", {"parts": [{"kind": "data", "data": exported}]}) + assert rpc(client, "data/get")["parts"][0]["data"] == exported + finally: + sqlalchemy.event.remove(sqlalchemy.engine.Engine, "connect", enforce) + + +def test_a_chain_longer_than_the_recursion_limit_is_ordered_parents_first() -> None: + import sys + + from agentenv_protocol.website.database import _parents_first + + metadata = sqlalchemy.MetaData() + table = sqlalchemy.Table("notes", metadata, sqlalchemy.Column("id", Integer, primary_key=True), + sqlalchemy.Column("parent_id", Integer, ForeignKey("notes.id"), nullable=True)) + count = sys.getrecursionlimit() * 2 + rows = [{"id": i, "parent_id": i + 1 if i + 1 < count else None} for i in range(count)] + ordered = _parents_first(table, rows) + assert [row["id"] for row in ordered] == list(range(count - 1, -1, -1)) + cycle = [{"id": 1, "parent_id": 2}, {"id": 2, "parent_id": 1}] + assert sorted(row["id"] for row in _parents_first(table, cycle)) == [1, 2] + + +def test_a_composite_self_reference_loads_back_parents_first_matching_the_whole_key(dist: Path, database: str) -> None: + class Tenants(DeclarativeBase): + pass + + class Node(Tenants): + __tablename__ = "nodes" + tenant: Mapped[str] = mapped_column(String, primary_key=True) + id: Mapped[int] = mapped_column(Integer, primary_key=True) + parent_tenant: Mapped[str | None] = mapped_column(String, nullable=True) + parent_id: Mapped[int | None] = mapped_column(Integer, nullable=True) + __table_args__ = (sqlalchemy.ForeignKeyConstraint(["parent_tenant", "parent_id"], + ["nodes.tenant", "nodes.id"]),) + + def enforce(connection, record) -> None: + connection.execute("PRAGMA foreign_keys=ON") + + sqlalchemy.event.listen(sqlalchemy.engine.Engine, "connect", enforce) + try: + client = TestClient(helpdesk(dist, Tenants)().create_app()) + rows = [{"tenant": t, "id": i, "parent_tenant": None, "parent_id": None} for t, i in (("b", 2), ("b", 5), ("c", 2))] + rows.append({"tenant": "a", "id": 1, "parent_tenant": "b", "parent_id": 2}) + rpc(client, "data/add", {"parts": [{"kind": "data", "data": {"nodes": rows}}]}) + exported = rpc(client, "data/get")["parts"][0]["data"] + assert [(row["tenant"], row["id"]) for row in exported["nodes"]] == [("a", 1), ("b", 2), ("b", 5), ("c", 2)] + rpc(client, "data/reset") + rpc(client, "data/add", {"parts": [{"kind": "data", "data": exported}]}) + assert rpc(client, "data/get")["parts"][0]["data"] == exported + finally: + sqlalchemy.event.remove(sqlalchemy.engine.Engine, "connect", enforce) + + +def test_an_export_of_an_enum_column_loads_back_as_its_members(dist: Path, database: str) -> None: + class Desk(DeclarativeBase): + pass + + class Issue(Desk): + __tablename__ = "issues" + id: Mapped[int] = mapped_column(Integer, primary_key=True) + state: Mapped[State] = mapped_column(sqlalchemy.Enum(State)) + + client = TestClient(helpdesk(dist, Desk)().create_app()) + rpc(client, "data/add", {"parts": [{"kind": "data", "data": {"issues": [{"id": 1, "state": "open"}, + {"id": 2, "state": "CLOSED"}]}}]}) + exported = rpc(client, "data/get")["parts"][0]["data"] + assert exported == {"issues": [{"id": 1, "state": "open"}, {"id": 2, "state": "closed"}]} + rpc(client, "data/reset") + rpc(client, "data/add", {"parts": [{"kind": "data", "data": exported}]}) + assert rpc(client, "data/get")["parts"][0]["data"] == exported + + +def test_a_row_the_gui_adds_is_the_mcp_servers_and_data_get_reports_it(dist: Path, database: str) -> None: + env = helpdesk(dist, Base)() + client = TestClient(env.create_app()) + rpc(client, "data/add", {"parts": [{"kind": "data", "data": SEED}]}) + assert client.post("/api/tickets", json={"title": "badge"}).json() == {"id": 2} + server = sqlalchemy.create_engine(database) + with server.connect() as connection: + assert connection.execute(select(Ticket.title).order_by(Ticket.id)).scalars().all() == ["printer", "badge"] + connection.execute(sqlalchemy.update(Ticket).where(Ticket.id == 1).values(title="printer jam")) + connection.commit() + tickets = rpc(client, "data/get")["parts"][0]["data"]["tickets"] + assert [t["title"] for t in tickets] == ["printer jam", "badge"] + + +def test_a_models_site_passes_the_conformance_check(dist: Path, database: str) -> None: + client = TestClient(helpdesk(dist, Base)().create_app()) + rpc(client, "data/add", {"parts": [{"kind": "data", "data": SEED}]}) + results = conformance.check("http://testserver", client=client) + assert all(r.status == "passed" for r in results), results + assert rpc(client, "data/get")["parts"][0]["data"] == SEED + + +def test_models_may_be_a_metadata_or_a_module_and_refuse_anything_else(dist: Path, database: str) -> None: + module = types.ModuleType("helpdesk_models") + module.Base, module.Ticket = Base, Ticket + for models in (Base.metadata, module): + assert helpdesk(dist, models)()._tables.metadata is Base.metadata + with pytest.raises(TypeError, match="models="): + helpdesk(dist, object()) + with pytest.raises(ValueError, match="data/add"): + website(models=Base, data="seed.json") + + +def test_unknown_tables_and_a_missing_database_url_are_refused(dist: Path, monkeypatch: pytest.MonkeyPatch, + database: str) -> None: + client = TestClient(helpdesk(dist, Base)().create_app()) + body = client.post(RPC, json={"jsonrpc": "2.0", "id": 1, "method": "data/add", + "params": {"parts": [{"kind": "data", "data": {"orders": []}}]}}).json() + assert "error" in body + monkeypatch.delenv("DATABASE_URL") + with pytest.raises(RuntimeError, match="DATABASE_URL"): + helpdesk(dist, Base)().db() + + +def test_a_database_url_whose_driver_isnt_installed_names_it(dist: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setitem(sys.modules, "psycopg2", None) + monkeypatch.setitem(sys.modules, "psycopg", None) + monkeypatch.setenv("DATABASE_URL", "postgresql://desk@localhost/service") + with pytest.raises(ImportError, match=r"postgresql:// needs the database driver psycopg, .* carries psycopg"): + helpdesk(dist, Base)().db() + + +def test_an_object_with_its_own_hooks_is_the_sites_data(dist: Path) -> None: + class Rows: + def __init__(self) -> None: + self.rows: list = [] + + def load(self, data: dict) -> None: + self.rows += data["rows"] + + async def export(self) -> dict: + return {"rows": list(self.rows)} + + def reset(self) -> None: + self.rows.clear() + + rows = Rows() + env = helpdesk(dist, rows)() + client = TestClient(env.create_app()) + rpc(client, "data/add", {"parts": [{"kind": "data", "data": {"current_user": "bo", "rows": [{"id": 1}]}}]}) + rows.rows.append({"id": 2}) + assert rpc(client, "data/get")["parts"][0]["data"] == {"current_user": "bo", "rows": [{"id": 1}, {"id": 2}]} + assert env.acting_user() == "bo" + assert env.db is None diff --git a/packages/agentenv-protocol/tests/test_website_environment.py b/packages/agentenv-protocol/tests/test_website_environment.py new file mode 100644 index 000000000..1cfb7c1d3 --- /dev/null +++ b/packages/agentenv-protocol/tests/test_website_environment.py @@ -0,0 +1,680 @@ +"""WebsiteEnvironment as a site's server: static pages, health, the card and its OpenAPI document, the default store +and its data plane, sessions, the universe's clock, and a mounted app.""" + +from __future__ import annotations + +import asyncio +import base64 +import datetime +import json +import shutil +import subprocess +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +import pytest +from starlette.applications import Starlette +from starlette.responses import PlainTextResponse +from starlette.routing import Route +from starlette.testclient import TestClient + +from agentenv_protocol import DataPart, get_data +from agentenv_protocol.website import ( + SESSION_COOKIE, + UNIVERSE_NOW_HEADER, + WEBSITE_EXTENSION_URI, + WEBSITE_READY_OP, + ApiContext, + Collections, + WebsiteEnvironment, + WebsiteReadiness, + api, + ready_check, + website, +) +from agentenv_protocol.website import clock as clock_module +from agentenv_protocol.website.clock import parse_time + +RPC = "/agentenv" +COOKIE = "agentenv_session_Shop" + + +def rpc(client: TestClient, method: str, params: dict | None = None) -> dict: + body = client.post(RPC, json={"jsonrpc": "2.0", "id": 1, "method": method, "params": params or {}}).json() + assert "error" not in body, body + return body["result"] + + +@pytest.fixture +def dist(tmp_path: Path) -> Path: + root = tmp_path / "dist" + (root / "assets").mkdir(parents=True) + (root / "index.html").write_text("Shophome") + (root / "assets" / "app.js").write_text("console.log(1)") + (tmp_path / "secret.txt").write_text("outside") + return root + + +def site(dist: Path, **declared) -> type[WebsiteEnvironment]: + @website(domains=["shop.example", "www.shop.example"], static=str(dist), **declared) + class Shop(WebsiteEnvironment): + @api.get("/api/orders", name="list_orders") + def list_orders(self, ctx: ApiContext) -> list: + """List the signed-in user's orders.""" + return self.store.find("orders", user=ctx.user) + + @api.post("/api/orders", name="place_order", roles=["default", "agent"]) + def place_order(self, order: dict, ctx: ApiContext) -> dict: + return self.store.insert("orders", {**order, "user": ctx.user, "at": self.now().isoformat()}) + + @api.post("/api/sign-in") + def sign_in(self, body: dict, ctx: ApiContext) -> dict: + ctx.sign_in(body["user"]) + return {"user": ctx.user} + + return Shop + + +SEED = {"current_user": "alice", "users": [{"id": "alice"}, {"id": "bob"}], "orders": [{"id": 1, "user": "bob"}]} + + +@pytest.fixture +def shop(dist: Path) -> WebsiteEnvironment: + env = site(dist)() + env.store.load(SEED) + return env + + +@pytest.fixture +def client(shop: WebsiteEnvironment) -> TestClient: + return TestClient(shop.create_app()) + + +def test_static_files_and_index_fallback_answer_for_any_host(client: TestClient) -> None: + for host in ("testserver", "shop.example", "10.0.0.7:8000", "anything.invalid"): + page = client.get("/", headers={"Host": host}) + assert page.status_code == 200 and "home" in page.text and page.headers["content-type"].startswith("text/html") + assert client.get("/assets/app.js").text == "console.log(1)" + assert "home" in client.get("/orders/42/edit").text + assert client.get("/assets/missing.js").status_code == 404 + assert client.get("/api/nothing-here").json() == {"error": "not found", "code": "not_found"} + assert client.get("/../secret.txt").status_code == 404 + assert client.post("/orders").status_code == 404 + + +def test_behind_a_forwarded_prefix_the_site_serves_its_own_paths_and_the_fallback_still_guards_reserved_ones( + client: TestClient) -> None: + behind = {"X-Forwarded-Prefix": "/website/shop"} + assert "home" in client.get("/", headers=behind).text + assert client.get("/assets/app.js", headers=behind).text == "console.log(1)" + assert "home" in client.get("/orders/42/edit", headers=behind).text + assert client.get("/api/orders", headers=behind).json() == [] + assert client.get("/.well-known/agent-env.json", headers=behind).json()["name"] == "Shop" + assert client.get("/website/shop/assets/app.js", headers=behind).status_code == 404 + assert client.post("/agentenv/x", headers=behind).status_code in (404, 405) + + +def test_a_link_out_of_the_static_folder_is_never_served(dist: Path) -> None: + secret = dist.parent / "secret.txt" + (dist / "leak.txt").symlink_to(secret) + (dist / "linked").mkdir() + (dist / "linked" / "index.html").symlink_to(secret) + client = TestClient(site(dist)().create_app()) + assert "outside" not in client.get("/leak.txt").text + assert "outside" not in client.get("/linked").text + assert "outside" not in client.get("/linked/").text + + +def test_the_fallback_never_shadows_api_health_or_reserved_paths(client: TestClient) -> None: + assert client.get("/api/orders").json() == [] + assert client.put("/api/orders").status_code == 405 + assert client.get("/api/health").json() == {"status": "ok"} + for path in ("/agentenv", "/agentenv/anything", "/agentenv/ext/website/nope", "/agentenv/../agentenv/x"): + assert client.get(path).status_code in (404, 405), path + assert client.get("/.well-known/agent-env.json").json()["name"] == "Shop" + + +def test_health_waits_for_every_ready_check(dist: Path) -> None: + class Slow(site(dist)): + ready = False + + @ready_check + def database_up(self) -> bool: + return self.ready + + @ready_check + async def cache_warm(self) -> bool: + return True + + env = Slow() + client = TestClient(env.create_app()) + waiting = client.get("/api/health") + assert waiting.status_code == 503 and waiting.json() == {"status": "starting", "waiting_for": ["database_up"]} + env.ready = True + assert client.get("/api/health").status_code == 200 + + +def test_the_cards_website_v1_ready_says_whether_the_site_is_usable_by_its_ready_checks(dist: Path) -> None: + class Slow(site(dist)): + ready = False + + @ready_check + def database_up(self) -> bool: + return self.ready + + env = Slow() + client = TestClient(env.create_app()) + card = client.get("/.well-known/agent-env.json").json() + ready = _extension(card, WEBSITE_EXTENSION_URI)["params"]["methods"][WEBSITE_READY_OP] + assert (ready["method"], ready["endpoint"]) == ("GET", "/agentenv/ext/website/ready") + assert client.get(ready["endpoint"]).json() == {"ready": False, "problems": ["database_up"]} + assert WebsiteReadiness.model_validate(client.get(ready["endpoint"]).json()) == WebsiteReadiness( + ready=False, problems=["database_up"]) + env.ready = True + assert client.get(ready["endpoint"]).json() == {"ready": True, "problems": []} + + + +def test_a_declared_health_path_is_served_there(dist: Path) -> None: + client = TestClient(site(dist, health="/healthz")().create_app()) + assert client.get("/healthz").json() == {"status": "ok"} + card = client.get("/.well-known/agent-env.json").json() + assert _extension(card, WEBSITE_EXTENSION_URI)["params"]["health"] == "/healthz" + + +def _extension(card: dict, uri: str) -> dict: + return next(e for e in card["capabilities"]["extensions"] if e["uri"] == uri) + + +def test_the_card_declares_website_v1_its_routes_clock_and_data_plane(client: TestClient) -> None: + card = client.get("/.well-known/agent-env.json").json() + assert card["capabilities"]["operations"] == ["data/reset", "data/add", "data/get"] + params = _extension(card, WEBSITE_EXTENSION_URI)["params"] + assert params["domains"] == ["shop.example", "www.shop.example"] + assert params["health"] == "/api/health" + assert params["routes"] == [ + {"method": "GET", "path": "/api/orders", "name": "list_orders", "roles": None}, + {"method": "POST", "path": "/api/orders", "name": "place_order", "roles": ["default", "agent"]}, + {"method": "POST", "path": "/api/sign-in", "name": "sign_in", "roles": None}, + ] + assert params["methods"]["openapi"] == {"method": "GET", "endpoint": "/agentenv/ext/website/openapi", + "request": {"type": "object", "properties": {}}} + assert params["methods"]["session"]["endpoint"] == "/agentenv/ext/website/session" + clock = _extension(card, "urn:agentenv:clock/v1")["params"] + assert clock["endpoint"] == "/agentenv/ext/sync_time" + + +def test_the_openapi_document_names_each_route(client: TestClient) -> None: + document = client.get("/agentenv/ext/website/openapi").json() + assert document["openapi"].startswith("3.") and document["info"]["title"] == "Shop" + listed = document["paths"]["/api/orders"]["get"] + assert listed["operationId"] == "list_orders" and listed["summary"] == "List the signed-in user's orders." + placed = document["paths"]["/api/orders"]["post"] + assert placed["x-agentenv-roles"] == ["default", "agent"] + assert placed["requestBody"]["content"]["application/json"]["schema"]["type"] == "object" + assert listed["parameters"] == [] + + +def test_the_default_store_round_trips_through_the_data_plane(client: TestClient, shop: WebsiteEnvironment) -> None: + assert rpc(client, "data/get")["parts"][0]["data"] == SEED + rpc(client, "data/reset") + assert rpc(client, "data/get")["parts"][0]["data"] == {} + as_file = base64.b64encode(json.dumps({"orders": [{"id": 7, "user": "carol"}]}).encode()).decode() + rpc(client, "data/add", {"parts": [{"kind": "data", "data": {"current_user": "carol", "users": [{"id": "carol"}]}}, + {"kind": "file", "file": {"bytes": as_file, "name": "orders.json"}}, + {"kind": "text", "text": json.dumps({"notes": [{"id": "n1"}]})}]}) + assert rpc(client, "data/get")["parts"][0]["data"] == { + "current_user": "carol", "users": [{"id": "carol"}], "orders": [{"id": 7, "user": "carol"}], + "notes": [{"id": "n1"}]} + assert shop.acting_user() == "carol" + + +def test_a_sites_data_is_read_through_the_data_plane_alone(client: TestClient) -> None: + client.post("/api/orders", json={"sku": "mug"}) + assert rpc(client, "data/get")["parts"][0]["data"]["orders"][-1]["sku"] == "mug" + assert "home" in client.get("/go").text + params = client.get("/.well-known/agent-env.json").json()["capabilities"]["extensions"] + website_params = next(e for e in params if e["uri"] == "urn:agentenv:website/v1")["params"] + assert "state" not in website_params and "state" not in website_params.get("methods", {}) + + +def test_broad_routes_leave_the_built_in_ones_answering_their_paths(dist: Path) -> None: + class Broad(site(dist)): + @api.get("/{slug}") + def page(self, slug: str) -> dict: + return {"page": slug} + + @api.get("/api/{name}") + def named(self, name: str) -> dict: + return {"named": name} + + env = Broad() + env.store.load(SEED) + client = TestClient(env.create_app()) + assert client.get("/api/session").json()["user"] == "alice" + assert client.get("/api/health").json() == {"status": "ok"} + assert client.get("/about").json() == {"page": "about"} and client.get("/api/other").json() == {"named": "other"} + + +def test_a_broad_route_never_serves_a_page_at_the_data_planes_or_cards_paths(dist: Path) -> None: + class Catchall(site(dist)): + @api.get("/{rest:path}") + def page(self, rest: str) -> dict: + return {"page": rest} + + client = TestClient(Catchall().create_app()) + for path in ("/agentenv/anything", "/agentenv/", "/x/../agentenv/y", "/.well-known/agent-env.json/"): + assert client.get(path).status_code == 404, path + assert client.get("/.well-known/agent-env.json").status_code == 200 + assert client.get("/docs/a").json() == {"page": "docs/a"} + + +def test_a_site_with_its_own_hooks_keeps_them(dist: Path) -> None: + class Own(site(dist)): + @get_data + def export(self) -> list: + return [DataPart(data={"mine": True})] + + client = TestClient(Own().create_app()) + assert rpc(client, "data/get")["parts"][0]["data"] == {"mine": True} + assert client.get("/.well-known/agent-env.json").json()["capabilities"]["operations"] == ["data/reset", "data/add", "data/get"] + + +def test_a_seed_data_file_is_the_sites_data_to_start(tmp_path: Path, dist: Path) -> None: + (tmp_path / "seed.json").write_text(json.dumps(SEED)) + env = site(dist, data=str(tmp_path / "seed.json"))() + assert env.store.export() == SEED and env.acting_user() == "alice" + + +def test_the_acting_user_is_signed_in_until_a_session_says_else(client: TestClient) -> None: + assert client.get("/api/orders").json() == [] + first = client.get("/api/session") + assert first.json() == {"user": "alice", "role": "default", "signed_in": True} + cookie = first.headers["set-cookie"] + assert cookie.startswith(f"{COOKIE}=") and "domain" not in cookie.lower() and "httponly" in cookie.lower() + client.post("/api/sign-in", json={"user": "bob"}) + assert client.get("/api/orders").json() == [{"id": 1, "user": "bob"}] + assert client.get("/api/session").json()["user"] == "bob" + assert client.delete("/api/session").json() == {"user": None, "role": "default", "signed_in": False} + assert client.get("/api/session").json()["user"] is None + # The jar holds the site's signed-out cookie under its own domain; a second cookie of the name would race it. + client.cookies.clear() + client.cookies.set(COOKIE, "forged.signature") + assert client.get("/api/session").json()["user"] == "alice" + + +def test_the_session_cookie_expires_when_the_session_does(client: TestClient) -> None: + from email.utils import parsedate_to_datetime + + cookie = client.get("/api/session").headers["set-cookie"] + expires = next(a.split("=", 1)[1] for a in cookie.split(";") if a.strip().lower().startswith("expires=")) + ahead = parsedate_to_datetime(expires) - datetime.datetime.now(datetime.timezone.utc) + assert datetime.timedelta(days=7) - datetime.timedelta(minutes=1) < ahead <= datetime.timedelta(days=7) + + +def test_the_session_operation_mints_cookies_for_a_user_or_signed_out(shop: WebsiteEnvironment) -> None: + client = TestClient(shop.create_app()) + minted = client.post("/agentenv/ext/website/session", json={"user": "bob"}).json() + assert minted["user"] == "bob" and minted["expires_at"].endswith("Z") + (cookie,) = minted["cookies"] + assert cookie["name"] == COOKIE and "domain" not in cookie and cookie["path"] == "/" + browser = TestClient(shop.asgi, cookies={COOKIE: cookie["value"]}) + assert browser.get("/api/session").json()["user"] == "bob" + signed_out = client.post("/agentenv/ext/website/session", json={}).json()["cookies"][0]["value"] + assert TestClient(shop.asgi, cookies={COOKIE: signed_out}).get("/api/session").json()["user"] is None + + +def test_each_site_keeps_its_own_session_cookie_and_reads_the_legacy_one(dist: Path, monkeypatch) -> None: + monkeypatch.setenv("ENVIRONMENT_NAME", "shop a/1") + first = site(dist)() + monkeypatch.setenv("ENVIRONMENT_NAME", "shop-b") + second = site(dist)() + for env in (first, second): + env.store.load(SEED) + named = "agentenv_session_shop_a_1~030f5785d67a" + assert (first.session_cookie, second.session_cookie) == (named, "agentenv_session_shop-b") + browser = TestClient(first.create_app()) + browser.post("/api/sign-in", json={"user": "bob"}) + other = TestClient(second.create_app(), cookies=browser.cookies) + other.post("/api/sign-in", json={"user": "alice"}) + assert set(other.cookies.keys()) == {named, "agentenv_session_shop-b"} + assert TestClient(first.asgi, cookies=other.cookies).get("/api/session").json()["user"] == "bob" + card = browser.get("/.well-known/agent-env.json").json() + assert _extension(card, WEBSITE_EXTENSION_URI)["params"]["session_cookie"] == named + legacy = first._session_signer().encode("bob")[0] + assert TestClient(first.asgi, cookies={SESSION_COOKIE: legacy}).get("/api/orders").json() == [{"id": 1, "user": "bob"}] + + +def test_sites_whose_names_spell_alike_keep_their_own_cookies(dist: Path, monkeypatch) -> None: + apps = [] + for name in ("shop a/1", "shop_a_1"): + monkeypatch.setenv("ENVIRONMENT_NAME", name) + env = site(dist)() + env.store.load(SEED) + apps.append(env.create_app()) + browser = TestClient(apps[0]) + browser.post("/api/sign-in", json={"user": "bob"}) + other = TestClient(apps[1], cookies=browser.cookies) + other.post("/api/sign-in", json={"user": "alice"}) + assert len(other.cookies.keys()) == 2 + back = TestClient(apps[0], cookies=other.cookies) + assert back.get("/api/session").json()["user"] == "bob" + + +def test_every_answer_carries_the_universe_time(client: TestClient) -> None: + for response in (client.get("/"), client.get("/api/orders"), client.get("/nope.js"), + client.post(RPC, json={"jsonrpc": "2.0", "id": 1, "method": "data/get"}), + client.get("/.well-known/agent-env.json")): + now = parse_time(response.headers[UNIVERSE_NOW_HEADER]) + assert abs((now - datetime.datetime.now(datetime.timezone.utc)).total_seconds()) < 60 + + +class _Clock: + """A gateway-shaped /clock/time whose time a test sets.""" + + def __init__(self, virtual: str, rate: object = None) -> None: + self.virtual, self.rate = virtual, rate + outer = self + + class Handler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + answer = {"virtual_time": outer.virtual} + if outer.rate is not None: + answer["virtual_seconds_per_real_second"] = outer.rate + body = json.dumps(answer).encode() + self.send_response(200) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args) -> None: + return None + + self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + threading.Thread(target=self.server.serve_forever, daemon=True).start() + self.url = f"http://127.0.0.1:{self.server.server_port}/clock/time" + + def close(self) -> None: + self.server.shutdown() + self.server.server_close() + + +@pytest.fixture +def gateway_clock(): + clock = _Clock("2031-01-01T00:00:00Z") + yield clock + clock.close() + + +def test_sync_time_ties_now_and_the_header_to_the_gateway_clock(client: TestClient, shop: WebsiteEnvironment, + gateway_clock: _Clock, monkeypatch) -> None: + synced = client.post("/agentenv/ext/sync_time", json={"env_get_time_url": gateway_clock.url}).json() + assert synced == {"env_get_time_url": gateway_clock.url, "gateway_virtual_time": "2031-01-01T00:00:00.000Z"} + assert shop.now().year == 2031 + assert client.get("/").headers[UNIVERSE_NOW_HEADER].startswith("2031-01-01T00:00:0") + order = client.post("/api/orders", json={"sku": "pen"}).json() + assert order["at"].startswith("2031-01-01") + gateway_clock.virtual = "2031-06-01T12:00:00Z" + monkeypatch.setattr(clock_module, "REFRESH_S", 0.0) + assert client.get("/").headers[UNIVERSE_NOW_HEADER].startswith("2031-06-01T12:00") + + +def test_sync_time_takes_the_universe_s_time_zone_and_a_later_sync_without_one_drops_it( + client: TestClient, shop: WebsiteEnvironment, gateway_clock: _Clock) -> None: + method = _extension(client.get("/.well-known/agent-env.json").json(), "urn:agentenv:clock/v1")["params"]["methods"]["sync_time"] + assert "timezone" in method["request"]["properties"] and method["request"]["required"] == ["env_get_time_url"] + zoned = client.post("/agentenv/ext/sync_time", json={"env_get_time_url": gateway_clock.url, "timezone": " America/Chicago "}) + assert zoned.json()["timezone"] == "America/Chicago" == shop.timezone + assert shop.now().tzinfo == datetime.timezone.utc + unzoned = client.post("/agentenv/ext/sync_time", json={"env_get_time_url": gateway_clock.url}).json() + assert "timezone" not in unzoned and shop.timezone is None + + +def test_a_clock_that_cannot_be_read_is_refused_and_a_lost_one_extrapolates(client: TestClient, + shop: WebsiteEnvironment, + gateway_clock: _Clock, monkeypatch) -> None: + refused = client.post("/agentenv/ext/sync_time", json={"env_get_time_url": "http://127.0.0.1:9/clock/time"}) + assert refused.status_code == 500 and shop.now().year == datetime.datetime.now().year + client.post("/agentenv/ext/sync_time", json={"env_get_time_url": gateway_clock.url}) + gateway_clock.close() + monkeypatch.setattr(clock_module, "REFRESH_S", 0.0) + assert client.get("/").headers[UNIVERSE_NOW_HEADER].startswith("2031-01-01") + + +def test_clock_patch_puts_the_clock_script_in_html_only(dist: Path, gateway_clock: _Clock) -> None: + client = TestClient(site(dist, clock_patch=True)().create_app()) + client.post("/agentenv/ext/sync_time", json={"env_get_time_url": gateway_clock.url}) + page = client.get("/") + head = page.text.index("") + len("") + assert page.text[head:].startswith("