From 218901f992efcd24bf8337f599726c0623072ddb Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 9 Nov 2026 19:57:39 -0800 Subject: [PATCH 1/2] feat(website): a website env's ready_timeout_s and shm_mb, for a site slow to boot A website env takes ready_timeout_s, how long its server may take from its container starting to ready, and shm_mb, its server's /dev/shm in MiB, in env.toml, the stored env and agent-env env website put. The gateway's health check of the server allows the timeout before it counts a failure (start_period), and the deploy waits it, or as long as the site's card says if longer, so a third-party image that answers nothing for minutes deploys rather than failing after about a minute. shm_mb sets the server container's shm_size. Without them, nothing changes. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- src/agent_env/cli/env/website.py | 17 +++++- src/agent_env/env/envs/website.py | 58 ++++++++++++++++++- .../env_providers/env_gateway_provider.py | 30 +++++++--- tst/data/slow_boot_website/Dockerfile | 8 +++ tst/data/slow_boot_website/server.py | 29 ++++++++++ tst/integration/env/gateway/gateway_test.py | 37 ++++++++++++ tst/unit/bundle/env_toml_test.py | 17 +++++- tst/unit/bundle/to_toml_round_trip_test.py | 3 +- .../env/envs/test_website_server_image.py | 23 ++++++++ .../env_gateway_provider_test.py | 56 ++++++++++++++++++ 11 files changed, 263 insertions(+), 17 deletions(-) create mode 100644 tst/data/slow_boot_website/Dockerfile create mode 100644 tst/data/slow_boot_website/server.py diff --git a/AGENTS.md b/AGENTS.md index 62d6457c7..f775bd93d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -121,7 +121,7 @@ and deserialized through a registry. | Package | Contents | |---|---| -| `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset`, `close` and `load_file_artifact_universe`; teardown closes each env a run deployed before terminating its sandboxes by id. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`, one site: a server image, `backend_image` in its env.toml, serving the pages and API, ready when its card's `website/v1` `ready` answers a `WebsiteReadiness` that says so, as `agentenv_protocol.website` (`website_readiness`, `env_readiness`) defines a website env's health for the deploy and `DeployedEnv.readiness()` alike, or, for a card without `ready`, when its `health` path answers 200, which is deprecated; the gateway container's `env/gateway/constants.py` mirrors the URI and op, a test keeping them equal; a separate frontend image is deprecated, warned of at put and optional in a bundle, and a stored `backend_docker_image_artifact` reads as the server image; a live one, `live = true`, is a real site whose server serves none of its pages, its `live`, `domains` and `allow_hosts` read at put from its server's `@website(...)` (`utils/card_naming.py` `WebsiteSource`) unless given, its `settings`' secrets only as references the gateway resolves as it writes the site's container (`WEBSITE_SETTINGS_ENV`); the gateway gives it no site route, a computer's site table leaves it out, a computer whose network is an allowlist is let reach its names (`MultiEnv.live_hosts`), a browser alone's forward port takes them straight to the real site (`site_proxy.py` `direct`), and a run that deploys one is marked live, `reproducible: false`), `GatewayEnv` (`gateway_server`), `ComputerDriverEnv` (`computer_driver`, `env/envs/computer/driver_env.py`: a computer driver's files and what its class declares for each platform it supports, read at put in a process of its own, `agentenv_protocol.computer._declare`, where the driver's own dependencies needn't be installed, with the `port` it serves on and its install by OS, `[install.]`, or for one distribution `[install."/"]`, which replaces its OS type's there, as its `known_failures` do (`env/envs/computer/utils/install.py`): files, apt `packages` with extra `sources`, a `python` requirements file for a venv, `downloads` by URL and sha256, `run` scripts, the services the base's init runs, and `ports`; it deploys only as a computer env's driver), `ComputerEnv` (`computer`, in `env/envs/computer/`: a base computer and a driver, matched at put by gate 1 before anything is stored, then composed by `composer.py` into an image for each form and arch on a builder of the put's sandbox provider, by default the env's (`utils/builder.py`; `--sandbox` on `env build`, `bundle register` and `run`): this machine's Docker for `local`, else a VM of the provider, which fetches the put's stored files by signed URL, takes the rest inline and pushes to the image store itself, so it builds only with stores a remote VM reaches; each arch goes to the first link that runs it natively, else one that emulates it (a VM registers QEMU; `--no-emulate` leaves it `skipped` instead, for a later put), and its pair records the provider that built it (`builder`); a run or register that reuses a computer env composes what it has left (`pending`), a run only the forms its sandbox's deploys boot (`booted_forms`: on each link, the first form it boots natively, by the deploy's own rule, `boot.first_native`; a compose of a disk adds its container), and a deploy composes first what it boots that no compose started, and a container it runs natively that was built but not probed, as one whose probe a setting that didn't resolve stopped, booting it only once ready (`ComputerEnv.compose_unstarted`); tagged `agentenv-computers:` in the image store by a key of the image's Dockerfile, build context and lock so any env with the same inputs reuses it; every layer's packages, requirements and downloads are resolved at put in a container of the base for each arch (`utils/packages.py`, `utils/resolve_in_base.py`), stored in the object store by sha256 under `agentenv-packages/`, and listed in a lock stored by a digest of its inputs, which any env with the same inputs reuses, and builds install only the lock's files, every install step `RUN --network=none`; each pair's status goes into its metadata, `composed`, the arches built are written as its computer, a `computer` artifact over one index, and `agent-env env build` resumes what a put left; gate 3 (`utils/probing.py`) then boots each native build on the put's sandbox provider and probes it at `full`, making it `ready` or `failed`, and a build no provider runs natively stays `unverified`. Its deploy, through the `computer` env provider (`providers/env_providers/env_computer_provider.py`), boots its first `ready` build on a sandbox provider that runs its arch natively (`native_platforms()`), with no gateway in front and the driver's card as the env card, its `/dev/shm` the most its apps' Linux installs ask for (`shm_mb`, in MiB, which its composed computer and the deploy record keep; `--shm-size` on a container, `AGENTENV_SHM_MB` read by a Linux disk's init, passed only to a provider whose `SHARED_MEMORY` says it gives that much, else the sandbox's default, which the deploy logs), then checks its `display`; `deploy_env` `env_params` take a `form`, `computer_settings` (deploy-phase values of the base's settings; a secret one, here or on the env, only as a bare `secret:`/`env:` reference) and `driver_settings` (the driver's `ComputerSettings`, such as its default coordinate space); a `network` that allows nothing runs it on local Docker's internal network behind a relay (`providers/sandbox_providers/local_docker/relay/`); `reset` and `close` have the driver end its recordings first (`computer-recording/v1` `end_all`, each uploading what's left and its manifest under the grant it started with), and `reset` then boots a fresh container in its place (`reset_container`), keeping the computer's address but not an agent's MCP session to the old driver; its `load_file_artifact_universe` writes files under the session user's home through the driver's `data/add`, `Downloads` by default, each file's bytes through a read grant when the store's grants reach it and its network allows everything, else inline (`env/envs/computer/utils/inside.py`)), `ComputerAppEnv` (`computer_app`, `env/envs/computer/app_env.py`: an app on a computer, a `ComputerAppEnvironment` class its env.toml's `module` names, read at put by the same `_declare`, which declares each OS with its own decorator (`@linux(...)`, or `@install(os_type, spec)` for any OS type; `@linux(distribution=...)` or `@install("/", spec)` for one distribution, which replaces its OS type's there) holding its install, its launch and, optionally, a skill and settings and build settings that add to the class's there, and install steps and ready checks for every OS or those their `os=` names; everything that reads its settings, launch or steps asks for the computer's OS type; deployed alone it is a one-app computer env, a `ComputerEnv` of the computer and driver its env.toml names, the composer installing it under `/opt/agentenv/apps//` with its services as supervisord programs beside its own server (`agentenv_protocol.computer.serve_app`, in a venv with the protocol's `app` extra, on `.agentenv/server.sock` in its folder), which adds its launcher, launches a `boot` app once per boot and says through `computer-app/v1` whether it's ready, as the driver's `computer-ready` asks, its card named after it, and `env_params` `app_settings` handing it values of its `Settings` in `AGENTENV_APP_SETTINGS`, a secret one only as a bare reference, over its env.toml's `[settings]` and a computer env's `settings` beside it in `apps`, each by top-level field (`params.app_settings`; checked at put, a required one left to the deploy; the deploy record keeps the merged values as `app_settings`, secrets withheld); its env.toml's `build_settings`, values of its `BuildSettings` checked at put, are written to `.agentenv/build_settings.json` in its folder, so they're part of the image's key, its install steps read them, and a deploy can't change them). An env type whose services take environment data defines `load_environment_artifact` and `load_environment_universe_artifact`; `load_artifact` refuses an environment artifact aimed at a type that doesn't, at save and before it loads. An env built on the protocol SDK can serve agents a skill (`@agent_skill`, `skill/v1`), which `add_skills` `env_ids` installs under its environment name (behind a gateway, each server's under that server's name); a skill with files beside its SKILL.md goes as a bundle stored once per content under `agentenv-env-skills/` in the object store. `env/gateway/gateway.py` is the env-side gateway server, container-only code. Its site route, `/website//`, never forwards a site's data plane or card (`/agentenv/…`, `/.well-known/agent-env.json`), which agentenv reaches through `/svc//`; a site with no frontend image is served by its server image there. Its CDP route, `/browser//cdp//` (`env/gateway/browser_proxy.py`), reaches the website browser behind the `/svc` child ``, found from that child's card (it, or an app among its children, offering `browser/v1` `cdp` with `/cdp/` in its `computer-app/v1` `agent_routes`): it relays the browser's discovery and WebSocket both ways, each leg pinged every 20 s, and every CDP address the browser hands out, in `json/version`, `json/list` and its `cdp` answer through `/svc//`, comes back at the gateway, everything before `/cdp//` becoming the address the caller reached the gateway at and `/browser/`; each connection is a `browser_connection` trajectory event, without its token. While its clock is armed it serves agents `get_time` (`env/gateway/get_time.py`), which answers in UTC and, given an IANA `time_zone`, also in that zone; executors' working `hours` (`env/gateway/triggers.py`) take a fixed `utc_offset` or an IANA `time_zone`. | +| `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset`, `close` and `load_file_artifact_universe`; teardown closes each env a run deployed before terminating its sandboxes by id. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`, one site: a server image, `backend_image` in its env.toml, serving the pages and API, ready when its card's `website/v1` `ready` answers a `WebsiteReadiness` that says so, as `agentenv_protocol.website` (`website_readiness`, `env_readiness`) defines a website env's health for the deploy and `DeployedEnv.readiness()` alike, or, for a card without `ready`, when its `health` path answers 200, which is deprecated; its env.toml's `ready_timeout_s`, how long it may take from its server's container starting to ready, is that container's health check `start_period` and what the deploy waits, or its card's `ready` timeout if longer, and its `shm_mb` is the container's `shm_size`, each left to the gateway and Docker without them; the gateway container's `env/gateway/constants.py` mirrors the URI and op, a test keeping them equal; a separate frontend image is deprecated, warned of at put and optional in a bundle, and a stored `backend_docker_image_artifact` reads as the server image; a live one, `live = true`, is a real site whose server serves none of its pages, its `live`, `domains` and `allow_hosts` read at put from its server's `@website(...)` (`utils/card_naming.py` `WebsiteSource`) unless given, its `settings`' secrets only as references the gateway resolves as it writes the site's container (`WEBSITE_SETTINGS_ENV`); the gateway gives it no site route, a computer's site table leaves it out, a computer whose network is an allowlist is let reach its names (`MultiEnv.live_hosts`), a browser alone's forward port takes them straight to the real site (`site_proxy.py` `direct`), and a run that deploys one is marked live, `reproducible: false`), `GatewayEnv` (`gateway_server`), `ComputerDriverEnv` (`computer_driver`, `env/envs/computer/driver_env.py`: a computer driver's files and what its class declares for each platform it supports, read at put in a process of its own, `agentenv_protocol.computer._declare`, where the driver's own dependencies needn't be installed, with the `port` it serves on and its install by OS, `[install.]`, or for one distribution `[install."/"]`, which replaces its OS type's there, as its `known_failures` do (`env/envs/computer/utils/install.py`): files, apt `packages` with extra `sources`, a `python` requirements file for a venv, `downloads` by URL and sha256, `run` scripts, the services the base's init runs, and `ports`; it deploys only as a computer env's driver), `ComputerEnv` (`computer`, in `env/envs/computer/`: a base computer and a driver, matched at put by gate 1 before anything is stored, then composed by `composer.py` into an image for each form and arch on a builder of the put's sandbox provider, by default the env's (`utils/builder.py`; `--sandbox` on `env build`, `bundle register` and `run`): this machine's Docker for `local`, else a VM of the provider, which fetches the put's stored files by signed URL, takes the rest inline and pushes to the image store itself, so it builds only with stores a remote VM reaches; each arch goes to the first link that runs it natively, else one that emulates it (a VM registers QEMU; `--no-emulate` leaves it `skipped` instead, for a later put), and its pair records the provider that built it (`builder`); a run or register that reuses a computer env composes what it has left (`pending`), a run only the forms its sandbox's deploys boot (`booted_forms`: on each link, the first form it boots natively, by the deploy's own rule, `boot.first_native`; a compose of a disk adds its container), and a deploy composes first what it boots that no compose started, and a container it runs natively that was built but not probed, as one whose probe a setting that didn't resolve stopped, booting it only once ready (`ComputerEnv.compose_unstarted`); tagged `agentenv-computers:` in the image store by a key of the image's Dockerfile, build context and lock so any env with the same inputs reuses it; every layer's packages, requirements and downloads are resolved at put in a container of the base for each arch (`utils/packages.py`, `utils/resolve_in_base.py`), stored in the object store by sha256 under `agentenv-packages/`, and listed in a lock stored by a digest of its inputs, which any env with the same inputs reuses, and builds install only the lock's files, every install step `RUN --network=none`; each pair's status goes into its metadata, `composed`, the arches built are written as its computer, a `computer` artifact over one index, and `agent-env env build` resumes what a put left; gate 3 (`utils/probing.py`) then boots each native build on the put's sandbox provider and probes it at `full`, making it `ready` or `failed`, and a build no provider runs natively stays `unverified`. Its deploy, through the `computer` env provider (`providers/env_providers/env_computer_provider.py`), boots its first `ready` build on a sandbox provider that runs its arch natively (`native_platforms()`), with no gateway in front and the driver's card as the env card, its `/dev/shm` the most its apps' Linux installs ask for (`shm_mb`, in MiB, which its composed computer and the deploy record keep; `--shm-size` on a container, `AGENTENV_SHM_MB` read by a Linux disk's init, passed only to a provider whose `SHARED_MEMORY` says it gives that much, else the sandbox's default, which the deploy logs), then checks its `display`; `deploy_env` `env_params` take a `form`, `computer_settings` (deploy-phase values of the base's settings; a secret one, here or on the env, only as a bare `secret:`/`env:` reference) and `driver_settings` (the driver's `ComputerSettings`, such as its default coordinate space); a `network` that allows nothing runs it on local Docker's internal network behind a relay (`providers/sandbox_providers/local_docker/relay/`); `reset` and `close` have the driver end its recordings first (`computer-recording/v1` `end_all`, each uploading what's left and its manifest under the grant it started with), and `reset` then boots a fresh container in its place (`reset_container`), keeping the computer's address but not an agent's MCP session to the old driver; its `load_file_artifact_universe` writes files under the session user's home through the driver's `data/add`, `Downloads` by default, each file's bytes through a read grant when the store's grants reach it and its network allows everything, else inline (`env/envs/computer/utils/inside.py`)), `ComputerAppEnv` (`computer_app`, `env/envs/computer/app_env.py`: an app on a computer, a `ComputerAppEnvironment` class its env.toml's `module` names, read at put by the same `_declare`, which declares each OS with its own decorator (`@linux(...)`, or `@install(os_type, spec)` for any OS type; `@linux(distribution=...)` or `@install("/", spec)` for one distribution, which replaces its OS type's there) holding its install, its launch and, optionally, a skill and settings and build settings that add to the class's there, and install steps and ready checks for every OS or those their `os=` names; everything that reads its settings, launch or steps asks for the computer's OS type; deployed alone it is a one-app computer env, a `ComputerEnv` of the computer and driver its env.toml names, the composer installing it under `/opt/agentenv/apps//` with its services as supervisord programs beside its own server (`agentenv_protocol.computer.serve_app`, in a venv with the protocol's `app` extra, on `.agentenv/server.sock` in its folder), which adds its launcher, launches a `boot` app once per boot and says through `computer-app/v1` whether it's ready, as the driver's `computer-ready` asks, its card named after it, and `env_params` `app_settings` handing it values of its `Settings` in `AGENTENV_APP_SETTINGS`, a secret one only as a bare reference, over its env.toml's `[settings]` and a computer env's `settings` beside it in `apps`, each by top-level field (`params.app_settings`; checked at put, a required one left to the deploy; the deploy record keeps the merged values as `app_settings`, secrets withheld); its env.toml's `build_settings`, values of its `BuildSettings` checked at put, are written to `.agentenv/build_settings.json` in its folder, so they're part of the image's key, its install steps read them, and a deploy can't change them). An env type whose services take environment data defines `load_environment_artifact` and `load_environment_universe_artifact`; `load_artifact` refuses an environment artifact aimed at a type that doesn't, at save and before it loads. An env built on the protocol SDK can serve agents a skill (`@agent_skill`, `skill/v1`), which `add_skills` `env_ids` installs under its environment name (behind a gateway, each server's under that server's name); a skill with files beside its SKILL.md goes as a bundle stored once per content under `agentenv-env-skills/` in the object store. `env/gateway/gateway.py` is the env-side gateway server, container-only code. Its site route, `/website//`, never forwards a site's data plane or card (`/agentenv/…`, `/.well-known/agent-env.json`), which agentenv reaches through `/svc//`; a site with no frontend image is served by its server image there. Its CDP route, `/browser//cdp//` (`env/gateway/browser_proxy.py`), reaches the website browser behind the `/svc` child ``, found from that child's card (it, or an app among its children, offering `browser/v1` `cdp` with `/cdp/` in its `computer-app/v1` `agent_routes`): it relays the browser's discovery and WebSocket both ways, each leg pinged every 20 s, and every CDP address the browser hands out, in `json/version`, `json/list` and its `cdp` answer through `/svc//`, comes back at the gateway, everything before `/cdp//` becoming the address the caller reached the gateway at and `/browser/`; each connection is a `browser_connection` trajectory event, without its token. While its clock is armed it serves agents `get_time` (`env/gateway/get_time.py`), which answers in UTC and, given an IANA `time_zone`, also in that zone; executors' working `hours` (`env/gateway/triggers.py`) take a fixed `utc_offset` or an IANA `time_zone`. | | `artifact/` | Immutable versioned artifacts; every `put()` creates a new version. The document goes to the document store, binary payloads to the object store. Every artifact but an environment universe, whose `metadata` key holds its legacy metadata file ids, carries a `metadata` dict outside its versioned fields, changed by compare-and-swap (`merge_metadata`). Built-ins: `CliArtifact` (`cli`), `ComputerArtifact` (`computer`, a base computer: its architectures, OS, session user, settings and container form, whose image `put` reads from its registry into `metadata.validated_images`), `FileArtifact` (`file`), `FileArtifactUniverse` (`file_artifact_universe`), `DockerImageArtifact` (`docker_image`), `EnvironmentArtifact` (`environment`: its environment's files by path, stored as the one wrapped `file_artifact_ref` every environment artifact had when it has one file at its filename, else as `file_artifact_refs`; `get_file_artifacts()` reads either, and a load hands each file to the env's `data/add` with its path, which a pre-v1 load, taking one file, refuses), `EnvironmentUniverseArtifact` (`environment_universe`), `SkillArtifact` (`skill`); `[artifacts] type_aliases` maps another stored spelling of a type onto one of them. A `DockerImageArtifact` holds a tar.gz of its image, or, written by `put_ref` (`--image-ref` on `env mcp-server put` and `a2a-agent put`), only a registry reference with its tag pinned to a digest, which sandboxes pull, or a published image's tag with its `image_digest` and its build context, which sandboxes pull by the digest and build from the context when they can't (the infra images, and a bundle agent built from a shipped agent's sources, `Plan.published`), or, written by `put_context`, only its build context, which a VM sandbox builds before running it (`VmSandbox.load_docker_images`) and Modal builds before creating its container (`ModalSandboxProvider.prepare_image`, outside a fallback chain's deadline); anything else that runs images by name (a container on the local provider, a lone server on any provider but Modal) refuses such an image. A build context (`utils/docker/context.py`) is what `docker build` would send, `.dockerignore` applied, written so equal files give equal bytes and an equal `source_digest`. | | `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, among them `start_computer_recording` and `end_computer_recording`, and more). On an env whose card serves computer-exec/v1 (a computer), `run_code` with `env_id` and `run_container_unit_tests_verifier` with `env_id` run inside it as its session user through the driver: files staged with `data/add` and read with `data/get`, commands run with `run`; `env_outcome_verifier` hands a `verify` that takes an `env_url` keyword the env's address. A computer's framework-only routes (everything but MCP and the card) take the per-deploy control token its provider boots it with (`DeployedEnv.control_token`, `AGENTENV_CONTROL_TOKEN`); framework calls send `deployed.control_headers`, `Task.run` runs every step under `protocol_v1.authorized` with each deployed env's (a `verify` included), and a `run_code` script inside the computer gets `input["env"]["headers"]`. `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | | `store/` | Four store ABCs with local and cloud implementations: `DocumentStore` (SQLite, MongoDB, DynamoDB, Firestore with MongoDB compatibility), `ObjectStore` (filesystem, S3, Cloud Storage), `ImageStore` (local OCI registry, ECR), `SecretStore` (env vars or file, AWS Secrets Manager, Google Cloud Secret Manager). `VersionedEntityStore` implements the shared versioned get/put logic, `QueryBuilder` is the immutable chained query API, `store/base.py` holds the error types. `store/routing.py` is the local namespaces, `@local` and `@hf` (`is_local_id`), each with a documents file of its own: while the CLI or a `build_app` server runs, a local id and what a local task's run records go to local stores beside the configured ones, and that run's writes to the configured stores are refused. A new backend must pass the conformance kits in `tst/store/`. | diff --git a/src/agent_env/cli/env/website.py b/src/agent_env/cli/env/website.py index f96191591..b12901939 100644 --- a/src/agent_env/cli/env/website.py +++ b/src/agent_env/cli/env/website.py @@ -17,7 +17,8 @@ ) from agent_env.utils.card_naming import card_name_from_github, card_name_from_source, website_source_from_github, website_source_from_source from agent_env.utils.docker.build import DEFAULT_BUILD_PLATFORM, build_image -from agent_env.env.envs.website import WebsiteEnv, declared_live, domains_problem, health_path_problem, live_problems +from agent_env.env.envs.website import (WebsiteEnv, declared_live, domains_problem, health_path_problem, live_problems, + ready_timeout_problem, shm_problem) from agent_env.providers.env_providers.constants import AGENT_ENV_WEBSITE_HEALTH_PATH from agent_env.store.ids import derive_id, image_repository @@ -57,6 +58,11 @@ def website(): @click.option("--allow-host", "allow_hosts", multiple=True, help="Another host a live site's pages reach, such as *.githubassets.com (repeatable; default: its " "@website(...)'s allow_hosts)") +@click.option("--ready-timeout-s", "ready_timeout_s", type=float, default=None, + help="How long the site may take from its server starting to ready, in seconds (default: the gateway's " + "health check and wait)") +@click.option("--shm-mb", "shm_mb", type=int, default=None, + help="The shared memory its server gets in /dev/shm, in MiB (default: Docker's 64 MiB)") @click.option("--setting", "setting_pairs", multiple=True, help="A setting its server is handed, KEY=VALUE (repeatable); a secret's value is a secret:KEY or " "env:NAME reference") @@ -81,6 +87,8 @@ def put( domains: tuple[str, ...], live: bool | None, allow_hosts: tuple[str, ...], + ready_timeout_s: float | None, + shm_mb: int | None, setting_pairs: tuple[str, ...], environment_name: str | None, env_provider_type: str, @@ -108,7 +116,9 @@ def put( click.echo("Error: --server-image-ref needs --environment-name, which a registry image's card can't give here", err=True) sys.exit(1) - if problem := health_path_problem(health_path) or domains_problem(list(domains)): + if problem := (health_path_problem(health_path) or domains_problem(list(domains)) + or (ready_timeout_s is not None and ready_timeout_problem(ready_timeout_s)) + or (shm_mb is not None and shm_problem(shm_mb))): click.echo(f"Error: {problem}", err=True) sys.exit(1) if frontend_dockerfile or frontend_dockerfile_github_url: @@ -138,7 +148,8 @@ def put( source = None declared, problems = declared_live(source, live=live, domains=list(domains) or None, allow_hosts=list(allow_hosts) or None) - site = {"health_path": health_path, "domains": list(domains), **declared, "settings": settings} + site = {"health_path": health_path, "domains": list(domains), **declared, "settings": settings, + "ready_timeout_s": ready_timeout_s, "shm_mb": shm_mb} problems += live_problems(site["live"], site["domains"], site["allow_hosts"], settings, site["secret_settings"]) if problems: click.echo("Error: " + "; ".join(problems), err=True) diff --git a/src/agent_env/env/envs/website.py b/src/agent_env/env/envs/website.py index 4ee1c3e93..001ab5d46 100644 --- a/src/agent_env/env/envs/website.py +++ b/src/agent_env/env/envs/website.py @@ -10,12 +10,19 @@ its own network, and a gateway gives it no site route. The env keeps them, as it keeps ``domains``, since a computer's network is fixed before any card is read. ``settings`` are what its server is handed as ``WEBSITE_SETTINGS_ENV``, each of its ``secret_settings`` (its ``Settings``' ``SecretStr`` fields) a ``secret:`` or ``env:`` reference, resolved only -when its container is written.""" +when its container is written. + +``ready_timeout_s`` (env.toml's too) is how long the site may take from its server's container starting to ready: the +gateway's health check of that container allows it before it counts a failure, and the deploy waits that long, or as +long as its card's ``ready`` says if longer, before it gives up. ``shm_mb`` is the shared memory its server's container +gets in ``/dev/shm``, in MiB. Without them, the health check and the wait are the gateway's own and ``/dev/shm`` is +Docker's.""" from __future__ import annotations import asyncio import logging +import math import os import posixpath import shlex @@ -30,6 +37,7 @@ from agent_env.artifact.artifacts.docker_image import ProgressCallback, refuse_local_github_build from agent_env.entity.refs import EntityRef from agent_env.env.env import Env, gateway_url_of +from agent_env.env.envs.computer.utils.install import MAX_SHM_MB from agent_env.env.envs.computer.utils.sites import name_problem from agent_env.store.ids import derive_id from agent_env.utils.card_naming import WebsiteSource @@ -63,7 +71,8 @@ class WebsiteEnv(Env): optional=True)) toml_keys: ClassVar[dict[str, type]] = {"backend_image": object, "frontend_image": object, "environment_name": str, "health": str, "domains": list, "env_provider_type": str, "live": bool, - "allow_hosts": list, "settings": dict} + "allow_hosts": list, "settings": dict, "ready_timeout_s": object, + "shm_mb": object} def __init__( self, @@ -82,6 +91,8 @@ def __init__( allow_hosts: Optional[list[str]] = None, settings: Optional[dict[str, Any]] = None, secret_settings: Optional[list[str]] = None, + ready_timeout_s: Optional[float] = None, + shm_mb: Optional[int] = None, ): super().__init__(id, version, metadata=metadata) if not environment_name: @@ -101,6 +112,11 @@ def __init__( self.domains = list(domains or []) if problems := live_problems(live, self.domains, allow_hosts or [], settings or {}, secret_settings or []): raise ValueError(f"website env {id!r}: {'; '.join(problems)}") + if problem := (ready_timeout_s is not None and ready_timeout_problem(ready_timeout_s)) or ( + shm_mb is not None and shm_problem(shm_mb)): + raise ValueError(f"website env {id!r}: {problem}") + self.ready_timeout_s = ready_timeout_s + self.shm_mb = shm_mb self.live = live self.allow_hosts = list(allow_hosts or []) self.settings = dict(settings or {}) @@ -150,6 +166,10 @@ def to_dict(self) -> dict: base["settings"] = dict(self.settings) if self.secret_settings: base["secret_settings"] = list(self.secret_settings) + if self.ready_timeout_s is not None: + base["ready_timeout_s"] = self.ready_timeout_s + if self.shm_mb is not None: + base["shm_mb"] = self.shm_mb base["service_name"] = self.environment_name base["environment_name"] = self.environment_name # dual-write with service_name during the rename base["env_provider_type"] = self.env_provider_type @@ -174,6 +194,8 @@ def from_dict(cls, data: dict) -> WebsiteEnv: allow_hosts=data.get("allow_hosts") or [], settings=data.get("settings") or {}, secret_settings=data.get("secret_settings") or [], + ready_timeout_s=data.get("ready_timeout_s"), + shm_mb=data.get("shm_mb"), ) @classmethod @@ -201,7 +223,9 @@ def from_toml(cls, data: dict, ctx: AuthoringContext) -> WebsiteEnv: live=fields.get("live", False), allow_hosts=fields.get("allow_hosts") or [], settings=fields.get("settings") or {}, - secret_settings=fields.get("secret_settings") or []) + secret_settings=fields.get("secret_settings") or [], + ready_timeout_s=fields.get("ready_timeout_s"), + shm_mb=fields.get("shm_mb")) def to_toml(self, out: ExportContext) -> dict: """The env.toml of a folder this env's images are built from, which ``from_toml`` reads back as it.""" @@ -219,6 +243,10 @@ def to_toml(self, out: ExportContext) -> dict: toml["allow_hosts"] = list(self.allow_hosts) if self.settings: toml["settings"] = dict(self.settings) + if self.ready_timeout_s is not None: + toml["ready_timeout_s"] = self.ready_timeout_s + if self.shm_mb is not None: + toml["shm_mb"] = self.shm_mb return {**toml, **self.provider_toml(out, self.env_provider_type)} @classmethod @@ -232,6 +260,9 @@ def accept_toml(cls, data: dict, ctx: AuthoringContext) -> dict: domains = data.get("domains", []) if isinstance(domains, list) and (problem := domains_problem(domains)): ctx.refuse([ctx.config_problem(problem)]) + if problems := [problem for key, check in (("ready_timeout_s", ready_timeout_problem), ("shm_mb", shm_problem)) + if key in data and (problem := check(data[key]))]: + ctx.refuse([ctx.config_problem(problem) for problem in problems]) fields = ctx.accept_env(data, cls, named_by="backend_image") declared, problems = declared_live(ctx.website_source("backend_image"), live=fields.get("live"), domains=fields.get("domains"), allow_hosts=fields.get("allow_hosts")) @@ -419,6 +450,8 @@ async def put_from_github( allow_hosts: list[str] | None = None, settings: dict[str, Any] | None = None, secret_settings: list[str] | None = None, + ready_timeout_s: float | None = None, + shm_mb: int | None = None, ) -> WebsiteEnv: """Build the server image, from ``backend_dockerfile_github_url``, and any deprecated frontend image from GitHub in parallel, and create a WebsiteEnv.""" @@ -473,6 +506,8 @@ async def put_from_github( allow_hosts=allow_hosts, settings=settings, secret_settings=secret_settings, + ready_timeout_s=ready_timeout_s, + shm_mb=shm_mb, ) return env @@ -485,6 +520,23 @@ def health_path_problem(health_path: str) -> str | None: return f"health: {health_path!r} is not a path of letters, digits, '.', '_', '~', '-' and '/' starting with '/'" +def ready_timeout_problem(ready_timeout_s: Any) -> str | None: + """Why ``ready_timeout_s`` can't be how long a site may take from its container starting to ready, or None: it's a + number of seconds above 0, as ``@website``'s is.""" + if isinstance(ready_timeout_s, bool) or not isinstance(ready_timeout_s, (int, float)) or not ( + 0 < ready_timeout_s < math.inf): + return f"ready_timeout_s is a number of seconds above 0, not {ready_timeout_s!r}" + return None + + +def shm_problem(shm_mb: Any) -> str | None: + """Why ``shm_mb`` can't be the shared memory a site's server gets in ``/dev/shm``, or None: it's 1 to MAX_SHM_MB + MiB, as an app's is.""" + if isinstance(shm_mb, bool) or not isinstance(shm_mb, int) or not 1 <= shm_mb <= MAX_SHM_MB: + return f"shm_mb is 1 to {MAX_SHM_MB} MiB, not {shm_mb!r}" + return None + + def declared_live(source: Optional[WebsiteSource], *, live: Optional[bool] = None, domains: Optional[list[str]] = None, allow_hosts: Optional[list[str]] = None) -> tuple[dict, list[str]]: """A website env's ``live``, ``allow_hosts``, ``secret_settings`` and, for a live one, ``domains``: each one diff --git a/src/agent_env/providers/env_providers/env_gateway_provider.py b/src/agent_env/providers/env_providers/env_gateway_provider.py index 78cd52454..5020fd550 100644 --- a/src/agent_env/providers/env_providers/env_gateway_provider.py +++ b/src/agent_env/providers/env_providers/env_gateway_provider.py @@ -5,6 +5,7 @@ import asyncio import functools import json +import math import logging import re import secrets @@ -155,6 +156,10 @@ class WebsiteConfig: # server, each ``secret:`` or ``env:`` reference resolved only as the compose file is written. live: bool = False settings: dict = field(default_factory=dict) + # How long its server may take from its container starting to ready, which its health check allows before it counts + # a failure and the deploy waits, and its server's /dev/shm in MiB; None keeps the gateway's own and Docker's. + ready_timeout_s: float | None = None + shm_mb: int | None = None @dataclass @@ -572,8 +577,9 @@ def create_docker_compose( f' || curl -sf -o /dev/null http://localhost:{AGENT_ENV_WEBSITE_BACKEND_PORT}{wc.health_path}"]', " interval: 10s", " timeout: 5s", - " start_period: 30s", + f" start_period: {_seconds(wc.ready_timeout_s) if wc.ready_timeout_s else '30s'}", " retries: 3", + *([f" shm_size: {wc.shm_mb}m"] if wc.shm_mb else []), *_network_lines(_ENV_NETWORK, *([] if wc.frontend_image or wc.live else sites_network)), "", ]) @@ -1552,12 +1558,13 @@ class _GatewayTopology: def website_config(env: WebsiteEnv) -> WebsiteConfig: - """A website env as the gateway runs it: its server image, any deprecated frontend image, and its health path.""" + """A website env as the gateway runs it: its server image, any deprecated frontend image, its health path, how long + it may take to be ready and its shared memory.""" frontend = env.frontend_docker_image_artifact return WebsiteConfig(backend_image=env.server_docker_image_artifact.image_name, frontend_image=frontend.image_name if frontend else None, environment_name=env.environment_name, health_path=env.health_path, live=env.live, - settings=dict(env.settings)) + settings=dict(env.settings), ready_timeout_s=env.ready_timeout_s, shm_mb=env.shm_mb) def _gateway_topology(env: Env) -> _GatewayTopology: @@ -1679,6 +1686,11 @@ def _settings_lines(settings: dict) -> list[str]: return [f" - {json.dumps(f'{WEBSITE_SETTINGS_ENV}={json.dumps(resolved)}'.replace('$', '$$'))}"] + +def _seconds(seconds: float) -> str: + """``seconds`` as a compose duration, in whole seconds rounded up, so a health check never allows less.""" + return f"{math.ceil(seconds)}s" + # Budget for reading the gateway's /state once its card is served: answering it runs tool discovery, one server at a # time, if startup's didn't complete. _STATE_READ_TIMEOUT_S = 60.0 @@ -1695,11 +1707,13 @@ async def _wait_for_websites(gateway_url: str, website_configs: list[WebsiteConf timeout_s: float = WEBSITE_READY_TIMEOUT_S, poll_s: float = WEBSITE_READY_POLL_S) -> None: """Wait until each site is ready, as its card says through ``/svc//``: by ``website/v1``'s ``ready``, or, for a card without it or a site serving none, by its stored health path answering 200, which is deprecated. - A site whose answer says how long it may take (``timeout_s``) is waited that long and ``WEBSITE_READY_MARGIN_S``, - any other ``timeout_s``. Raises naming each site still not ready, and why, once one is waited past its time, and at - once, with ``WebsiteContractError``, for a site that answers its card's path with something that isn't a card.""" + A site whose answer says how long it may take (``timeout_s``) is waited that long, or its own ``ready_timeout_s`` + if longer, and ``WEBSITE_READY_MARGIN_S``; any other its ``ready_timeout_s`` and the margin, or ``timeout_s`` if + longer. Raises naming each site still not ready, and why, once one is waited past its time, and at once, with + ``WebsiteContractError``, for a site that answers its card's path with something that isn't a card.""" started = time.monotonic() waiting = {wc.environment_name: wc for wc in website_configs} + own = {wc.environment_name: wc.ready_timeout_s or 0 for wc in website_configs} why: dict[str, str] = {} limit: dict[str, float] = {} warned: set[str] = set() @@ -1707,7 +1721,7 @@ async def _wait_for_websites(gateway_url: str, website_configs: list[WebsiteConf for name, wc in list(waiting.items()): ready, why[name], declared = await _website_ready(f"{gateway_url}/svc/{name}", wc.health_path) if declared is not None: - limit[name] = declared + WEBSITE_READY_MARGIN_S + limit[name] = max(declared, own[name]) + WEBSITE_READY_MARGIN_S if ready is None and name not in warned: warned.add(name) logger.warning(f"website {name!r} has no {WEBSITE_EXTENSION_URI} {WEBSITE_READY_OP} on its card, so its " @@ -1717,7 +1731,7 @@ async def _wait_for_websites(gateway_url: str, website_configs: list[WebsiteConf if not waiting: return waited = time.monotonic() - started - if any(waited >= limit.get(name, timeout_s) for name in waiting): + if any(waited >= limit.get(name, max(timeout_s, own[name] + WEBSITE_READY_MARGIN_S)) for name in waiting): raise RuntimeError("websites not ready after %ds: %s" % ( waited, "; ".join(f"{name} ({why[name]})" for name in waiting))) await asyncio.sleep(poll_s) diff --git a/tst/data/slow_boot_website/Dockerfile b/tst/data/slow_boot_website/Dockerfile new file mode 100644 index 000000000..238a4d09a --- /dev/null +++ b/tst/data/slow_boot_website/Dockerfile @@ -0,0 +1,8 @@ +FROM mirror.gcr.io/library/python:3.12-slim@sha256:423ed6ab25b1921a477529254bfeeabf5855151dc2c3141699a1bfc852199fbf +RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/* +COPY server.py /app/server.py +ENV PORT=8000 +EXPOSE 8000 +RUN useradd --create-home --uid 10001 appuser +USER appuser +CMD ["python", "/app/server.py"] diff --git a/tst/data/slow_boot_website/server.py b/tst/data/slow_boot_website/server.py new file mode 100644 index 000000000..99b9d3f91 --- /dev/null +++ b/tst/data/slow_boot_website/server.py @@ -0,0 +1,29 @@ +"""A site that answers nothing until it has booted, as a third-party app's server can take minutes to: it sleeps the +seconds its settings' ``boot_s`` gives, then serves 200 at its health path and, at ``/shm``, the bytes its ``/dev/shm`` +holds. It serves no card, so a deploy reads its readiness at its health path.""" + +import http.server +import json +import os +import time + +time.sleep(float(json.loads(os.environ.get("AGENTENV_WEBSITE_SETTINGS", "{}")).get("boot_s", 0))) + + +class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + if self.path == "/api/health": + body = b"ok" + elif self.path == "/shm": + shm = os.statvfs("/dev/shm") + body = str(shm.f_frsize * shm.f_blocks).encode() + else: + self.send_error(404) + return + self.send_response(200) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + +http.server.ThreadingHTTPServer(("0.0.0.0", int(os.environ["PORT"])), Handler).serve_forever() diff --git a/tst/integration/env/gateway/gateway_test.py b/tst/integration/env/gateway/gateway_test.py index 29a64e1d4..aff73d746 100644 --- a/tst/integration/env/gateway/gateway_test.py +++ b/tst/integration/env/gateway/gateway_test.py @@ -5891,3 +5891,40 @@ async def test_whoop_clock_consumer(sandbox_provider): assert "whoop" in {s["service"] for s in cfg["synced"]}, cfg finally: await env.close() + + +# A site slower to boot than the gateway's own health check allows: a ready timeout to match, and its shared memory. +SLOW_BOOT_S = 180 + + +@pytest.fixture(scope="module") +def slow_boot_site_image() -> DockerImageArtifact: + """The image of a site that answers nothing until it has booted, from ``tst/data/slow_boot_website``.""" + result = subprocess.run(["docker", "build", "--platform", "linux/amd64", "-t", "agentenv-slow-boot-website", + str(TST_DATA_DIR / "slow_boot_website")], capture_output=True, text=True) + if result.returncode != 0: + raise RuntimeError(f"slow boot website build failed: {result.stderr}") + return DockerImageArtifact.put(id="integ-test-slow-boot-website", description="A site slow to boot", + image_name="agentenv-slow-boot-website") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("ready_timeout_s", [SLOW_BOOT_S + 60, None], ids=["ready-timeout", "gateway-default"]) +async def test_a_site_slower_to_boot_than_the_health_check_allows_deploys_with_a_ready_timeout_and_its_shared_memory( + slow_boot_site_image, ready_timeout_s): + env = WebsiteEnv.put(id="integ-test-slow-boot-site", server_docker_image_artifact=slow_boot_site_image, + environment_name="slowboot", settings={"boot_s": SLOW_BOOT_S}, + ready_timeout_s=ready_timeout_s, shm_mb=1024) + try: + if ready_timeout_s is None: + with pytest.raises(RuntimeError, match="slowboot-website-backend"): + await env.deploy() + return + deployed = await env.deploy() + async with httpx.AsyncClient() as client: + shm = await client.get(f"{deployed.gateway_url}/svc/slowboot/shm", timeout=30) + finally: + await env.close() + + assert shm.status_code == 200 and int(shm.text) == 1024 << 20, shm.text + diff --git a/tst/unit/bundle/env_toml_test.py b/tst/unit/bundle/env_toml_test.py index a7da0bd72..c5dea367d 100644 --- a/tst/unit/bundle/env_toml_test.py +++ b/tst/unit/bundle/env_toml_test.py @@ -155,6 +155,16 @@ def test_a_website_with_only_a_server_dockerfile_builds_one_image(bundle_dir, bu assert [call["build"][0] for call in builds] == ["Dockerfile.backend"] +def test_a_website_keeps_how_long_its_env_toml_says_it_may_take_to_be_ready_and_its_shared_memory(bundle_dir, builds): + _bundle(bundle_dir, {"envs/gitlab/Dockerfile.backend": DOCKERFILE, "envs/gitlab/server.py": CARD.format("gitlab"), + "envs/gitlab/env.toml": 'type = "website"\nready_timeout_s = 240\nshm_mb = 1024\n'}, "gitlab") + + _run(bundle_dir) + env = Env.get(f"{ROOT}/gitlab") + + assert (env.ready_timeout_s, env.shm_mb) == (240, 1024) + + def test_a_multi_names_bundle_and_store_envs_and_is_rewritten_when_a_child_is(bundle_dir, builds): with namespace_routing(): MCPServerEnv.put(id="inbox", docker_image_artifact=_store_image("inbox-image"), environment_name="inbox") @@ -212,6 +222,10 @@ def _website(name, toml=""): "envs/x: env.toml: env_provider_type 'server' deploys one MCP server, not a website env"), (_website("x", 'domains = ["Slack.com"]\n'), "envs/x: env.toml: domains: 'Slack.com' must be lowercase, with no trailing dot"), + (_website("x", 'ready_timeout_s = -5\n'), + "envs/x: env.toml: ready_timeout_s is a number of seconds above 0, not -5"), + (_website("x", 'shm_mb = "1g"\n'), + "envs/x: env.toml: shm_mb is 1 to 65536 MiB, not '1g'"), ({**_mcp("x"), "envs/x/env.toml": 'image = { dockerfile = "Dockerfile", build_args = { A = "1" } }\n'}, "envs/x: image: an image built from this folder takes only dockerfile, not build_args"), ({"envs/x/env.toml": 'type = "multi"\nname = "suite"\n'}, @@ -239,7 +253,8 @@ def _website(name, toml=""): "(default_service_db_env_id), and a run builds it when it's missing"), ], ids=["no-card", "several-cards", "store-image-unnamed", "metadata", "empty-name", "name-escapes-its-line", "name-with-a-space", "reserved-name", - "website-on-server", "website-bad-domain", "build-args", "multi-without-envs", "multi-name", "multi-wrong-child", "multi-same-names", + "website-on-server", "website-bad-domain", "website-bad-ready-timeout", "website-bad-shm", + "build-args", "multi-without-envs", "multi-name", "multi-wrong-child", "multi-same-names", "multi-same-container", "multi-same-domain", "gateway", "service-db"]) def test_an_env_toml_that_cant_be_written_is_refused_before_any_write(bundle_dir, files, problem): diff --git a/tst/unit/bundle/to_toml_round_trip_test.py b/tst/unit/bundle/to_toml_round_trip_test.py index 255aa451c..7db25ab91 100644 --- a/tst/unit/bundle/to_toml_round_trip_test.py +++ b/tst/unit/bundle/to_toml_round_trip_test.py @@ -100,10 +100,11 @@ def test_a_website_env_reads_back_with_both_images_built_from_its_one_folder(hom backend = context_image(home, "shop-backend", files, "Dockerfile.backend") frontend = context_image(home, "shop-frontend", files, "Dockerfile.frontend") WebsiteEnv.put(id="shop", backend_docker_image_artifact=backend, frontend_docker_image_artifact=frontend, - environment_name="shop", domains=["shop.test"]) + environment_name="shop", domains=["shop.test"], ready_timeout_s=240, shm_mb=1024) with _read_back(home, [{"id": "env", "type": "deploy_env", "env_id": "shop"}], "env", "shop") as (read, _): assert (type(read), read.environment_name, read.domains) == (WebsiteEnv, "shop", ["shop.test"]) + assert (read.ready_timeout_s, read.shm_mb) == (240, 1024) assert (read.backend_docker_image_artifact.dockerfile_path, read.frontend_docker_image_artifact.dockerfile_path) == ("Dockerfile.backend", "Dockerfile.frontend") assert set(_context(read.frontend_docker_image_artifact)) >= {"site/index.html", "Dockerfile.frontend"} diff --git a/tst/unit/env/envs/test_website_server_image.py b/tst/unit/env/envs/test_website_server_image.py index f7bd47483..a70172a34 100644 --- a/tst/unit/env/envs/test_website_server_image.py +++ b/tst/unit/env/envs/test_website_server_image.py @@ -39,6 +39,29 @@ def test_a_site_declares_its_health_path_and_a_bad_one_is_refused(local_stores): "'~', '-' and '/' starting with '/'") +@pytest.mark.parametrize("field, value, problem", [ + ("ready_timeout_s", 0, "ready_timeout_s is a number of seconds above 0, not 0"), + ("ready_timeout_s", float("inf"), "ready_timeout_s is a number of seconds above 0, not inf"), + ("ready_timeout_s", True, "ready_timeout_s is a number of seconds above 0, not True"), + ("shm_mb", 0, "shm_mb is 1 to 65536 MiB, not 0"), + ("shm_mb", 65537, "shm_mb is 1 to 65536 MiB, not 65537"), + ("shm_mb", 1.5, "shm_mb is 1 to 65536 MiB, not 1.5"), +]) +def test_a_site_keeps_how_long_it_may_take_to_be_ready_and_its_shared_memory_and_refuses_either_out_of_range( + local_stores, field, value, problem): + WebsiteEnv.put(id="gitlab-site", server_docker_image_artifact=_image("gitlab-server"), environment_name="gitlab", + ready_timeout_s=240.5, shm_mb=1024) + stored = Env.get("gitlab-site") + plain = WebsiteEnv.put(id="shop-site", server_docker_image_artifact=_image("shop-server"), environment_name="shop") + + assert (stored.ready_timeout_s, stored.shm_mb) == (240.5, 1024) + assert {"ready_timeout_s", "shm_mb"}.isdisjoint(plain.to_dict()) + with pytest.raises(ValueError) as refused: + WebsiteEnv(id="bad", version=None, server_docker_image_artifact=_image("bad-server"), environment_name="bad", + **{field: value}) + assert str(refused.value) == f"website env 'bad': {problem}" + + def test_a_site_stored_before_the_server_image_reads_its_backend_image_as_it(local_stores): backend, frontend = _image("old-backend"), _image("old-frontend") old = {"type": "website", "id": "old-site", "version": 1, "environment_name": "old", "metadata": {}, diff --git a/tst/unit/providers/env_providers/env_gateway_provider_test.py b/tst/unit/providers/env_providers/env_gateway_provider_test.py index 6e74ff935..76608d34f 100644 --- a/tst/unit/providers/env_providers/env_gateway_provider_test.py +++ b/tst/unit/providers/env_providers/env_gateway_provider_test.py @@ -2584,6 +2584,19 @@ def test_a_websites_server_reaches_the_gateways_clock_at_the_url_the_gateway_han assert gateway["networks"] == site["networks"] == ["env-network"] +@pytest.mark.parametrize("ready_timeout_s, shm_mb, start_period", [(None, None, "30s"), (240, 1024, "240s"), + (0.5, None, "1s")]) +def test_a_sites_ready_timeout_starts_its_servers_health_check_and_its_shared_memory_sizes_its_dev_shm( + ready_timeout_s, shm_mb, start_period): + site = yaml.safe_load(_render([], [WebsiteConfig(backend_image="gitlab-be", frontend_image=None, + environment_name="gitlab", ready_timeout_s=ready_timeout_s, + shm_mb=shm_mb)]))["services"]["gitlab-website-backend"] + + assert site["healthcheck"]["start_period"] == start_period + assert (site["healthcheck"]["interval"], site["healthcheck"]["retries"]) == ("10s", 3) + assert site.get("shm_size") == (f"{shm_mb}m" if shm_mb else None) + + def _sites(handler): """Route the deploy's site checks to ``handler(request) -> httpx.Response``.""" real = httpx.AsyncClient @@ -2694,6 +2707,49 @@ def site(request: httpx.Request) -> httpx.Response: timeout_s=300, poll_s=0) +@pytest.mark.asyncio +async def test_a_sites_own_ready_timeout_waits_it_longer_than_its_card_says_and_than_the_default(monkeypatch): + monkeypatch.setattr(env_gateway_provider, "WEBSITE_READY_MARGIN_S", 0) + answers = iter([{"ready": False, "problems": ["booting"], "waited_s": 1, "timeout_s": 0}, + {"ready": True, "problems": [], "waited_s": 2, "timeout_s": 0}]) + health = iter([503, 200]) + + def site(request: httpx.Request) -> httpx.Response: + if request.url.path == "/svc/mail/.well-known/agent-env.json": + return httpx.Response(200, json=_READY_CARD) + if request.url.path == "/svc/mail/agentenv/ext/website/ready": + return httpx.Response(200, json=next(answers)) + if request.url.path == "/svc/shop/.well-known/agent-env.json": + return httpx.Response(404) + return httpx.Response(next(health)) if request.url.path == "/svc/shop/api/health" else httpx.Response(500) + + with _sites(site): + await env_gateway_provider._wait_for_websites( + "http://gw", [WebsiteConfig(backend_image="m", frontend_image=None, environment_name="mail", + ready_timeout_s=600), + WebsiteConfig(backend_image="s", frontend_image=None, environment_name="shop", + ready_timeout_s=600)], timeout_s=0, poll_s=0) + assert (next(answers, "done"), next(health, "done")) == ("done", "done") + + +@pytest.mark.asyncio +async def test_a_sites_own_ready_timeout_never_shortens_the_wait_its_card_or_the_default_gives_it(monkeypatch): + monkeypatch.setattr(env_gateway_provider, "WEBSITE_READY_MARGIN_S", 0) + answers = iter([{"ready": False, "problems": ["booting"], "waited_s": 1, "timeout_s": 600}, + {"ready": True, "problems": [], "waited_s": 2, "timeout_s": 600}]) + + def site(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/.well-known/agent-env.json"): + return httpx.Response(200, json=_READY_CARD) + return httpx.Response(200, json=next(answers)) + + with _sites(site): + await env_gateway_provider._wait_for_websites( + "http://gw", [WebsiteConfig(backend_image="m", frontend_image=None, environment_name="mail", + ready_timeout_s=1e-9)], timeout_s=300, poll_s=0) + assert next(answers, "done") == "done" + + @pytest.mark.asyncio async def test_a_deploy_reads_a_sites_readiness_through_the_protocols_website_readiness(): asked = [] From 018cc77e32ef4990aad5dde992aeb9f422d02da7 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 9 Nov 2026 20:57:48 -0800 Subject: [PATCH 2/2] test(website): boot the slow site for 100 s, past the health check's minute, to keep CI's shard short Co-Authored-By: Claude Opus 5.5 --- tst/integration/env/gateway/gateway_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tst/integration/env/gateway/gateway_test.py b/tst/integration/env/gateway/gateway_test.py index aff73d746..9c45bd58f 100644 --- a/tst/integration/env/gateway/gateway_test.py +++ b/tst/integration/env/gateway/gateway_test.py @@ -5894,7 +5894,7 @@ async def test_whoop_clock_consumer(sandbox_provider): # A site slower to boot than the gateway's own health check allows: a ready timeout to match, and its shared memory. -SLOW_BOOT_S = 180 +SLOW_BOOT_S = 100 @pytest.fixture(scope="module")