Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion AGENTS.md

Large diffs are not rendered by default.

17 changes: 14 additions & 3 deletions src/agent_env/cli/env/website.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@
)
from agent_env.utils.card_naming import card_name_from_github, card_name_from_source, website_source_from_github, website_source_from_source
from agent_env.utils.docker.build import DEFAULT_BUILD_PLATFORM, build_image
from agent_env.env.envs.website import WebsiteEnv, declared_live, domains_problem, health_path_problem, live_problems
from agent_env.env.envs.website import (WebsiteEnv, declared_live, domains_problem, health_path_problem, live_problems,
ready_timeout_problem, shm_problem)
from agent_env.providers.env_providers.constants import AGENT_ENV_WEBSITE_HEALTH_PATH
from agent_env.store.ids import derive_id, image_repository

Expand Down Expand Up @@ -57,6 +58,11 @@ def website():
@click.option("--allow-host", "allow_hosts", multiple=True,
help="Another host a live site's pages reach, such as *.githubassets.com (repeatable; default: its "
"@website(...)'s allow_hosts)")
@click.option("--ready-timeout-s", "ready_timeout_s", type=float, default=None,
help="How long the site may take from its server starting to ready, in seconds (default: the gateway's "
"health check and wait)")
@click.option("--shm-mb", "shm_mb", type=int, default=None,
help="The shared memory its server gets in /dev/shm, in MiB (default: Docker's 64 MiB)")
@click.option("--setting", "setting_pairs", multiple=True,
help="A setting its server is handed, KEY=VALUE (repeatable); a secret's value is a secret:KEY or "
"env:NAME reference")
Expand All @@ -81,6 +87,8 @@ def put(
domains: tuple[str, ...],
live: bool | None,
allow_hosts: tuple[str, ...],
ready_timeout_s: float | None,
shm_mb: int | None,
setting_pairs: tuple[str, ...],
environment_name: str | None,
env_provider_type: str,
Expand Down Expand Up @@ -108,7 +116,9 @@ def put(
click.echo("Error: --server-image-ref needs --environment-name, which a registry image's card can't give here",
err=True)
sys.exit(1)
if problem := health_path_problem(health_path) or domains_problem(list(domains)):
if problem := (health_path_problem(health_path) or domains_problem(list(domains))
or (ready_timeout_s is not None and ready_timeout_problem(ready_timeout_s))
or (shm_mb is not None and shm_problem(shm_mb))):
click.echo(f"Error: {problem}", err=True)
sys.exit(1)
if frontend_dockerfile or frontend_dockerfile_github_url:
Expand Down Expand Up @@ -138,7 +148,8 @@ def put(
source = None
declared, problems = declared_live(source, live=live, domains=list(domains) or None,
allow_hosts=list(allow_hosts) or None)
site = {"health_path": health_path, "domains": list(domains), **declared, "settings": settings}
site = {"health_path": health_path, "domains": list(domains), **declared, "settings": settings,
"ready_timeout_s": ready_timeout_s, "shm_mb": shm_mb}
problems += live_problems(site["live"], site["domains"], site["allow_hosts"], settings, site["secret_settings"])
if problems:
click.echo("Error: " + "; ".join(problems), err=True)
Expand Down
58 changes: 55 additions & 3 deletions src/agent_env/env/envs/website.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,19 @@
its own network, and a gateway gives it no site route. The env keeps them, as it keeps ``domains``, since a computer's
network is fixed before any card is read. ``settings`` are what its server is handed as ``WEBSITE_SETTINGS_ENV``,
each of its ``secret_settings`` (its ``Settings``' ``SecretStr`` fields) a ``secret:`` or ``env:`` reference, resolved only
when its container is written."""
when its container is written.

``ready_timeout_s`` (env.toml's too) is how long the site may take from its server's container starting to ready: the
gateway's health check of that container allows it before it counts a failure, and the deploy waits that long, or as
long as its card's ``ready`` says if longer, before it gives up. ``shm_mb`` is the shared memory its server's container
gets in ``/dev/shm``, in MiB. Without them, the health check and the wait are the gateway's own and ``/dev/shm`` is
Docker's."""

from __future__ import annotations

import asyncio
import logging
import math
import os
import posixpath
import shlex
Expand All @@ -30,6 +37,7 @@
from agent_env.artifact.artifacts.docker_image import ProgressCallback, refuse_local_github_build
from agent_env.entity.refs import EntityRef
from agent_env.env.env import Env, gateway_url_of
from agent_env.env.envs.computer.utils.install import MAX_SHM_MB
from agent_env.env.envs.computer.utils.sites import name_problem
from agent_env.store.ids import derive_id
from agent_env.utils.card_naming import WebsiteSource
Expand Down Expand Up @@ -63,7 +71,8 @@ class WebsiteEnv(Env):
optional=True))
toml_keys: ClassVar[dict[str, type]] = {"backend_image": object, "frontend_image": object, "environment_name": str,
"health": str, "domains": list, "env_provider_type": str, "live": bool,
"allow_hosts": list, "settings": dict}
"allow_hosts": list, "settings": dict, "ready_timeout_s": object,
"shm_mb": object}

def __init__(
self,
Expand All @@ -82,6 +91,8 @@ def __init__(
allow_hosts: Optional[list[str]] = None,
settings: Optional[dict[str, Any]] = None,
secret_settings: Optional[list[str]] = None,
ready_timeout_s: Optional[float] = None,
shm_mb: Optional[int] = None,
):
super().__init__(id, version, metadata=metadata)
if not environment_name:
Expand All @@ -101,6 +112,11 @@ def __init__(
self.domains = list(domains or [])
if problems := live_problems(live, self.domains, allow_hosts or [], settings or {}, secret_settings or []):
raise ValueError(f"website env {id!r}: {'; '.join(problems)}")
if problem := (ready_timeout_s is not None and ready_timeout_problem(ready_timeout_s)) or (
shm_mb is not None and shm_problem(shm_mb)):
raise ValueError(f"website env {id!r}: {problem}")
self.ready_timeout_s = ready_timeout_s
self.shm_mb = shm_mb
self.live = live
self.allow_hosts = list(allow_hosts or [])
self.settings = dict(settings or {})
Expand Down Expand Up @@ -150,6 +166,10 @@ def to_dict(self) -> dict:
base["settings"] = dict(self.settings)
if self.secret_settings:
base["secret_settings"] = list(self.secret_settings)
if self.ready_timeout_s is not None:
base["ready_timeout_s"] = self.ready_timeout_s
if self.shm_mb is not None:
base["shm_mb"] = self.shm_mb
base["service_name"] = self.environment_name
base["environment_name"] = self.environment_name # dual-write with service_name during the rename
base["env_provider_type"] = self.env_provider_type
Expand All @@ -174,6 +194,8 @@ def from_dict(cls, data: dict) -> WebsiteEnv:
allow_hosts=data.get("allow_hosts") or [],
settings=data.get("settings") or {},
secret_settings=data.get("secret_settings") or [],
ready_timeout_s=data.get("ready_timeout_s"),
shm_mb=data.get("shm_mb"),
)

@classmethod
Expand Down Expand Up @@ -201,7 +223,9 @@ def from_toml(cls, data: dict, ctx: AuthoringContext) -> WebsiteEnv:
live=fields.get("live", False),
allow_hosts=fields.get("allow_hosts") or [],
settings=fields.get("settings") or {},
secret_settings=fields.get("secret_settings") or [])
secret_settings=fields.get("secret_settings") or [],
ready_timeout_s=fields.get("ready_timeout_s"),
shm_mb=fields.get("shm_mb"))

def to_toml(self, out: ExportContext) -> dict:
"""The env.toml of a folder this env's images are built from, which ``from_toml`` reads back as it."""
Expand All @@ -219,6 +243,10 @@ def to_toml(self, out: ExportContext) -> dict:
toml["allow_hosts"] = list(self.allow_hosts)
if self.settings:
toml["settings"] = dict(self.settings)
if self.ready_timeout_s is not None:
toml["ready_timeout_s"] = self.ready_timeout_s
if self.shm_mb is not None:
toml["shm_mb"] = self.shm_mb
return {**toml, **self.provider_toml(out, self.env_provider_type)}

@classmethod
Expand All @@ -232,6 +260,9 @@ def accept_toml(cls, data: dict, ctx: AuthoringContext) -> dict:
domains = data.get("domains", [])
if isinstance(domains, list) and (problem := domains_problem(domains)):
ctx.refuse([ctx.config_problem(problem)])
if problems := [problem for key, check in (("ready_timeout_s", ready_timeout_problem), ("shm_mb", shm_problem))
if key in data and (problem := check(data[key]))]:
ctx.refuse([ctx.config_problem(problem) for problem in problems])
fields = ctx.accept_env(data, cls, named_by="backend_image")
declared, problems = declared_live(ctx.website_source("backend_image"), live=fields.get("live"),
domains=fields.get("domains"), allow_hosts=fields.get("allow_hosts"))
Expand Down Expand Up @@ -419,6 +450,8 @@ async def put_from_github(
allow_hosts: list[str] | None = None,
settings: dict[str, Any] | None = None,
secret_settings: list[str] | None = None,
ready_timeout_s: float | None = None,
shm_mb: int | None = None,
) -> WebsiteEnv:
"""Build the server image, from ``backend_dockerfile_github_url``, and any deprecated frontend image from
GitHub in parallel, and create a WebsiteEnv."""
Expand Down Expand Up @@ -473,6 +506,8 @@ async def put_from_github(
allow_hosts=allow_hosts,
settings=settings,
secret_settings=secret_settings,
ready_timeout_s=ready_timeout_s,
shm_mb=shm_mb,
)
return env

Expand All @@ -485,6 +520,23 @@ def health_path_problem(health_path: str) -> str | None:
return f"health: {health_path!r} is not a path of letters, digits, '.', '_', '~', '-' and '/' starting with '/'"


def ready_timeout_problem(ready_timeout_s: Any) -> str | None:
"""Why ``ready_timeout_s`` can't be how long a site may take from its container starting to ready, or None: it's a
number of seconds above 0, as ``@website``'s is."""
if isinstance(ready_timeout_s, bool) or not isinstance(ready_timeout_s, (int, float)) or not (
0 < ready_timeout_s < math.inf):
return f"ready_timeout_s is a number of seconds above 0, not {ready_timeout_s!r}"
return None


def shm_problem(shm_mb: Any) -> str | None:
"""Why ``shm_mb`` can't be the shared memory a site's server gets in ``/dev/shm``, or None: it's 1 to MAX_SHM_MB
MiB, as an app's is."""
if isinstance(shm_mb, bool) or not isinstance(shm_mb, int) or not 1 <= shm_mb <= MAX_SHM_MB:
return f"shm_mb is 1 to {MAX_SHM_MB} MiB, not {shm_mb!r}"
return None


def declared_live(source: Optional[WebsiteSource], *, live: Optional[bool] = None, domains: Optional[list[str]] = None,
allow_hosts: Optional[list[str]] = None) -> tuple[dict, list[str]]:
"""A website env's ``live``, ``allow_hosts``, ``secret_settings`` and, for a live one, ``domains``: each one
Expand Down
30 changes: 22 additions & 8 deletions src/agent_env/providers/env_providers/env_gateway_provider.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import asyncio
import functools
import json
import math
import logging
import re
import secrets
Expand Down Expand Up @@ -155,6 +156,10 @@ class WebsiteConfig:
# server, each ``secret:`` or ``env:`` reference resolved only as the compose file is written.
live: bool = False
settings: dict = field(default_factory=dict)
# How long its server may take from its container starting to ready, which its health check allows before it counts
# a failure and the deploy waits, and its server's /dev/shm in MiB; None keeps the gateway's own and Docker's.
ready_timeout_s: float | None = None
shm_mb: int | None = None


@dataclass
Expand Down Expand Up @@ -572,8 +577,9 @@ def create_docker_compose(
f' || curl -sf -o /dev/null http://localhost:{AGENT_ENV_WEBSITE_BACKEND_PORT}{wc.health_path}"]',
" interval: 10s",
" timeout: 5s",
" start_period: 30s",
f" start_period: {_seconds(wc.ready_timeout_s) if wc.ready_timeout_s else '30s'}",
" retries: 3",
*([f" shm_size: {wc.shm_mb}m"] if wc.shm_mb else []),
*_network_lines(_ENV_NETWORK, *([] if wc.frontend_image or wc.live else sites_network)),
"",
])
Expand Down Expand Up @@ -1552,12 +1558,13 @@ class _GatewayTopology:


def website_config(env: WebsiteEnv) -> WebsiteConfig:
"""A website env as the gateway runs it: its server image, any deprecated frontend image, and its health path."""
"""A website env as the gateway runs it: its server image, any deprecated frontend image, its health path, how long
it may take to be ready and its shared memory."""
frontend = env.frontend_docker_image_artifact
return WebsiteConfig(backend_image=env.server_docker_image_artifact.image_name,
frontend_image=frontend.image_name if frontend else None,
environment_name=env.environment_name, health_path=env.health_path, live=env.live,
settings=dict(env.settings))
settings=dict(env.settings), ready_timeout_s=env.ready_timeout_s, shm_mb=env.shm_mb)


def _gateway_topology(env: Env) -> _GatewayTopology:
Expand Down Expand Up @@ -1679,6 +1686,11 @@ def _settings_lines(settings: dict) -> list[str]:
return [f" - {json.dumps(f'{WEBSITE_SETTINGS_ENV}={json.dumps(resolved)}'.replace('$', '$$'))}"]



def _seconds(seconds: float) -> str:
"""``seconds`` as a compose duration, in whole seconds rounded up, so a health check never allows less."""
return f"{math.ceil(seconds)}s"

# Budget for reading the gateway's /state once its card is served: answering it runs tool discovery, one server at a
# time, if startup's didn't complete.
_STATE_READ_TIMEOUT_S = 60.0
Expand All @@ -1695,19 +1707,21 @@ async def _wait_for_websites(gateway_url: str, website_configs: list[WebsiteConf
timeout_s: float = WEBSITE_READY_TIMEOUT_S, poll_s: float = WEBSITE_READY_POLL_S) -> None:
"""Wait until each site is ready, as its card says through ``/svc/<name>/``: by ``website/v1``'s ``ready``, or,
for a card without it or a site serving none, by its stored health path answering 200, which is deprecated.
A site whose answer says how long it may take (``timeout_s``) is waited that long and ``WEBSITE_READY_MARGIN_S``,
any other ``timeout_s``. Raises naming each site still not ready, and why, once one is waited past its time, and at
once, with ``WebsiteContractError``, for a site that answers its card's path with something that isn't a card."""
A site whose answer says how long it may take (``timeout_s``) is waited that long, or its own ``ready_timeout_s``
if longer, and ``WEBSITE_READY_MARGIN_S``; any other its ``ready_timeout_s`` and the margin, or ``timeout_s`` if
longer. Raises naming each site still not ready, and why, once one is waited past its time, and at once, with
``WebsiteContractError``, for a site that answers its card's path with something that isn't a card."""
started = time.monotonic()
waiting = {wc.environment_name: wc for wc in website_configs}
own = {wc.environment_name: wc.ready_timeout_s or 0 for wc in website_configs}
why: dict[str, str] = {}
limit: dict[str, float] = {}
warned: set[str] = set()
while waiting:
for name, wc in list(waiting.items()):
ready, why[name], declared = await _website_ready(f"{gateway_url}/svc/{name}", wc.health_path)
if declared is not None:
limit[name] = declared + WEBSITE_READY_MARGIN_S
limit[name] = max(declared, own[name]) + WEBSITE_READY_MARGIN_S
if ready is None and name not in warned:
warned.add(name)
logger.warning(f"website {name!r} has no {WEBSITE_EXTENSION_URI} {WEBSITE_READY_OP} on its card, so its "
Expand All @@ -1717,7 +1731,7 @@ async def _wait_for_websites(gateway_url: str, website_configs: list[WebsiteConf
if not waiting:
return
waited = time.monotonic() - started
if any(waited >= limit.get(name, timeout_s) for name in waiting):
if any(waited >= limit.get(name, max(timeout_s, own[name] + WEBSITE_READY_MARGIN_S)) for name in waiting):
raise RuntimeError("websites not ready after %ds: %s" % (
waited, "; ".join(f"{name} ({why[name]})" for name in waiting)))
await asyncio.sleep(poll_s)
Expand Down
8 changes: 8 additions & 0 deletions tst/data/slow_boot_website/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
FROM mirror.gcr.io/library/python:3.12-slim@sha256:423ed6ab25b1921a477529254bfeeabf5855151dc2c3141699a1bfc852199fbf
RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/*
COPY server.py /app/server.py
ENV PORT=8000
EXPOSE 8000
RUN useradd --create-home --uid 10001 appuser
USER appuser
CMD ["python", "/app/server.py"]
29 changes: 29 additions & 0 deletions tst/data/slow_boot_website/server.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
"""A site that answers nothing until it has booted, as a third-party app's server can take minutes to: it sleeps the
seconds its settings' ``boot_s`` gives, then serves 200 at its health path and, at ``/shm``, the bytes its ``/dev/shm``
holds. It serves no card, so a deploy reads its readiness at its health path."""

import http.server
import json
import os
import time

time.sleep(float(json.loads(os.environ.get("AGENTENV_WEBSITE_SETTINGS", "{}")).get("boot_s", 0)))


class Handler(http.server.BaseHTTPRequestHandler):
def do_GET(self):
if self.path == "/api/health":
body = b"ok"
elif self.path == "/shm":
shm = os.statvfs("/dev/shm")
body = str(shm.f_frsize * shm.f_blocks).encode()
else:
self.send_error(404)
return
self.send_response(200)
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)


http.server.ThreadingHTTPServer(("0.0.0.0", int(os.environ["PORT"])), Handler).serve_forever()
Loading
Loading