diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 910541a4e..5c0de2b4f 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -70,8 +70,8 @@ updates: groups: infra-base-images: patterns: ["*"] - # The base computers' egress proxy, a Go program their Dockerfiles build: its modules' fixes, uTLS's and - # golang.org/x/crypto's among them. A bump changes the bases' sources, so the next release rebuilds and publishes them. + # The egress proxy, a Go program the base computers' and the Claude Code image's Dockerfiles build: its modules' fixes, + # uTLS's and golang.org/x/crypto's among them. A bump changes their sources, so the next release rebuilds them. # The browser base's copy is the desktop's, byte for byte (a unit test checks it), so one grouped pull request bumps # both. - package-ecosystem: "gomod" diff --git a/.github/workflows/local-backends.yml b/.github/workflows/local-backends.yml index 8c9253afc..ee15008e6 100644 --- a/.github/workflows/local-backends.yml +++ b/.github/workflows/local-backends.yml @@ -251,7 +251,7 @@ jobs: run: | .venv/bin/python .github/scripts/check_skip_policy.py junit-integration-slow.xml \ --allow model_endpoint_configured remote_sandbox default_a2a_agent mcp_server_sources windows_media \ - omarchy_image + omarchy_image agent_clock_image # The tests' times, for .github/scripts/slow_test_durations.py to rebalance the shards with. - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 if: always() diff --git a/src/agent_env/agents/README.md b/src/agent_env/agents/README.md index f4041d557..76ce7a0b7 100644 --- a/src/agent_env/agents/README.md +++ b/src/agent_env/agents/README.md @@ -34,6 +34,22 @@ turn off becomes `--disallowedTools`, which keeps the CLI's defaults for the res `claude_bridge.py`: every tool the pinned CLI has, though which it offers by default depends on the model. A test fails when the Dockerfile's `CLAUDE_CODE_VERSION` moves, until `ClaudeCodeTool` and `TOOLS_CLI_VERSION` move with it. +The image follows a task env's clock the way a computer does (FD-5916): it runs the computer base's clock follower +and egress proxy, taken from `utils/faketime_clock` and `utils/egress_proxy` through its `shared_sources.toml`, each as +its own user under root's supervisord (`start-agent`, `root.conf`), with agent-env's libfaketime preloaded for the +agent's user. +The A2A server starts with the real-time marker, so it and its records stay on real time, and serves clock/v1's +`sync_time`, which the protocol's framework serves on the card of any agent in an image with the follower +(`agentenv_protocol.a2a_agent.clock`). Once `sync_env_clock` has armed the clock, it declares the agent at the gateway +as a follower with a read token of its own and calls that `sync_time` with the follower's config, which the server +writes; `deploy_agent` does the same for an agent it deploys while the clock is armed. From then on the CLI, its own +"Today's date is" line included, and every command the model runs read the env's time, and a clock write that waits for +followers waits for the agent. While the clock is armed the CLI goes out through the egress proxy, which keeps a live +site's TLS working at any date the clock shows, and to the env's MCP servers and a plain-HTTP model endpoint directly. +The image's proxy reaches private networks too (`AGENTENV_EGRESS_PRIVATE`), so a model endpoint over TLS at a private +address works at any date as well. Unarmed, the page holds real time and the CLI goes out directly, as before. Started +as a user other than root, the image runs the A2A server alone, on real time, as before. + `install_agent` installs it too, into a container a task already runs or onto a `vm` sandbox's Linux host, with `install.sh` from its folder: Claude Code's native build for the target's CPU and libc, uv, and the agent's requirements in a Python of its own, so the target's Python is left alone. The CLI runs its commands in bash or zsh, so a target with diff --git a/src/agent_env/agents/agents/claude-code/Dockerfile b/src/agent_env/agents/agents/claude-code/Dockerfile index cbe95d6e6..b56404d44 100644 --- a/src/agent_env/agents/agents/claude-code/Dockerfile +++ b/src/agent_env/agents/agents/claude-code/Dockerfile @@ -1,6 +1,20 @@ # Node for the scripts the model runs, without npm or npx; Claude Code itself is a native build. FROM public.ecr.aws/docker/library/node:22-slim@sha256:c3de60bf2f9dd0ac6370e6117950ff62d6e339527e7472301c9c78a017978392 AS node +# The env's clock: the egress proxy, one static Go binary, and agent-env's libfaketime and its TLS shim, taken from +# agent-env's sources as clock/ and egress/ (shared_sources.toml) and built for this image's libc (Debian bookworm) in +# stages of their own, so no toolchain is left in the image. +FROM public.ecr.aws/docker/library/golang:1.26.11-bookworm@sha256:dc40a13b61304647ed7e5d581d246b86413dfc29d63c6d018310feb93cd98fc0 AS egress +COPY egress/ /egress/ +RUN cd /egress && go vet ./... && go test -count=1 ./... \ + && CGO_ENABLED=0 GOFLAGS=-mod=readonly go build -trimpath -ldflags=-s -o /out/egress . + +FROM public.ecr.aws/docker/library/python:3.11-slim@sha256:c20888b6acdd1e63e1c433a185bf3ad162c0288fe484616ce062e0d28add2900 AS clock +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + build-essential patch curl ca-certificates +COPY clock/ /clock/ +RUN sh /clock/build.sh /out + FROM public.ecr.aws/docker/library/python:3.11-slim@sha256:c20888b6acdd1e63e1c433a185bf3ad162c0288fe484616ce062e0d28add2900 # poppler-utils: the CLI's Read tool runs pdftoppm for a PDF's pages. RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl poppler-utils \ @@ -24,13 +38,35 @@ COPY requirements.txt model-requirements.txt ./ RUN pip install --no-cache-dir -r requirements.txt -r model-requirements.txt # install.sh and run.sh are install/v1's: in the image, so they stay in its build context. COPY a2a_server.py claude_bridge.py kit.py tar_changelog.py entrypoint.sh install.sh run.sh ./ -# The CLI's cwd, /app, and its state, fixed so neither follows the uid: Modal runs the image as root. +# The CLI's cwd, /app, and its state, fixed so neither follows the uid. RUN useradd --create-home --uid 10001 agent && mkdir -p /app /home/agent/.claude \ && chown agent:agent /app /home/agent/.claude && chmod 0755 entrypoint.sh +# The env's clock: the follower and the egress proxy, each as its own user, under root's supervisord (root.conf), and +# agent-env's libfaketime and its TLS shim preloaded into every process, faking only the agent's (clock.conf). The +# follower's config is the agent's user's to write (agentenv_protocol.a2a_agent.clock) and the follower's to read. +RUN pip install --no-cache-dir supervisor==4.2.5 \ + && useradd --system --no-create-home --home-dir /nonexistent --shell /usr/sbin/nologin agentenv-clock \ + && useradd --system --no-create-home --home-dir /nonexistent --shell /usr/sbin/nologin agentenv-egress \ + && mkdir -p /usr/local/lib/agentenv-clock /usr/local/lib/agentenv-egress /etc/agentenv \ + && install -d -o agent -g agentenv-clock -m 2750 /etc/agentenv/clock \ + && touch /etc/agentenv/ca.pem \ + && cp /etc/ssl/certs/ca-certificates.crt /etc/agentenv/ca-bundle.pem \ + && cp /etc/ssl/certs/ca-certificates.crt /usr/local/lib/agentenv-egress/roots.pem +COPY --from=clock /out/libfaketimeMT.so.1 /out/libtls_realtime.so /usr/local/lib/agentenv-clock/ +COPY --from=egress /out/egress /usr/local/lib/agentenv-egress/egress +COPY clock/clockpage.py clock/clock-follower /usr/local/lib/agentenv-clock/ +COPY clock/clock.conf /etc/agentenv/clock.conf +COPY root.conf /etc/agentenv/root.conf +COPY start-agent /usr/local/bin/start-agent +RUN sed -i -e "s/^AGENTENV_SESSION_UID=.*/AGENTENV_SESSION_UID=$(id -u agent)/" \ + -e "s/^AGENTENV_CLOCK_PAGE_UID=.*/AGENTENV_CLOCK_PAGE_UID=$(id -u agentenv-clock)/" /etc/agentenv/clock.conf \ + && chmod 0755 /usr/local/lib/agentenv-clock/clock-follower /usr/local/bin/start-agent \ + && printf '%s\n' /usr/local/lib/agentenv-clock/libfaketimeMT.so.1 /usr/local/lib/agentenv-clock/libtls_realtime.so \ + > /etc/ld.so.preload ENV A2A_PORT=8000 PYTHONPATH=/opt/agent CLAUDE_CONFIG_DIR=/home/agent/.claude \ DISABLE_AUTOUPDATER=1 CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1 -USER agent WORKDIR /app EXPOSE 8000 -# CMD rather than ENTRYPOINT: Modal passes a sandbox's env to the image's CMD only. -CMD ["/opt/agent/entrypoint.sh"] +# Starts as root, which start-agent needs for the clock; the A2A server and the CLI run as the agent's user. CMD rather +# than ENTRYPOINT: Modal passes a sandbox's env to the image's CMD only. +CMD ["/usr/local/bin/start-agent"] diff --git a/src/agent_env/agents/agents/claude-code/claude_bridge.py b/src/agent_env/agents/agents/claude-code/claude_bridge.py index e8fcfc651..e1a30956d 100644 --- a/src/agent_env/agents/agents/claude-code/claude_bridge.py +++ b/src/agent_env/agents/agents/claude-code/claude_bridge.py @@ -11,12 +11,14 @@ import re import signal import stat +import struct import time -from collections.abc import Callable, Mapping +from collections.abc import Callable, Iterable, Mapping from dataclasses import dataclass, field, replace from functools import partial from pathlib import Path from typing import Literal +from urllib.parse import urlsplit from pydantic import BaseModel, ConfigDict, model_validator @@ -173,8 +175,74 @@ def _model_headers(raw: str | None) -> dict[str, str]: return {str(name): str(value) for name, value in headers.items()} +#: Set by the image (root.conf) where the egress proxy runs: the CLI and the model's commands go out through it +#: while the agent follows an armed clock. +EGRESS_PROXY_ENV = "AGENTENV_EGRESS_PROXY" +#: The clock page agent-env's libfaketime reads the CLI's time from (clock/clockpage.py, version 2): it keeps the last +#: armed clock while the follower reconnects or restarts, which the follower's status doesn't say. +CLOCK_PAGE = "/dev/shm/agentenv-clock" +_PAGE = struct.Struct(" bool: + """Whether the clock page holds an armed clock, so the CLI started now reads the env's time: the page itself, + as libfaketime reads it, retried while a write is under way (seq odd) or its check doesn't match.""" + try: + with open(page_path or CLOCK_PAGE, "rb") as page: + for _ in range(_PAGE_READ_TRIES): + page.seek(0) + raw = page.read(_PAGE.size) + if len(raw) < _PAGE.size: + return False + magic, version, seq, universe, anchor, rate, armed, _, generation, checked, mono_u, mono_a = \ + _PAGE.unpack(raw) + if magic != _PAGE_MAGIC or version != _PAGE_VERSION: + return False + rate_bits = struct.unpack(" list[str]: + """The model endpoint's host when the CLI reaches it directly on an armed clock: over plain HTTP, which has no + certificate to check at the env's date. Over TLS it goes through the egress proxy, which checks the certificate on + real time and, in an agent's image, reaches a private network's address too (``AGENTENV_EGRESS_PRIVATE``).""" + endpoint = urlsplit(base_url or "") + return [endpoint.hostname] if endpoint.hostname and endpoint.scheme == "http" else [] + + +def egress_env(environ: Mapping[str, str], direct: Iterable[str] = (), *, armed: bool = False) -> dict[str, str]: + """The egress proxy for the CLI and what it starts, where the image runs one (FD-5916) and the clock is ``armed``: + they read the env's time, so a client checking a live site's certificate against it would refuse the site once + the clock is months off, and the proxy, on real time, hands it a certificate that holds then. The env's MCP servers + (``direct``) are reached directly: the proxy refuses an address that isn't global, as a local or private gateway's + is. Unarmed, the CLI reads real time and goes out directly, as an image without the proxy does, so a model endpoint + or mirror at a private address keeps working. Hosts the agent was deployed to reach directly (``NO_PROXY``, + ``no_proxy``) stay direct.""" + proxy = environ.get(EGRESS_PROXY_ENV) + if not proxy or not armed: + return {} + configured = [host.strip() for name in ("NO_PROXY", "no_proxy") for host in (environ.get(name) or "").split(",")] + hosts = ",".join(dict.fromkeys([*_DIRECT_HOSTS, *filter(None, configured), *filter(None, direct)])) + return {**{name: proxy for base in ("HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY") for name in (base, base.lower())}, + "NO_PROXY": hosts, "no_proxy": hosts, "NODE_USE_ENV_PROXY": "1"} + + def cli_env(environ: Mapping[str, str], *, model: str | None = None, project_id: str | None = None, - task_id: str | None = None, max_thinking_tokens: int | None = None) -> dict[str, str]: + task_id: str | None = None, max_thinking_tokens: int | None = None, + direct_hosts: Iterable[str] = (), clock_armed: bool = False) -> dict[str, str]: """The CLI's environment: this process's, pointed at the endpoint agent-env configured (its key as a bearer token, so an API key too would conflict), with the endpoint's headers and the run's attribution, and the model's aliases pinned to ``model`` on that endpoint. A variable already set is kept, LITELLM_* too, so a script the model @@ -203,6 +271,8 @@ def cli_env(environ: Mapping[str, str], *, model: str | None = None, project_id: # Older CLIs prepend a changing attribution block to the system prompt on custom base URLs, which defeats the # provider's prefix cache. env["CLAUDE_CODE_ATTRIBUTION_HEADER"] = "0" + model_hosts = model_direct_hosts(env.get("ANTHROPIC_BASE_URL")) if clock_armed else [] + env.update(egress_env(environ, [*direct_hosts, *model_hosts], armed=clock_armed)) return env @@ -682,7 +752,9 @@ async def _spawn_claude( stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, env=cli_env(os.environ, model=model, project_id=project_id, task_id=task_id, - max_thinking_tokens=max_thinking_tokens), + max_thinking_tokens=max_thinking_tokens, clock_armed=clock_armed(), + direct_hosts=[urlsplit(str(entry.get("url") or "")).hostname or "" + for entry in (mcp_servers or {}).values()]), cwd=WORKSPACE_DIR, start_new_session=True, )) diff --git a/src/agent_env/agents/agents/claude-code/requirements.txt b/src/agent_env/agents/agents/claude-code/requirements.txt index 464574902..6053dbde7 100644 --- a/src/agent_env/agents/agents/claude-code/requirements.txt +++ b/src/agent_env/agents/agents/claude-code/requirements.txt @@ -1,4 +1,5 @@ # The image installs the newest agentenv-framework-protocol at or above this floor; a release doesn't pin it, so the -# image is built again only when this folder changes. The floor is the first that serves peer-agents/v1's tools, which -# the agent advertises without handlers of its own; raise it when the agent needs a newer protocol. -agentenv-framework-protocol[agent]>=0.1.351 +# image is built again only when this folder changes. The floor is the first that serves clock/v1's sync_time on an +# agent whose image runs the clock follower (agentenv_protocol.a2a_agent.clock), which this image does; raise it when +# the agent needs a newer protocol. +agentenv-framework-protocol[agent]>=0.1.575 diff --git a/src/agent_env/agents/agents/claude-code/root.conf b/src/agent_env/agents/agents/claude-code/root.conf new file mode 100644 index 000000000..af730a2d9 --- /dev/null +++ b/src/agent_env/agents/agents/claude-code/root.conf @@ -0,0 +1,62 @@ +; Root's supervisord (start-agent, FD-5916): the clock follower and the egress proxy, each as its own user, on real +; time, and the A2A server as the agent's user. The server starts with the real-time marker, which agent-env's +; libfaketime takes out of its environment, so the server's own records keep real timestamps and everything it starts, +; the CLI and the model's commands, reads the env's time. Its socket and logs are root's alone, so the agent's user can +; neither stop nor restart any of them. The follower follows nothing until the server writes its config, which +; sync_env_clock has it do, then picks it up within a quarter second. +[supervisord] +nodaemon=true +user=root +logfile=/var/log/agentenv-root/supervisord.log +pidfile=/run/agentenv-root/supervisord.pid +childlogdir=/var/log/agentenv-root + +[unix_http_server] +file=/run/agentenv-root/supervisor.sock +chmod=0700 + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[supervisorctl] +serverurl=unix:///run/agentenv-root/supervisor.sock + +[program:clock] +command=/usr/local/bin/python3 -I /usr/local/lib/agentenv-clock/clock-follower +user=agentenv-clock +directory=/ +priority=3 +autorestart=true +startsecs=1 +startretries=1000 +redirect_stderr=true +stdout_logfile=/var/log/agentenv-root/clock.log +environment=HOME="/nonexistent",USER="agentenv-clock",LOGNAME="agentenv-clock" + +[program:egress] +command=/usr/local/lib/agentenv-egress/egress serve +user=agentenv-egress +directory=/ +priority=3 +autorestart=true +startsecs=1 +startretries=1000 +redirect_stderr=true +stdout_logfile=/var/log/agentenv-root/egress.log +environment=HOME="/nonexistent",USER="agentenv-egress",LOGNAME="agentenv-egress",AGENTENV_EGRESS_PRIVATE="allow" + +; uvicorn itself, not entrypoint.sh: the marker holds for the process it's given to alone, and a shell that execs +; uvicorn would hand uvicorn an environment without it. +[program:agent] +command=/usr/local/bin/python3 -m uvicorn a2a_server:app --host 0.0.0.0 --port %(ENV_A2A_PORT)s +user=agent +directory=/app +priority=10 +autorestart=true +stopasgroup=true +killasgroup=true +stopwaitsecs=30 +redirect_stderr=true +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +environment=HOME="/home/agent",USER="agent",LOGNAME="agent",AGENTENV_REAL_TIME="1",AGENTENV_EGRESS_PROXY="http://127.0.0.1:3128",NODE_EXTRA_CA_CERTS="/etc/agentenv/ca.pem",SSL_CERT_FILE="/etc/agentenv/ca-bundle.pem",REQUESTS_CA_BUNDLE="/etc/agentenv/ca-bundle.pem" diff --git a/src/agent_env/agents/agents/claude-code/shared_sources.toml b/src/agent_env/agents/agents/claude-code/shared_sources.toml new file mode 100644 index 000000000..dc9d15a01 --- /dev/null +++ b/src/agent_env/agents/agents/claude-code/shared_sources.toml @@ -0,0 +1,4 @@ +# What this image's build context takes from agent-env's own sources, at the name its Dockerfile copies it from: one +# copy of each, shared with the other images that take it (utils/docker/context.py). Paths are under src/agent_env. +clock = "utils/faketime_clock" +egress = "utils/egress_proxy" diff --git a/src/agent_env/agents/agents/claude-code/start-agent b/src/agent_env/agents/agents/claude-code/start-agent new file mode 100755 index 000000000..1f170978d --- /dev/null +++ b/src/agent_env/agents/agents/claude-code/start-agent @@ -0,0 +1,19 @@ +#!/bin/sh +# The image's start (FD-5916). As root, it makes what the agent's user must not touch, the clock page and file (the +# clock user's from their first byte) and the egress proxy's CA (its key the egress user's alone, trusted in the +# system store, agent-env's trust files and every certifi bundle), then starts root's supervisord (root.conf), which +# runs the clock follower and the egress proxy, each as its own user on real time, and the A2A server as the agent's +# user on real time, everything it starts reading the env's time from the page. Until sync_env_clock has the agent +# follow a clock, the page holds real time and the proxy relays TLS untouched. Started as another user, the image runs +# the A2A server alone, as before, on real time. +set -eu +if [ "$(id -u)" != 0 ]; then + exec /opt/agent/entrypoint.sh +fi +install -d -o root -g root -m 0700 /var/log/agentenv-root /run/agentenv-root +install -d -o agentenv-clock -g agentenv-clock -m 0755 /run/agentenv-clock +python3 -I /usr/local/lib/agentenv-clock/clockpage.py create agentenv-clock +AGENTENV_SESSION_USER=agent AGENTENV_EGRESS_NSS=off \ + /usr/local/lib/agentenv-egress/egress boot > /var/log/agentenv-root/egress-boot.log 2>&1 \ + || { cat /var/log/agentenv-root/egress-boot.log >&2; echo "the egress proxy's boot failed" >&2; exit 2; } +exec supervisord -c /etc/agentenv/root.conf diff --git a/src/agent_env/utils/egress_proxy/README.md b/src/agent_env/utils/egress_proxy/README.md index 72cc086d5..439e5abc1 100644 --- a/src/agent_env/utils/egress_proxy/README.md +++ b/src/agent_env/utils/egress_proxy/README.md @@ -1,16 +1,17 @@ # The egress proxy `egress` is the `ubuntu-24.04` base computer's explicit proxy, the way every session program goes out. The session's -programs read the universe's time (the clock library in `../clock/`), so a client that checks a site's certificate with +programs read the universe's time (the clock library in `../faketime_clock/`), so a client that checks a site's certificate with its own TLS, Chromium, Java or an official Node build, refuses a live site once the clock is months back or ahead. The proxy runs on real time and hands such a client a certificate that holds at the universe's date when it must. -It is one static Go program. The base's Dockerfile builds it in a stage of its own, after `go vet` and its tests, so no -Go toolchain is left in the image. `go.sum` pins its modules, and Dependabot's `gomod` entry keeps them current. +It is one static Go program. Each image that runs it takes this folder through its `shared_sources.toml` (the two +bases and the Claude Code agent image), and its Dockerfile builds it in a stage of its own, after `go vet` and its +tests, so no Go toolchain is left in the image. `go.sum` pins its modules, and Dependabot's `gomod` entry keeps them current. ## What runs where -- `egress boot` runs once, as root, in the base's init (`../start-desktop`), before root's supervisord starts: +- `egress boot` runs once, as root, in the base's init (`computers/artifacts/ubuntu-24.04/start-desktop`), before root's supervisord starts: - it mints the computer's CA once, valid from 1950 to 9999 as routed sites' certificates are, its key in `/etc/agentenv/egress/` and readable by `agentenv-egress` alone; - it trusts the CA in the system store, agent-env's trust files (`/etc/agentenv/ca.pem`, `ca-bundle.pem`) and the @@ -18,10 +19,10 @@ Go toolchain is left in the image. `go.sum` pins its modules, and Dependabot's ` - it trusts the CA in every certifi bundle and every JDK's store an app brings, naming any that can't take it among its warnings, and the computer boots; - its report is `/var/log/agentenv-root/egress-boot.log`. -- `egress serve` runs as `agentenv-egress` under root's supervisord (`../root.conf`), on `127.0.0.1:3128`, or where +- `egress serve` runs as `agentenv-egress` under root's supervisord (the base's `root.conf`), on `127.0.0.1:3128`, or where `AGENTENV_EGRESS_PROXY` names, as a base whose own proxy holds that address does. The clock library leaves that user on real time. -- The session's supervisord (`../supervisord.conf`), and a login shell through `agentenv-egress.sh`, point +- The session's supervisord (the base's `supervisord.conf`), and a login shell through `agentenv-egress.sh`, point `HTTP(S)_PROXY`, `ALL_PROXY`, Java's proxy properties (`JAVA_TOOL_OPTIONS`) and Node's `fetch` (`NODE_USE_ENV_PROXY`) at it. - agent-env's own servers in the computer, the driver and its apps' servers, run on real time and send their own @@ -34,7 +35,10 @@ Go toolchain is left in the image. `go.sum` pins its modules, and Dependabot's ` rewritten, then bytes are relayed both ways. An upgrade keeps its `Connection` and `Upgrade` headers; any other request closes once answered. - **`CONNECT`**: the destination's name is resolved, and the connection is refused unless every address is global - (`notGlobal` lists the blocks that aren't). A routed site's name, in the hosts file's `# agentenv sites` block, goes + (`notGlobal` lists the blocks that aren't). An agent's image sets `AGENTENV_EGRESS_PRIVATE=allow` for the proxy, + which then reaches private networks' addresses too (RFC 1918, RFC 6598's shared space, IPv6 unique local), as the + agent's processes do directly anyway, so a model endpoint there is reached on real time; loopback, link-local and the + rest stay refused. A computer's proxy doesn't set it. A routed site's name, in the hosts file's `# agentenv sites` block, goes to the site proxy there. Bytes that aren't TLS, and a routed site's TLS, are relayed untouched. - **TLS, with the clock unarmed**: relayed untouched. - **TLS, with the clock armed**: each of the name's addresses is decided by its own probe, a handshake the proxy makes @@ -62,7 +66,7 @@ the session user's reach. Its fields, each there when it applies: |---|---| | `t` | When the line was written, in seconds | | `client`, `method`, `host`, `port` | Who asked, how, and for what | -| `allowed`, `dst` | `global` or `routed`, and the address dialled | +| `allowed`, `dst` | `global`, `private` or `routed`, and the address dialled | | `mode` | `pass`, `splice`, `intercept`, `http`, `upgrade` or `refused` | | `decision` | `real`, `inside`, `outside`, `unverified`, `probe_failed`, `routed` or `not TLS` | | `sni`, `alpn_offered`, `alpn` | The client's server name and protocols, and the one the site picked | diff --git a/src/agent_env/utils/egress_proxy/boot.go b/src/agent_env/utils/egress_proxy/boot.go index e1ea67894..031555627 100644 --- a/src/agent_env/utils/egress_proxy/boot.go +++ b/src/agent_env/utils/egress_proxy/boot.go @@ -31,8 +31,10 @@ type bootPaths struct { NSS func(paths bootPaths) error } +// defaultBootPaths are the computer base's. An image whose session user isn't the base's names it in +// AGENTENV_SESSION_USER, and one with no browser to read an NSS database, an agent's, sets AGENTENV_EGRESS_NSS=off. func defaultBootPaths() bootPaths { - return bootPaths{ + p := bootPaths{ Folder: Folder, PublicCA: PublicCA, Anchor: "/usr/local/share/ca-certificates/agentenv-egress.crt", Bundles: []string{"/etc/agentenv/ca.pem", "/etc/agentenv/ca-bundle.pem"}, UpdateTrust: []string{"update-ca-certificates"}, @@ -41,6 +43,13 @@ func defaultBootPaths() bootPaths { EgressUser: EgressUser, SessionUser: SessionUser, NSS: sessionNSS, } + if name := os.Getenv("AGENTENV_SESSION_USER"); name != "" { + p.SessionUser = name + } + if os.Getenv("AGENTENV_EGRESS_NSS") == "off" { + p.NSS = nil + } + return p } // boot, as root before the session starts: the CA, minted once per computer, its key the egress user's alone, and diff --git a/src/agent_env/utils/egress_proxy/boot_test.go b/src/agent_env/utils/egress_proxy/boot_test.go index 239d8db0a..c5f59ad0a 100644 --- a/src/agent_env/utils/egress_proxy/boot_test.go +++ b/src/agent_env/utils/egress_proxy/boot_test.go @@ -86,3 +86,14 @@ func TestAnAppsStoreThatCantTakeTheCAIsNamedAndTheBootGoesOn(t *testing.T) { t.Fatalf("%v", report) } } + +func TestAnImageNamesItsSessionUserAndLeavesOutTheNSSDatabase(t *testing.T) { + if p := defaultBootPaths(); p.SessionUser != SessionUser || p.NSS == nil { + t.Fatalf("the base's defaults changed: %q, NSS %v", p.SessionUser, p.NSS != nil) + } + t.Setenv("AGENTENV_SESSION_USER", "agent") + t.Setenv("AGENTENV_EGRESS_NSS", "off") + if p := defaultBootPaths(); p.SessionUser != "agent" || p.NSS != nil { + t.Fatalf("an agent image's settings weren't taken: %q, NSS %v", p.SessionUser, p.NSS != nil) + } +} diff --git a/src/agent_env/utils/egress_proxy/main.go b/src/agent_env/utils/egress_proxy/main.go index dd9760e99..20169dd06 100644 --- a/src/agent_env/utils/egress_proxy/main.go +++ b/src/agent_env/utils/egress_proxy/main.go @@ -63,6 +63,9 @@ const ( PageReads = 1000 ) +// PrivateEnv set to "allow" lets the proxy reach private networks' addresses too (Proxy.PrivateNetworks). +const PrivateEnv = "AGENTENV_EGRESS_PRIVATE" + // The dates the CA and every leaf it signs are valid between: the widest a certificate carries, as routed sites' // (sites.UNIVERSE_FROM and UNIVERSE_UNTIL in agent-env's site routing), so they hold at any date the clock can show. var ( diff --git a/src/agent_env/utils/egress_proxy/proxy.go b/src/agent_env/utils/egress_proxy/proxy.go index 3755399b9..096d8d21b 100644 --- a/src/agent_env/utils/egress_proxy/proxy.go +++ b/src/agent_env/utils/egress_proxy/proxy.go @@ -35,6 +35,10 @@ type Proxy struct { Dial func(dst string) (net.Conn, error) Log func(row map[string]any) probes probes + // PrivateNetworks lets a destination at a private network's address (RFC 1918, RFC 6598's shared space, IPv6 + // unique local) through as well as a global one: an agent's image sets it, whose processes reach those directly + // anyway, so its model endpoint is reached on real time wherever it is. A computer's proxy leaves it off. + PrivateNetworks bool } func serveForever(listen string) error { @@ -56,8 +60,10 @@ func serveForever(listen string) error { if err != nil { return err } - p := &Proxy{Leaves: leaves, Universe: universe, Roots: pool, Log: jsonLines(os.Stdout)} - p.Log(map[string]any{"event": "serving", "listen": listen, "uid": os.Getuid(), "pid": os.Getpid()}) + p := &Proxy{Leaves: leaves, Universe: universe, Roots: pool, Log: jsonLines(os.Stdout), + PrivateNetworks: os.Getenv(PrivateEnv) == "allow"} + p.Log(map[string]any{"event": "serving", "listen": listen, "uid": os.Getuid(), "pid": os.Getpid(), + "private_networks": p.PrivateNetworks}) return p.Serve(ln) } @@ -231,7 +237,8 @@ func headerName(line string) string { // destination is the addresses to dial for host:port, in the order to try them, and why they're allowed: "routed" for // a name the hosts file's agentenv sites block maps (the site proxy, as a client that bypasses this proxy reaches it), -// "global" for a name whose every address is global, IPv4 first. Anything else is refused. +// "global" for a name whose every address is global, "private" for one with an address on a private network where +// PrivateNetworks lets those through, IPv4 first. Anything else is refused. func (p *Proxy) destination(host string, port int) ([]string, string, error) { host = strings.TrimSuffix(strings.ToLower(strings.Trim(host, "[]")), ".") routed := p.Routed @@ -262,11 +269,16 @@ func (p *Proxy) destination(host string, port int) ([]string, string, error) { } var private []string var v4, v6 []string + why := "global" for _, a := range addresses { a = a.Unmap() dst := netip.AddrPortFrom(a, uint16(port)).String() + ok := allowed(a) + if !ok && p.PrivateNetworks && isPrivateNetwork(a) { + ok, why = true, "private" + } switch { - case !allowed(a): + case !ok: private = append(private, a.String()) case a.Is4(): v4 = append(v4, dst) @@ -277,7 +289,7 @@ func (p *Proxy) destination(host string, port int) ([]string, string, error) { if len(private) > 0 { return nil, "", fmt.Errorf("%s:%d isn't public (%s)", host, port, strings.Join(private, ", ")) } - return append(v4, v6...), "global", nil + return append(v4, v6...), why, nil } // notGlobal are the special-purpose blocks (IANA's registries) no global address is in. @@ -295,6 +307,20 @@ var notGlobal = func() []netip.Prefix { return out }() +// privateNetworks are the blocks a network of one's own is numbered from, which PrivateNetworks lets through. +var privateNetworks = []netip.Prefix{netip.MustParsePrefix("10.0.0.0/8"), netip.MustParsePrefix("172.16.0.0/12"), + netip.MustParsePrefix("192.168.0.0/16"), netip.MustParsePrefix("100.64.0.0/10"), netip.MustParsePrefix("fc00::/7")} + +func isPrivateNetwork(a netip.Addr) bool { + a = a.Unmap() + for _, block := range privateNetworks { + if block.Contains(a) { + return true + } + } + return false +} + func isGlobal(a netip.Addr) bool { a = a.Unmap() if !a.IsValid() { diff --git a/src/agent_env/utils/egress_proxy/proxy_test.go b/src/agent_env/utils/egress_proxy/proxy_test.go index c8e57b3e7..772afe1f8 100644 --- a/src/agent_env/utils/egress_proxy/proxy_test.go +++ b/src/agent_env/utils/egress_proxy/proxy_test.go @@ -236,6 +236,41 @@ func TestPlainHTTPInAbsoluteFormIsForwarded(t *testing.T) { } } +func TestAProxyThatAllowsPrivateNetworksReachesThemAndStillRefusesTheRest(t *testing.T) { + p := &Proxy{Routed: func() map[string]string { return nil }, PrivateNetworks: true} + resolving := func(addresses ...string) { + p.Resolve = func(context.Context, string) ([]netip.Addr, error) { + var out []netip.Addr + for _, a := range addresses { + out = append(out, netip.MustParseAddr(a)) + } + return out, nil + } + } + for _, address := range []string{"10.0.0.5", "172.16.0.1", "192.168.1.1", "100.64.0.1", "fd00::5", "::ffff:10.0.0.5"} { + resolving(address) + if dst, why, err := p.destination("model.internal", 4000); err != nil || why != "private" || len(dst) != 1 { + t.Errorf("%s: %q %q %v", address, dst, why, err) + } + } + for _, address := range []string{"127.0.0.1", "::1", "0.0.0.0", "169.254.169.254", "fe80::1", "224.0.0.1", + "192.0.2.1"} { + resolving(address) + if _, _, err := p.destination("model.internal", 4000); err == nil { + t.Errorf("%s dialled", address) + } + } + resolving("1.1.1.1") + if _, why, err := p.destination("one.one.one.one", 443); err != nil || why != "global" { + t.Fatalf("a public name: %q %v", why, err) + } + resolving("1.1.1.1", "10.0.0.5") + dst, why, err := p.destination("split.example", 443) + if err != nil || why != "private" || strings.Join(dst, " ") != "1.1.1.1:443 10.0.0.5:443" { + t.Fatalf("a name with a private address: %q %q %v", dst, why, err) + } +} + func TestARefusedDestinationIsAnswered403(t *testing.T) { s := site(t, false) h := newHarness(t, s, fixedUniverse{}, true) diff --git a/src/agent_env/utils/egress_proxy/upstream.go b/src/agent_env/utils/egress_proxy/upstream.go index ccf32704f..bad2ea7db 100644 --- a/src/agent_env/utils/egress_proxy/upstream.go +++ b/src/agent_env/utils/egress_proxy/upstream.go @@ -182,11 +182,11 @@ func (p *Proxy) probeAt(dst, sni, verifyName string, hello []byte) (probe, bool) var unanswered dialError switch { case errors.As(err, &unanswered): - hit.unreachable, hit.problem = true, truncate(err.Error(), 200) + hit.unreachable, hit.problem = true, truncate(err.Error(), problemChars) case err != nil: - hit.failed, hit.problem = true, truncate(err.Error(), 200) + hit.failed, hit.problem = true, truncate(err.Error(), problemChars) case !found.Verified: - hit.problem = truncate(found.VerifyErr, 200) + hit.problem = truncate(found.VerifyErr, problemChars) default: hit.from, hit.until, hit.verified = found.NotBefore, found.NotAfter, true } @@ -260,6 +260,9 @@ func (p *Proxy) dial(dst string) (net.Conn, error) { return net.DialTimeout("tcp", dst, DialTimeout) } +// problemChars is the most of an upstream's error a probe keeps as its problem. +const problemChars = 200 + func truncate(s string, n int) string { if len(s) > n { return s[:n] diff --git a/src/agent_env/utils/faketime_clock/README.md b/src/agent_env/utils/faketime_clock/README.md index 8dcbb5ece..57d09d2fa 100644 --- a/src/agent_env/utils/faketime_clock/README.md +++ b/src/agent_env/utils/faketime_clock/README.md @@ -5,12 +5,12 @@ server does. Instead each session program loads agent-env's build of [libfaketim (through `/etc/ld.so.preload`), which answers clock reads with universe time from a page a follower process keeps in step with the gateway. -It is part of the ubuntu-24.04 base, which builds the library and the shim from this folder in a stage of its -Dockerfile, since an image builds only from its own folder; the ubuntu-24.04-browser base has a copy, which a unit test -keeps identical. The base starts as root: the root start (`../root/init`) sets the zone, makes the clock page and file -for the follower's user, `agentenv-clock`, and starts root's supervisord (`../root.conf`), which runs the follower -(`clock-follower`) as that user and the session's supervisord as the session user. The session user can read the clock but neither write, replace nor -stop it. +Each image that runs it takes this folder into its build context through its `shared_sources.toml` (the ubuntu-24.04 +and ubuntu-24.04-browser bases, and the Claude Code agent image) and builds the library and the shim for its own libc +in a stage of its Dockerfile. A base starts as root: its root start (`computers/artifacts/ubuntu-24.04/root/init`) sets +the zone, makes the clock page and file for the follower's user, `agentenv-clock`, and starts root's supervisord +(`root.conf` beside it), which runs the follower (`clock-follower`) as that user and the session's supervisord as the +session user. The session user can read the clock but neither write, replace nor stop it. The follower long-polls the gateway's `/clock/watch`, writes each state it answers to the page and acknowledges it with its next watch. Core writes its config, the gateway's watch URL as the computer reaches it, the computer's name and its @@ -35,6 +35,8 @@ What follows the universe, and what doesn't: their own (Chromium's BoringSSL, GnuTLS, NSS, Java's) and static binaries aren't covered. - **Who:** only the session user's processes (`AGENTENV_SESSION_UID`). A program started with `AGENTENV_REAL_TIME=1` stays real; the marker is taken out of its environment block, so its children are faked. +- **Init:** a clock read made while the library initializes (an allocator `dlsym` calls, say) reads the kernel's + clock rather than wait on the initialization it is part of; Claude Code's CLI never started without it. - **Settings:** read once as the library loads, from `/etc/agentenv/clock.conf` (`clock.conf` is the default), or from the environment where there is no such file. While the file exists the environment changes none of them; the marker above is the one switch a program's environment has. No `getenv()` runs once the program has started: a `getenv()` diff --git a/src/agent_env/utils/faketime_clock/clock-follower b/src/agent_env/utils/faketime_clock/clock-follower index 652431687..fee4586b7 100755 --- a/src/agent_env/utils/faketime_clock/clock-follower +++ b/src/agent_env/utils/faketime_clock/clock-follower @@ -141,6 +141,8 @@ class Follower: return # its watch is left to end on its own; nothing it answers is applied if error is not None: self.failed(error) + if read_config()[0] != self.raw: + return # a gateway that refuses at once answers every watch before the poll above sees a change continue if read_config()[0] != self.raw: return diff --git a/src/agent_env/utils/faketime_clock/libfaketime.patch b/src/agent_env/utils/faketime_clock/libfaketime.patch index d26148eed..17555d2b8 100644 --- a/src/agent_env/utils/faketime_clock/libfaketime.patch +++ b/src/agent_env/utils/faketime_clock/libfaketime.patch @@ -1525,3 +1525,41 @@ diff -ruN a/src/libfaketime.map b/src/libfaketime.map + pthread_cond_timedwait; } GLIBC_2.2.5; +diff -ruN a/src/libfaketime.c b/src/libfaketime.c +--- a/src/libfaketime.c ++++ b/src/libfaketime.c +@@ -329,6 +329,9 @@ + check_missing_real(#name, (NULL == real_##name)) + + static pthread_once_t initialized_once_control = PTHREAD_ONCE_INIT; ++/* Set on the thread running ftpl_really_init: a time call it makes, as an allocator that reads the clock does when ++ * dlsym allocates, takes the kernel's clock instead of waiting on the init it is part of. */ ++static __thread int ftpl_initializing = 0; + + /* prototypes */ + static int fake_gettimeofday(struct timeval *tv); +@@ -2326,6 +2329,7 @@ + int result; + + ftpl_init(); ++ if (ftpl_initializing) return syscall(SYS_clock_gettime, clk_id, tp); + // If ftpl_init ends up recursing, pthread_once will deadlock. + // (Previously we attempted to detect this situation, and bomb out, + // but the approach taken wasn't thread-safe and broke in practice.) +@@ -3103,8 +3107,15 @@ + dont_fake = dont_fake_final; + } + ++static void ftpl_really_init_guarded(void) { ++ ftpl_initializing = 1; ++ ftpl_really_init(); ++ ftpl_initializing = 0; ++} ++ + inline static void ftpl_init(void) { +- pthread_once(&initialized_once_control, ftpl_really_init); ++ if (ftpl_initializing) return; ++ pthread_once(&initialized_once_control, ftpl_really_init_guarded); + } + + void *ft_dlvsym(void *handle, const char *symbol, const char *version, diff --git a/tst/integration/task_step/test_agent_env_clock_local.py b/tst/integration/task_step/test_agent_env_clock_local.py new file mode 100644 index 000000000..139c81b89 --- /dev/null +++ b/tst/integration/task_step/test_agent_env_clock_local.py @@ -0,0 +1,209 @@ +"""An agent image on a task env's clock (FD-5916), end to end on local Docker: the agent deployed with the env serves +clock/v1 sync_time; sync_env_clock declares it a follower at the real gateway and hands it its config; from then on +every program the agent's user runs, its CLI included, reads the env's time while the A2A server stays on real time; a +clock write that waits for followers waits for it; and once teardown_sandboxes takes it down, writes stop waiting for +it. An agent whose CLI speaks a protocol the stub endpoint serves also runs one real turn. + +Runs over images built beforehand, from this branch: AGENTENV_E2E_AGENT_IMAGES names one per agent, as +``claude-code=,codex=``; an agent it doesn't name is skipped as a missing capability.""" + +import asyncio +import json +import re +import subprocess +import time +import uuid +from collections.abc import Awaitable, Callable +from dataclasses import dataclass +from datetime import datetime, timezone + +import pytest +from agentenv_protocol.client import ExtensionCallError + +from agent_env.a2a_agent.a2a_agent import A2AAgent +from agent_env.a2a_agent.clock import FOLLOWING_KEY +from agent_env.artifact import DockerImageArtifact +from agent_env.env.gateway.constants import EXT_CLOCK_URI +from agent_env.providers.sandbox_providers.local_docker.provider import ( + _host_ips, + host_url_for, +) +from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.task_steps.deploy_agent import DeployAgentTaskStep +from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep +from agent_env.task_step.task_steps.sync_env_clock import SyncEnvClockTaskStep +from agent_env.task_step.task_steps.teardown_sandboxes import TeardownSandboxesTaskStep +from tst.integration.store.local_parallel_deploy_test import _seed_bootstrap_envs +from tst.integration.task_step.test_prompt_agent_live_trajectory_local import ( + local_registry as local_registry, +) +from tst.integration.task_step.test_register_env_triggers_local import _deploy_items +from tst.util.capabilities import agent_clock_image, skip_without_agent_clock_image +from tst.util.fake_anthropic import FakeAnthropic, tool_then_answer + +pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow] + +VIRTUAL = "2031-03-04T05:06:07Z" +LATER = "2031-03-05T10:00:00Z" +PAUSED = "2031-03-05T12:00:00Z" + + +def _in_agent(container: str, *command: str, real_time: bool = False) -> str: + env = ["-e", "AGENTENV_REAL_TIME=1"] if real_time else [] + return subprocess.run(["docker", "exec", "-u", "agent", *env, container, *command], capture_output=True, + text=True, check=True, timeout=30).stdout.strip() + + +def _routes_through_proxy(container: str, bridge: str) -> bool: + """Whether the CLI the agent starts now goes out through the egress proxy, as the image's bridge decides: every + bridge has the same ``clock_armed`` and ``egress_env``.""" + # The A2A server's environment has the proxy (root.conf); this shell's doesn't, so it's passed as the server has it. + return _in_agent(container, "python3", "-c", f"import {bridge} as b; print('HTTPS_PROXY' in b.egress_env(" + "{b.EGRESS_PROXY_ENV: 'http://127.0.0.1:3128'}, armed=b.clock_armed()))", + real_time=True) == "True" + + +async def _claude_turn(context: TaskStepContext, endpoint: FakeAnthropic) -> None: + """One turn of the real Claude Code CLI: the date it gives the model and a Bash `date` the model runs are the env's.""" + endpoint.respond = tool_then_answer("Bash", {"command": "date -u +%FT%TZ"}, prefix="the shell said: ") + await PromptAgentTaskStep(id="ask", version=None, prompt="What's the date? Run date.", agent_name="solver", + max_conversation_turns=1).execute(context) + bodies = [json.dumps(call.body) for call in endpoint.messages()] + dated = {m for body in bodies for m in re.findall(r"Today's date is ([0-9]{4}-[0-9]{2}-[0-9]{2})", body)} + print("the CLI's date line:", dated, "Bash date:", endpoint.tool_results()) + assert dated == {VIRTUAL[:10]}, dated + assert any(VIRTUAL in result for result in endpoint.tool_results()), endpoint.tool_results() + + +@dataclass(frozen=True) +class Agent: + """What the test needs of an agent's image: its bridge module, a command its CLI answers without a model (which + must return under the preloaded library), and a turn against the stub endpoint, for a CLI that speaks its protocol.""" + + bridge: str + version: tuple[str, ...] + shows: str + turn: Callable[[TaskStepContext, FakeAnthropic], Awaitable[None]] | None = None + + +AGENTS = { + "claude-code": Agent(bridge="claude_bridge", version=("claude", "--version"), shows="(Claude Code)", + turn=_claude_turn), +} + + +def _container_of(sandbox_id: str, *, required: bool = True) -> str | None: + """The container behind ``sandbox_id``, stopped or running; None, when not ``required``, for one that's gone.""" + names = subprocess.run(["docker", "ps", "-a", "--format", "{{.Names}}"], capture_output=True, text=True, + check=True).stdout.split() + found = [n for n in names if sandbox_id in n] + if not required and len(found) != 1: + return None + [name] = found + return name + + +async def _remove_agents(context: TaskStepContext) -> None: + """Remove the agent containers teardown didn't take down; one already gone, or failing to go, is left.""" + gone = set(context.metadata.get("torn_down_sandbox_ids") or []) + for agent in context.deployed_agents: + name = agent.sandbox_id and agent.sandbox_id not in gone and _container_of(agent.sandbox_id, required=False) + if name: + await asyncio.to_thread(subprocess.run, ["docker", "rm", "-f", name], capture_output=True) + + +async def _followers(deployed) -> dict: + return (await deployed.invoke(EXT_CLOCK_URI, "state", {}, timeout=10)).get("followers") or {} + + +@pytest.mark.parametrize("name", [pytest.param(name, marks=skip_without_agent_clock_image(name)) for name in AGENTS]) +@pytest.mark.asyncio +async def test_an_agent_image_follows_the_envs_clock_until_it_is_torn_down(local_registry, name): + with FakeAnthropic(host=_host_ips()[-1]) as endpoint: # where this machine's containers reach it + await _follows_until_torn_down(name, AGENTS[name], agent_clock_image(name), endpoint) + + +async def _follows_until_torn_down(name: str, spec: Agent, tag: str, endpoint: FakeAnthropic) -> None: + _seed_bootstrap_envs() + env, deployed, _ = await _deploy_items("agentclock") + uid = uuid.uuid4().hex[:8] + image = DockerImageArtifact.put(id=f"{name}-clock-img-{uid}", description="e2e", image_name=tag) + agent = A2AAgent.put(id=f"{name}-clock-{uid}", docker_image_artifact=image, + default_env_vars={"LITELLM_API_KEY": "fake-key", + # as the agent's container reaches it: on macOS, loopback is the host's + "LITELLM_BASE_URL": host_url_for(endpoint.url, "local")}) + context = TaskStepContext(deployed_envs=[deployed], deployed_agents=[], metadata={}) + try: + await DeployAgentTaskStep(id="deploy-solver", version=None, env_ids=[env.id], a2a_agent_id=agent.id, + a2a_agent_version=agent.version, agent_name="solver", + sandbox_type="local").execute(context) + container = _container_of(context.deployed_agents[-1].sandbox_id) + assert _in_agent(container, "date", "-u", "+%Y") == str(datetime.now(timezone.utc).year) # not armed: real + assert not _routes_through_proxy(container, spec.bridge) # and the CLI goes out directly, as before + # The CLI starts under the preloaded library: without its init guard, Claude Code's first clock read deadlocked. + assert spec.shows in _in_agent(container, "timeout", "30", *spec.version) + + await SyncEnvClockTaskStep(id="clock", version=None, env_id=env.id, virtual_time=VIRTUAL, + virtual_seconds_per_real_second=0, require_agent_clock=True, + timeout_seconds=60).execute(context) + + [outcome] = context.metadata["clock_configurations"][-1]["agents"] + print("sync outcome:", {k: v for k, v in outcome.items() if k != "result"}, outcome.get("result")) + assert (outcome["agent"], outcome["synced"], outcome["follower"]) == ("solver", True, "agent-solver") + assert context.metadata[FOLLOWING_KEY]["solver"]["env_id"] == env.id + assert "agent-solver" in await _followers(deployed) + + # Every program the agent's user runs reads the env's time, frozen at rate 0; the marker opts out. + shown = {"date": _in_agent(container, "date", "-u", "+%FT%TZ"), + "python": _in_agent(container, "python3", "-c", "import datetime; print(datetime.datetime.now(" + "datetime.timezone.utc).isoformat())"), + "real": _in_agent(container, "date", "-u", "+%FT%TZ", real_time=True)} + print("agent reads:", shown) + assert shown["date"] == VIRTUAL and shown["python"].startswith("2031-03-04T05:06:07") + assert shown["real"].startswith(str(datetime.now(timezone.utc).year)) + assert _routes_through_proxy(container, spec.bridge) # armed: live sites' TLS goes through the proxy + assert spec.shows in _in_agent(container, "timeout", "30", *spec.version) # the CLI starts on the env's clock + if spec.turn is not None: + await spec.turn(context, endpoint) + + # A write that waits for followers waits for the agent, which then shows it at once. + started = time.monotonic() + await deployed.invoke(EXT_CLOCK_URI, "advance", {"virtual_time": LATER, "wait_followers_s": 20}, timeout=30) + print(f"advance acknowledged by the agent in {time.monotonic() - started:.2f}s") + assert _in_agent(container, "date", "-u", "+%FT%TZ") == LATER + assert (await _followers(deployed))["agent-solver"]["version"] is not None + + # It is really waited for: frozen, the agent can't acknowledge, and the write answers 504 naming it; thawed, + # its follower catches up on its own. + subprocess.run(["docker", "pause", container], check=True, capture_output=True) + try: + started = time.monotonic() + with pytest.raises(ExtensionCallError) as frozen: + await deployed.invoke(EXT_CLOCK_URI, "advance", {"virtual_time": PAUSED, "wait_followers_s": 3}, + timeout=30) + print(f"advance while the agent was frozen: {time.monotonic() - started:.2f}s, " + f"{frozen.value.response.status_code} {frozen.value.response.text[:200]}") + finally: + subprocess.run(["docker", "unpause", container], check=True, capture_output=True) + assert frozen.value.response.status_code == 504 and frozen.value.response.json()["stale"] == ["agent-solver"] + deadline = time.monotonic() + 30 + while _in_agent(container, "date", "-u", "+%FT%TZ") != PAUSED: + assert time.monotonic() < deadline, "the agent's follower didn't catch up after it was thawed" + await asyncio.sleep(0.5) + + await TeardownSandboxesTaskStep(id="down", version=None, agent_names=["solver"]).execute(context) + assert "agent-solver" not in await _followers(deployed) + assert "solver" not in (context.metadata.get(FOLLOWING_KEY) or {}) + + # Torn down, the agent is no longer waited for: a write answers at once, not after 20 s or with a 504. + started = time.monotonic() + await deployed.invoke(EXT_CLOCK_URI, "advance", {"virtual_time": "2031-03-06T00:00:00Z", + "wait_followers_s": 20}, timeout=30) + waited = time.monotonic() - started + print(f"advance after teardown answered in {waited:.2f}s") + assert waited < 5 + finally: + try: + await _remove_agents(context) + finally: + await env.close() diff --git a/tst/unit/agents/claude_code/test_env_clock_image.py b/tst/unit/agents/claude_code/test_env_clock_image.py new file mode 100644 index 000000000..1f5432a96 --- /dev/null +++ b/tst/unit/agents/claude_code/test_env_clock_image.py @@ -0,0 +1,102 @@ +"""Claude Code's bridge on the env's clock (FD-5916): while the clock page is armed, the CLI goes out through the +egress proxy, and to the env's MCP servers, the hosts NO_PROXY names and a plain-HTTP model endpoint directly; unarmed, +it goes out as before. What every clocked image shares is checked in tst/unit/agents/test_agent_clock_images.py.""" + +from __future__ import annotations + +import json +import types +from urllib.parse import urlsplit + +import pytest + +from tst.unit.agents.claude_code.agent import AGENT_DIR, claude_bridge +from tst.util.build_context import sent + +PROXY = "http://127.0.0.1:3128" + + +def test_on_an_armed_clock_the_cli_goes_out_through_the_proxy_and_to_the_envs_servers_directly(): + env = claude_bridge.cli_env({claude_bridge.EGRESS_PROXY_ENV: PROXY}, direct_hosts=["gateway", "", "gateway"], + clock_armed=True) + + assert {env[name] for name in ("HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "https_proxy")} == {PROXY} + assert env["NO_PROXY"] == env["no_proxy"] == "localhost,127.0.0.1,::1,host.docker.internal,gateway" + assert env["NODE_USE_ENV_PROXY"] == "1" + + +def test_hosts_the_agent_was_deployed_to_reach_directly_stay_direct_on_an_armed_clock(): + # A private registry the agent's NO_PROXY names worked before the clock was armed; the proxy would refuse it. + env = claude_bridge.cli_env({claude_bridge.EGRESS_PROXY_ENV: PROXY, "NO_PROXY": "registry.internal, .corp", + "no_proxy": "mirror.internal"}, direct_hosts=["gateway"], clock_armed=True) + + hosts = env["NO_PROXY"].split(",") + assert env["no_proxy"] == env["NO_PROXY"] + assert {"registry.internal", ".corp", "mirror.internal", "gateway", "localhost"} <= set(hosts) + assert len(hosts) == len(set(hosts)) and "" not in hosts + + +@pytest.mark.parametrize("base_url, direct", [ + ("http://10.0.0.5:4000", True), # plain HTTP: nothing to check at the env's date + # TLS, at a private address or a public one: its certificate is checked through the proxy, on real time, and the + # agent's proxy reaches private networks (root.conf's AGENTENV_EGRESS_PRIVATE) + ("https://10.0.0.5:4000", False), + ("https://[fd00::5]:4000", False), + ("https://litellm.internal:4000", False), + ("https://litellm.example.com", False), +]) +def test_a_model_endpoint_over_plain_http_is_reached_directly_and_one_over_tls_through_the_proxy_on_an_armed_clock( + base_url, direct): + env = claude_bridge.cli_env({claude_bridge.EGRESS_PROXY_ENV: PROXY, "LITELLM_BASE_URL": base_url, + "LITELLM_API_KEY": "k"}, clock_armed=True) + assert (urlsplit(base_url).hostname in env["NO_PROXY"].split(",")) is direct and env["HTTPS_PROXY"] == PROXY + + +@pytest.mark.parametrize("environ, armed", [({}, True), ({claude_bridge.EGRESS_PROXY_ENV: PROXY}, False)]) +def test_without_the_proxy_or_an_armed_clock_the_cli_goes_out_directly_as_before(environ, armed): + # A model endpoint at a private address, which the proxy refuses, keeps working until a task arms the clock. + assert not {"HTTPS_PROXY", "NO_PROXY"} & set(claude_bridge.cli_env(environ, clock_armed=armed)) + + +def _clockpage(): + """The image's clock page writer, whose encoding libfaketime and the egress proxy read too.""" + # Run from its source, so no __pycache__ lands in agent-env's sources. + module = types.ModuleType("clockpage") + path = sent(AGENT_DIR, "clock/clockpage.py") + exec(compile(path.read_text(), str(path), "exec"), module.__dict__) + return module + + +def _page(armed: int, seq: int = 2) -> bytes: + return _clockpage().encode(seq, 1_900_000_000 * 10**9 if armed else 0, 10**18 if armed else 0, 1.0, armed, 3, 0, 0) + + +@pytest.mark.parametrize("page, armed", [ + (_page(1), True), + (_page(0), False), + # A write under way (seq odd) that never finishes: the armed flag it left is the best guess. + (_page(1, seq=3), True), + (_page(1)[:40], False), # not a page + (b"\0" * 80, False), # not one yet: no magic + (None, False), # no page: an image without the clock, or install_agent's target +]) +def test_the_clock_is_armed_when_the_page_libfaketime_reads_says_so(tmp_path, page, armed): + path = tmp_path / "agentenv-clock" + if page is not None: + path.write_bytes(page) + assert claude_bridge.clock_armed(str(path)) is armed + + +@pytest.mark.parametrize("status", [ + {"connected": False, "error": "URLError: gateway restarting"}, # the follower lost its gateway + {"connected": False, "waiting": "no config yet"}, # the follower restarted, before its config is read + {"connected": True, "armed": False, "version": None}, # ... and before its first watch answers +]) +def test_while_the_follower_reconnects_or_restarts_the_page_still_routes_the_cli_through_the_proxy(tmp_path, status): + # The page keeps the last armed clock in both windows, so the CLI reads the env's time and must go through egress. + (tmp_path / "status.json").write_text(json.dumps(status)) + page = tmp_path / "agentenv-clock" + page.write_bytes(_page(1)) + env = claude_bridge.cli_env({claude_bridge.EGRESS_PROXY_ENV: PROXY}, clock_armed=claude_bridge.clock_armed(str(page))) + assert env["HTTPS_PROXY"] == PROXY + diff --git a/tst/unit/agents/claude_code/test_image_contract.py b/tst/unit/agents/claude_code/test_image_contract.py index c8e528df3..1bd602caf 100644 --- a/tst/unit/agents/claude_code/test_image_contract.py +++ b/tst/unit/agents/claude_code/test_image_contract.py @@ -1,23 +1,35 @@ -"""The agent's folder is its build context: the Dockerfile copies only its own files, keeps TLS checks on, and -installs the protocol at or above its floor, which a release doesn't pin.""" +"""The agent's folder is its build context: the Dockerfile copies only its own files and the folders its +shared_sources.toml takes from agent-env's sources, keeps TLS checks on, and installs the protocol at or above its +floor, which a release doesn't pin.""" from __future__ import annotations import re +import tomllib from tst.unit.agents.claude_code.agent import AGENT_DIR DOCKERFILE = (AGENT_DIR / "Dockerfile").read_text() +SOURCES = AGENT_DIR.parents[2] +SHARED = tomllib.loads((AGENT_DIR / "shared_sources.toml").read_text()) # Harbor bundles pin the same versions (agentenvhub-sdk's _DOC_TOOL_PACKAGES), so a task's documents read alike in both. DOC_LIBS = ["openpyxl==3.1.5", "pypdf==6.21.3", "python-docx==1.2.0", "python-pptx==1.0.2"] +def _in_context(name: str) -> bool: + """Whether the build context has ``name``: from the folder, or under a folder shared_sources.toml takes from + agent-env's own sources at that name.""" + first, _, rest = name.partition("/") + path = SOURCES / SHARED[first] / rest if first in SHARED else AGENT_DIR / name + return path.is_dir() if name.endswith("/") else path.is_file() + + def test_every_file_the_image_copies_is_in_the_folder(): copied = [name for line in re.findall(r"^COPY (?!--from=)(.+) \S+$", DOCKERFILE, re.MULTILINE) for name in line.split()] - assert copied and all((AGENT_DIR / name).is_file() for name in copied), copied + assert copied and all(_in_context(name) for name in copied), copied # install.sh and run.sh are install/v1's; a build context pruned to what the Dockerfile reads keeps them so. assert {"a2a_server.py", "claude_bridge.py", "kit.py", "tar_changelog.py", "entrypoint.sh", "install.sh", "run.sh"} <= set(copied) @@ -26,7 +38,7 @@ def test_the_image_installs_the_protocol_alone_from_its_requirements(): requirements = [line for line in (AGENT_DIR / "requirements.txt").read_text().splitlines() if line and not line.startswith("#")] - assert requirements == ["agentenv-framework-protocol[agent]>=0.1.351"] + assert requirements == ["agentenv-framework-protocol[agent]>=0.1.575"] def test_the_image_gives_the_model_node_alone_from_a_pinned_image(): @@ -54,5 +66,5 @@ def test_the_image_pins_its_cli_checks_its_checksum_and_keeps_tls_on(): def test_the_image_starts_through_cmd_which_modal_passes_its_env_to(): - assert 'CMD ["/opt/agent/entrypoint.sh"]' in DOCKERFILE + assert 'CMD ["/usr/local/bin/start-agent"]' in DOCKERFILE assert not re.search(r"^ENTRYPOINT", DOCKERFILE, re.MULTILINE) diff --git a/tst/unit/agents/test_agent_clock_images.py b/tst/unit/agents/test_agent_clock_images.py new file mode 100644 index 000000000..055ed3683 --- /dev/null +++ b/tst/unit/agents/test_agent_clock_images.py @@ -0,0 +1,102 @@ +"""Every agent image on the env's clock (FD-5916), whichever agent it is: each folder whose ``shared_sources.toml`` +takes the clock takes the computer bases' clock library and egress proxy from agent-env's one copy, whose libfaketime +patch has the init guard; runs the follower and the proxy, each as its own user, and the A2A server as the agent's user +on real time, under root's supervisord; preloads agent-env's libfaketime for the agent's user alone; starts as another +user as it did before; and its bridge decides the CLI's proxy with the same ``clock_armed`` and ``egress_env`` as every +other agent's.""" + +from __future__ import annotations + +import ast +import re +import tomllib +from pathlib import Path + +import pytest + +AGENTS_DIR = Path(__file__).parents[3] / "src" / "agent_env" / "agents" / "agents" +SOURCES = AGENTS_DIR.parents[1] +CLOCKED = sorted(p.parent.name for p in AGENTS_DIR.glob("*/shared_sources.toml") + if "clock" in tomllib.loads(p.read_text())) +PROXY = "http://127.0.0.1:3128" + + +def test_claude_code_follows_the_clock(): + assert "claude-code" in CLOCKED + + +@pytest.mark.parametrize("agent", CLOCKED) +@pytest.mark.parametrize(("part", "shared"), [("clock", "utils/faketime_clock"), ("egress", "utils/egress_proxy")]) +def test_the_clock_pieces_are_taken_from_the_one_copy_the_computer_bases_take(agent, part, shared): + taken = tomllib.loads((AGENTS_DIR / agent / "shared_sources.toml").read_text()) + + assert taken[part] == shared + assert not (AGENTS_DIR / agent / part).exists(), f"{agent}/{part} is a copy: take it in shared_sources.toml instead" + + +def test_the_libfaketime_every_clocked_image_builds_has_the_init_guard(): + # A CLI that reads the clock while libfaketime's init runs (an allocator, as dlsym allocates) has pthread_once wait + # on itself: Claude Code's `--version` never returned. The guard has a clock call made during the init read the + # kernel's clock. + patch = (SOURCES / "utils" / "faketime_clock" / "libfaketime.patch").read_text() + + assert "static __thread int ftpl_initializing = 0;" in patch + assert "if (ftpl_initializing) return syscall(SYS_clock_gettime, clk_id, tp);" in patch + assert "pthread_once(&initialized_once_control, ftpl_really_init_guarded);" in patch + + +def _program(agent: str, name: str) -> dict[str, str]: + conf = (AGENTS_DIR / agent / "root.conf").read_text() + section = re.search(rf"^\[program:{name}\]\n(.*?)(?=^\[|\Z)", conf, re.MULTILINE | re.DOTALL).group(1) + return dict(line.split("=", 1) for line in section.splitlines() if "=" in line and not line.startswith(";")) + + +@pytest.mark.parametrize("agent", CLOCKED) +def test_the_follower_and_the_proxy_run_as_their_own_users_and_the_a2a_server_as_the_agents_on_real_time(agent): + assert (_program(agent, "clock")["user"], _program(agent, "egress")["user"]) == ("agentenv-clock", "agentenv-egress") + server = _program(agent, "agent") + assert server["user"] == "agent" + # The server's own interpreter, so the marker reaches it, not a shell that drops it on exec. + assert server["command"].split()[0].rsplit("/", 1)[-1].startswith("python") + assert 'AGENTENV_REAL_TIME="1"' in server["environment"] + assert f'AGENTENV_EGRESS_PROXY="{PROXY}"' in server["environment"] + + +@pytest.mark.parametrize("agent", CLOCKED) +def test_the_agents_proxy_reaches_private_networks_where_its_model_endpoint_may_be_and_a_computers_doesnt(agent): + # The agent's processes reach those directly anyway; its model endpoint, over TLS, goes through the proxy. + assert 'AGENTENV_EGRESS_PRIVATE="allow"' in _program(agent, "egress")["environment"].split(",") + base = (SOURCES / "computers" / "artifacts" / "ubuntu-24.04" / "root.conf").read_text() + assert "AGENTENV_EGRESS_PRIVATE" not in base + + +@pytest.mark.parametrize("agent", CLOCKED) +def test_the_image_preloads_the_library_for_the_agents_user_and_the_follower_alone_reads_its_config(agent): + dockerfile = (AGENTS_DIR / agent / "Dockerfile").read_text() + assert "> /etc/ld.so.preload" in dockerfile and "libfaketimeMT.so.1" in dockerfile + assert 's/^AGENTENV_SESSION_UID=.*/AGENTENV_SESSION_UID=$(id -u agent)/' in dockerfile + assert "install -d -o agent -g agentenv-clock -m 2750 /etc/agentenv/clock" in dockerfile + assert 'CMD ["/usr/local/bin/start-agent"]' in dockerfile + assert not re.search(r"^USER ", dockerfile, re.MULTILINE) + + +@pytest.mark.parametrize("agent", CLOCKED) +def test_started_as_another_user_the_image_runs_the_server_alone_as_before(agent): + start = (AGENTS_DIR / agent / "start-agent").read_text() + assert 'if [ "$(id -u)" != 0 ]; then\n exec ' in start + assert "AGENTENV_SESSION_USER=agent AGENTENV_EGRESS_NSS=off" in start + + +def _clock_helpers(agent: str) -> dict[str, str]: + """The bridge's clock helpers, as source: the one module in the agent's folder that defines ``egress_env``.""" + [module] = [p for p in (AGENTS_DIR / agent).glob("*.py") if "def egress_env(" in p.read_text()] + tree = ast.parse(module.read_text()) + names = {"clock_armed", "model_direct_hosts", "egress_env"} + found = {node.name: ast.unparse(node) for node in tree.body if isinstance(node, ast.FunctionDef) and node.name in names} + assert set(found) == names, (module, sorted(found)) + return found + + +@pytest.mark.parametrize("agent", CLOCKED) +def test_every_bridge_decides_the_clis_proxy_as_claude_codes_does(agent): + assert _clock_helpers(agent) == _clock_helpers("claude-code") diff --git a/tst/unit/env/envs/test_computer_clock_follower.py b/tst/unit/env/envs/test_computer_clock_follower.py index 9415e2a8b..a0ce82c4c 100644 --- a/tst/unit/env/envs/test_computer_clock_follower.py +++ b/tst/unit/env/envs/test_computer_clock_follower.py @@ -181,6 +181,19 @@ def test_a_gateway_it_cant_reach_is_named_in_its_status_and_it_keeps_trying(foll follower.until(lambda: follower.status().get("errors", 0) >= 2) +def test_a_changed_config_is_followed_while_the_gateway_it_follows_refuses_at_once(follower): + gateway = Gateway() + try: + follower.configure({"watch_url": "http://127.0.0.1:9/clock/watch", "name": "desktop", "token": "t"}) + follower.until(lambda: follower.status().get("errors", 0) >= 3) + + gateway.set("boot:1", datetime(2020, 1, 1, tzinfo=timezone.utc), 1.0) + follower.configure({"watch_url": gateway.url, "name": "desktop", "token": "t"}) + follower.until(lambda: follower.status().get("version") == "boot:1", within=3) + finally: + gateway.close() + + def test_a_config_that_isnt_one_is_named_and_a_good_one_then_followed(follower): gateway = Gateway() try: diff --git a/tst/util/capabilities.py b/tst/util/capabilities.py index fcbdf763b..3c315d9b9 100644 --- a/tst/util/capabilities.py +++ b/tst/util/capabilities.py @@ -78,6 +78,10 @@ #: Composing an Omarchy computer builds its base from Omarchy's own VM image, which a run fetches into this machine's #: cache once, about 4 GB; a test that composes one needs it there already, and KVM, which boots linux/amd64 disks. OMARCHY_IMAGE = "omarchy_image" +#: The agent clock end to end (FD-5916) runs over agent images built from the branch beforehand, which +#: ``AGENTENV_E2E_AGENT_IMAGES`` names one per agent (``claude-code=,codex=``): no CI job builds agent images. +AGENT_CLOCK_IMAGE = "agent_clock_image" +AGENT_CLOCK_IMAGES_ENV = "AGENTENV_E2E_AGENT_IMAGES" #: The shipped Omarchy base, whose source the capability looks for. OMARCHY_ARTIFACT = Path(__file__).parents[2] / "src" / "agent_env" / "computers" / "artifacts" / "omarchy-4.5" / "artifact.toml" @@ -241,6 +245,17 @@ def omarchy_image_is_available() -> bool: return "linux/amd64" in LocalSandboxProvider.disk_image_platforms() and source_path(sha256).exists() +def agent_clock_image(agent: str) -> str: + """The image ``AGENTENV_E2E_AGENT_IMAGES`` names for ``agent``, or ''.""" + named = dict(entry.split("=", 1) for entry in os.environ.get(AGENT_CLOCK_IMAGES_ENV, "").split(",") if "=" in entry) + return named.get(agent, "").strip() + + +def skip_without_agent_clock_image(agent: str) -> pytest.MarkDecorator: + """Collection-time ``skipif`` for a test that runs over ``agent``'s image built with the agent clock.""" + return pytest.mark.skipif(not agent_clock_image(agent), reason=missing_capability_reason(AGENT_CLOCK_IMAGE)) + + def skip_without_omarchy_image() -> pytest.MarkDecorator: """Collection-time ``skipif`` for tests that compose an Omarchy computer on local Docker.""" return pytest.mark.skipif(not omarchy_image_is_available(), reason=missing_capability_reason(OMARCHY_IMAGE))