diff --git a/README.md b/README.md index 8932bee01..2bde04d3e 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,16 @@ A package of your own can add env types, providers, task steps and stores to age Every task in a bundle is an [OpenEnv](https://github.com/openenv/OpenEnv) env, with the `openenv` extra installed (`pip install 'agentenv-framework[openenv]'`). `agent_env.openenv.build_app(bundle)` returns the OpenEnv server for a bundle, a folder or an installed bundle's name, or for a mapping of split names to bundles. Any OpenEnv client resets it to a task, which deploys the task's envs and opens its agent's turn, acts through the envs' tools and an added `end_turn` tool, and gets the task's verifier score as its reward. +An agent a task installs into a container (`install_agent` with `container_name`) is played through AgentEnv's proxy, which an episode installs there in its place. A task image can carry the proxy, so an episode only starts it: copy the installed `agent_env` package's `agents/agents/proxy` folder into the image's build context as `agentenv-proxy`, and end the Dockerfile with + +```dockerfile +COPY --from=ghcr.io/astral-sh/uv:0.13.1@sha256:4a1f3666816dafadd20ceb71eac9a7d56d7ef30279f7d64048519ab901ada336 /uv /opt/agentenv-proxy/bin/uv +COPY agentenv-proxy /opt/agentenv-proxy/src +RUN sh /opt/agentenv-proxy/src/bake.sh +``` + +A container whose baked proxy holds other requirements than this agent-env's proxy gets the proxy installed afresh. Name the image as `run_docker_container`'s `image_artifact_id`, so it's built once, not every episode. + ## Time zones Servers and the env clock work in UTC. To run an agent in a user's time zone, the task sets it on the agent's `deploy_agent` step as `env_vars: {"TZ": "America/New_York"}`, which reaches the agent's sandbox, and names the zone in the agent's prompt. While the env clock is armed, the gateway's `get_time` tool takes that IANA name as `time_zone` and adds the `local_time` in that zone, and a trigger executor's working `hours` can take a `time_zone` instead of a fixed `utc_offset`, so they keep their local hours across daylight-saving changes. diff --git a/src/agent_env/agents/agents/proxy/bake.sh b/src/agent_env/agents/agents/proxy/bake.sh new file mode 100644 index 000000000..5d9b86479 --- /dev/null +++ b/src/agent_env/agents/agents/proxy/bake.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# Bakes the proxy into an image as it's built, so an install into one of its containers only starts it: a private +# Python and a venv holding requirements.txt in /opt/agentenv-proxy, and a marker naming the requirements it holds, +# which install.sh checks. Plain sh, for images with no bash. A task image's Dockerfile ends with +# +# COPY --from=ghcr.io/astral-sh/uv:0.13.1@sha256:4a1f3666816dafadd20ceb71eac9a7d56d7ef30279f7d64048519ab901ada336 /uv /opt/agentenv-proxy/bin/uv +# COPY agentenv-proxy /opt/agentenv-proxy/src +# RUN sh /opt/agentenv-proxy/src/bake.sh +# +# where agentenv-proxy is this folder, copied into the image's build context. +set -eu + +here=$(cd "$(dirname "$0")" && pwd) +. "$here/versions.sh" + +export UV_PYTHON_PREFERENCE=only-managed UV_PYTHON_INSTALL_DIR="$BAKED/python" UV_CACHE_DIR="$BAKED/cache" +"$BAKED/bin/uv" venv --quiet --python "$PYTHON_VERSION" "$BAKED/venv" +"$BAKED/bin/uv" pip install --quiet --python "$BAKED/venv/bin/python" -r "$here/requirements.txt" +rm -rf "$BAKED/cache" +"$BAKED/venv/bin/python" -c 'import hashlib, sys; print(hashlib.sha256(open(sys.argv[1], "rb").read()).hexdigest())' \ + "$here/requirements.txt" > "$BAKED/baked" diff --git a/src/agent_env/agents/agents/proxy/install.sh b/src/agent_env/agents/agents/proxy/install.sh index 17e701c3d..1bff9b21f 100644 --- a/src/agent_env/agents/agents/proxy/install.sh +++ b/src/agent_env/agents/agents/proxy/install.sh @@ -5,16 +5,14 @@ # install.sh container NAME PORT into the running container NAME, as root, in its working directory # # On a host it runs in a venv of the host's own Python, which apt installs when the host has none. In a container it -# brings uv, checked against its published checksum, and a private Python, so the container's own are never touched. +# brings uv, checked against its published checksum and kept on the host, and a private Python, so the container's own +# are never touched; in one whose image bake.sh baked the proxy into, for these requirements, it only starts it. set -euo pipefail -UV_VERSION=0.13.1 -UV_SHA256_X86_64=6c86040a8f045ab9adc9e3ec4cdafa03fd00e9a988ed2bf28971c7fc2d0debd0 -UV_SHA256_AARCH64=345a8308e4d1835cdfaadf692836fd3137a6d45864773f05ad7bf59442b1ba16 -PYTHON_VERSION=3.12 - mode=$1 name=$2 port=$3 here=$(cd "$(dirname "$0")" && pwd) +. "$here/versions.sh" +sha256() { if command -v sha256sum >/dev/null; then sha256sum "$1"; else shasum -a 256 "$1"; fi | cut -d' ' -f1; } case $mode in host) @@ -40,28 +38,37 @@ case $mode in target=$name root=/opt/agentenv-proxy-$port # one per port: two agents in a container listen on two on() { docker exec -u 0 "$target" sh -c "$1"; } - case $(on 'uname -m') in - x86_64|amd64) uv_cpu=x86_64 uv_sha256=$UV_SHA256_X86_64 ;; - aarch64|arm64) uv_cpu=aarch64 uv_sha256=$UV_SHA256_AARCH64 ;; - *) echo "uv has no build for $(on 'uname -m')" >&2; exit 1 ;; - esac - stage=$(mktemp -d "$here/../stage.XXXXXX") - trap 'rm -rf "$stage"' EXIT - uv_archive=uv-$uv_cpu-unknown-linux-musl.tar.gz - curl -fsSL --retry 3 "https://github.com/astral-sh/uv/releases/download/$UV_VERSION/$uv_archive" -o "$stage/$uv_archive" - sum=$(if command -v sha256sum >/dev/null; then sha256sum "$stage/$uv_archive"; else shasum -a 256 "$stage/$uv_archive"; fi) - [ "${sum%% *}" = "$uv_sha256" ] || { echo "$uv_archive doesn't match its published checksum" >&2; exit 1; } - tar -xzf "$stage/$uv_archive" -C "$stage" # An earlier install's proxy, a retried step's say, stops first, so its port is free. on "if [ -f $root/proxy.pid ]; then kill \$(cat $root/proxy.pid) 2>/dev/null || true; fi; rm -rf $root; mkdir -p $root/bin" - docker cp "$stage/uv-$uv_cpu-unknown-linux-musl/uv" "$target:$root/bin/uv" >/dev/null + if [ "$(on "cat $BAKED/baked 2>/dev/null" || true)" = "$(sha256 "$here/requirements.txt")" ]; then + python=$BAKED/venv/bin/python + else + case $(on 'uname -m') in + x86_64|amd64) uv_cpu=x86_64 uv_sha256=$UV_SHA256_X86_64 ;; + aarch64|arm64) uv_cpu=aarch64 uv_sha256=$UV_SHA256_AARCH64 ;; + *) echo "uv has no build for $(on 'uname -m')" >&2; exit 1 ;; + esac + stage=$(mktemp -d "$here/../stage.XXXXXX") + trap 'rm -rf "$stage"' EXIT + uv_archive=uv-$uv_cpu-unknown-linux-musl.tar.gz + cached=${XDG_CACHE_HOME:-${HOME:-/tmp}/.cache}/agentenv-proxy/$UV_VERSION-$uv_archive + if [ ! -f "$cached" ] || [ "$(sha256 "$cached")" != "$uv_sha256" ]; then + mkdir -p "$(dirname "$cached")" + curl -fsSL --retry 3 "https://github.com/astral-sh/uv/releases/download/$UV_VERSION/$uv_archive" -o "$stage/$uv_archive" + [ "$(sha256 "$stage/$uv_archive")" = "$uv_sha256" ] || { echo "$uv_archive doesn't match its published checksum" >&2; exit 1; } + mv -f "$stage/$uv_archive" "$cached.$$" && mv -f "$cached.$$" "$cached" + fi + tar -xzf "$cached" -C "$stage" + docker cp "$stage/uv-$uv_cpu-unknown-linux-musl/uv" "$target:$root/bin/uv" >/dev/null + docker cp "$here/requirements.txt" "$target:$root/requirements.txt" >/dev/null + on "cd $root && export UV_PYTHON_PREFERENCE=only-managed UV_PYTHON_INSTALL_DIR=$root/python UV_CACHE_DIR=$root/cache \ + && bin/uv venv --quiet --python $PYTHON_VERSION venv && bin/uv pip install --quiet --python venv/bin/python \ + -r requirements.txt && rm -rf cache" + python=$root/venv/bin/python + fi docker cp "$here/a2a_server.py" "$target:$root/a2a_server.py" >/dev/null - docker cp "$here/requirements.txt" "$target:$root/requirements.txt" >/dev/null - on "cd $root && export UV_PYTHON_PREFERENCE=only-managed UV_PYTHON_INSTALL_DIR=$root/python UV_CACHE_DIR=$root/cache \ - && bin/uv venv --quiet --python $PYTHON_VERSION venv && bin/uv pip install --quiet --python venv/bin/python \ - -r requirements.txt && rm -rf cache" docker exec -d -u 0 -e A2A_PORT="$port" "$target" \ - sh -c "echo \$\$ > $root/proxy.pid; exec $root/venv/bin/python $root/a2a_server.py > $root/proxy.log 2>&1" + sh -c "echo \$\$ > $root/proxy.pid; exec $python $root/a2a_server.py > $root/proxy.log 2>&1" ;; *) echo "usage: install.sh host NAME PORT WORKDIR USER | install.sh container NAME PORT" >&2; exit 2 ;; esac diff --git a/src/agent_env/agents/agents/proxy/versions.sh b/src/agent_env/agents/agents/proxy/versions.sh new file mode 100644 index 000000000..4708f37dc --- /dev/null +++ b/src/agent_env/agents/agents/proxy/versions.sh @@ -0,0 +1,8 @@ +# What install.sh and bake.sh share, sourced by both. A uv bump changes UV_VERSION, both checksums and UV_IMAGE_DIGEST, +# the digest of ghcr.io/astral-sh/uv:$UV_VERSION's image index, which the bake recipe pins. +UV_VERSION=0.13.1 +UV_SHA256_X86_64=6c86040a8f045ab9adc9e3ec4cdafa03fd00e9a988ed2bf28971c7fc2d0debd0 +UV_SHA256_AARCH64=345a8308e4d1835cdfaadf692836fd3137a6d45864773f05ad7bf59442b1ba16 +UV_IMAGE_DIGEST=sha256:4a1f3666816dafadd20ceb71eac9a7d56d7ef30279f7d64048519ab901ada336 +PYTHON_VERSION=3.12 +BAKED=/opt/agentenv-proxy # where bake.sh puts the proxy in an image diff --git a/src/agent_env/task_step/task_steps/install_agent.py b/src/agent_env/task_step/task_steps/install_agent.py index 7ddbbd8f9..6fdd517a9 100644 --- a/src/agent_env/task_step/task_steps/install_agent.py +++ b/src/agent_env/task_step/task_steps/install_agent.py @@ -22,6 +22,7 @@ import httpx from agent_env.a2a_agent import A2AAgent +from agent_env.artifact.artifacts.docker_image import DockerImageArtifact from agent_env.config import get_config from agent_env.entity.scopes import FieldScope from agent_env.providers.sandbox_providers.local_docker.provider import ( @@ -47,6 +48,9 @@ USER_HOST_KEY = "user_host" # The per-step param that installs this agent in place of the step's own, as an episode plays it through a proxy. PROXY_PARAM = "proxy" +# Each agent image's read of its card's install/v1 extension, by the image artifact's id and version, under which the +# image never changes. +_INSTALL_EXTENSIONS: dict[tuple[str, Optional[int]], asyncio.Future] = {} class HostPlace(NamedTuple): @@ -266,7 +270,7 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: logger.info(f"Loading agent image {agent.docker_image_artifact.image_name} onto VM (needed to read agent card)...") await sandbox.load_docker_images([agent.docker_image_artifact]) - install_ext = await self._extract_install_extension(sandbox, agent.docker_image_artifact.image_name) + install_ext = await self._extract_install_extension(sandbox, agent.docker_image_artifact) params = install_ext.get("params") or {} host_mode = self.container_name is None commands_key = INSTALL_COMMANDS_HOST_KEY if host_mode else INSTALL_COMMANDS_KEY @@ -406,7 +410,7 @@ async def _host_place_of(self, sandbox, agent: A2AAgent) -> HostPlace: ) logger.info(f"Loading agent image {agent.docker_image_artifact.image_name} onto VM (to read where it works)...") await sandbox.load_docker_images([agent.docker_image_artifact]) - install_ext = await self._extract_install_extension(sandbox, agent.docker_image_artifact.image_name) + install_ext = await self._extract_install_extension(sandbox, agent.docker_image_artifact) params = install_ext.get("params") or {} workdir, user = params.get(WORKDIR_HOST_KEY), params.get(USER_HOST_KEY) if not all(isinstance(value, str) and value for value in (workdir, user)): @@ -448,10 +452,20 @@ def _instance_ttl_seconds(ds) -> int: pass return TaskStep.DEFAULT_TTL_SECONDS - async def _extract_install_extension(self, sandbox, agent_image_name: str) -> dict: - """Run the agent image briefly with --entrypoint python3 to dump its AGENT_CARD and find the install/v1 extension.""" - from agent_env.a2a_agent import A2AAgent + async def _extract_install_extension(self, sandbox, image: DockerImageArtifact) -> dict: + """The install/v1 extension of the card ``image`` serves, read once in this process for each image artifact + version: installs at once share one read, and one that failed, or ran on another event loop, is made again.""" + key = (image.id, image.version) + read = _INSTALL_EXTENSIONS.get(key) + if (read is None or read.get_loop() is not asyncio.get_running_loop() + or (read.done() and (read.cancelled() or read.exception() is not None))): + read = asyncio.ensure_future(self._read_install_extension(sandbox, image.image_name)) + _INSTALL_EXTENSIONS[key] = read + return await asyncio.shield(read) + @staticmethod + async def _read_install_extension(sandbox, agent_image_name: str) -> dict: + """Run the agent image briefly with --entrypoint python3 to dump its AGENT_CARD and find the install/v1 extension.""" script = ( "import sys, json; sys.path.insert(0, '/app'); " "from a2a_server import AGENT_CARD; " @@ -481,7 +495,7 @@ def _check_tunnel_collision(self, context: TaskStepContext, sandbox_id: Optional if other.sandbox_id == sandbox_id and other.api_url == a2a_url: raise RuntimeError(f"Tunnel for port {a2a_port} ({a2a_url}) already serves deployed agent '{other.agent_name}' on sandbox {sandbox_id}; pass a distinct a2a_port on this step") - async def _wait_for_agent_card(self, a2a_url: str, timeout_s: int = 300, poll_s: int = 5) -> dict: + async def _wait_for_agent_card(self, a2a_url: str, timeout_s: int = 300, poll_s: float = 0.5) -> dict: deadline = asyncio.get_event_loop().time() + timeout_s last_err: Exception | None = None async with httpx.AsyncClient() as client: diff --git a/tst/integration/agent/proxy_container_install_test.py b/tst/integration/agent/proxy_container_install_test.py index f2ef19fd4..58d90fa61 100644 --- a/tst/integration/agent/proxy_container_install_test.py +++ b/tst/integration/agent/proxy_container_install_test.py @@ -41,7 +41,7 @@ def test_installed_into_a_container_the_proxy_serves_and_its_bash_runs_the_shell assert has_bash == (shell == "bash") folder = tmp_path / "agent" folder.mkdir() - for file in ("install.sh", "a2a_server.py"): + for file in ("install.sh", "versions.sh", "a2a_server.py"): shutil.copy(AGENT_DIR / file, folder / file) (folder / "requirements.txt").write_text("agentenv-framework-protocol[agent] @ file:///tmp/protocol\n") subprocess.run(["docker", "cp", f"{PROTOCOL_DIR}/.", f"{name}:/tmp/protocol"], check=True, capture_output=True) diff --git a/tst/integration/agent/proxy_local_container_test.py b/tst/integration/agent/proxy_local_container_test.py new file mode 100644 index 000000000..223c6e689 --- /dev/null +++ b/tst/integration/agent/proxy_local_container_test.py @@ -0,0 +1,98 @@ +"""An agent installed into a task container on the local sandbox, played from this machine through the proxy installed +in its place: an image with the proxy baked in only starts it, one without gets uv and a venv, and both play the turn +and pass the task's tests. Needs a Docker daemon, and network access for the images, uv, Python and the proxy's +requirements.""" + +import shutil +import subprocess +import tempfile +import uuid +from pathlib import Path + +import pytest + +from agent_env.a2a_agent.a2a_agent import A2AAgent +from agent_env.artifact import DockerImageArtifact +from agent_env.artifact.store import reset_artifact_store +from agent_env.config import configure, reset_config +from agent_env.task import EpisodeSpec, Task +from tst.util.proxy_bake import BAKE_RECIPE, PROXY_DIR + +pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow] + +TASK_DOCKERFILE = """FROM mirror.gcr.io/library/debian:bookworm-slim +RUN mkdir -p /work && echo hello > /work/greeting.txt +WORKDIR /work +""" +# An agent the proxy replaces in a container needs no more than a build context. +REPLACED_DOCKERFILE = "FROM mirror.gcr.io/library/python:3.12-slim\n" + + +@pytest.fixture +def local_backends(monkeypatch, tmp_path): + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("XDG_STATE_HOME", str(tmp_path / "state")) + monkeypatch.setenv("AGENT_ENV_DOCUMENT_STORE", "local") + monkeypatch.setenv("AGENT_ENV_OBJECT_STORE", "local") + monkeypatch.setenv("AGENT_ENV_LOCAL_SANDBOX_DIR", str(tmp_path / "sandboxes")) + configure() + reset_artifact_store() + try: + yield tmp_path + finally: + reset_artifact_store() + reset_config() + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("baked", "proxy"), [(True, "baked"), (False, "installed")], ids=["baked", "installed"]) +async def test_an_agent_installed_into_a_task_container_is_played_through_the_proxy_in_its_place(local_backends, baked, + proxy): + uid = uuid.uuid4().hex[:8] + image = _task_image(local_backends / "task-image", uid, baked) + task = Task.from_dict({"id": f"proxy-local-{uid}", "steps": [ + {"id": "box", "type": "deploy_sandbox", "sandbox_name": "box", "sandbox_mode": "vm", "sandbox_type": "local", + "exposed_ports": [8000]}, + {"id": "task", "type": "run_docker_container", "sandbox_name": "box", "container_name": "task", + "image_artifact_id": image.id, "ports": [8000], "command_override": "sleep infinity"}, + {"id": "install", "type": "install_agent", "sandbox_name": "box", "container_name": "task", + "a2a_agent_id": _replaced(uid).id, "agent_name": "solver"}, + {"id": "solve", "type": "prompt_agent", "agent_name": "solver", "prompt_id": "solve", + "prompt": "Copy greeting.txt to answer.txt."}, + {"id": "check", "type": "run_container_unit_tests_verifier", "sandbox_name": "box", "container_name": "task", + "command": "grep -qx hello /work/answer.txt"}, + ]}) + + try: + episode = await EpisodeSpec.from_task(task).start() + try: + turn = await episode.next_turn() + async with turn.tools() as tools: + where = "test -e /opt/agentenv-proxy-8000/bin/uv && echo installed || echo baked" + assert (await tools.call("bash", {"command": f"cat greeting.txt; {where}"})).text == f"hello\n{proxy}\n" + await tools.call("bash", {"command": "cp greeting.txt answer.txt"}) + await turn.end("Copied.") + assert await episode.result() == {"check": 1.0} + finally: + await episode.abandon("the test ended") + finally: + subprocess.run(["docker", "rmi", "-f", image.image_name], capture_output=True) + + +def _task_image(context: Path, uid: str, baked: bool) -> DockerImageArtifact: + context.mkdir() + bake = "".join(f"{line}\n" for line in BAKE_RECIPE) if baked else "" + (context / "Dockerfile").write_text(TASK_DOCKERFILE + bake) + if baked: + shutil.copytree(PROXY_DIR, context / "agentenv-proxy", ignore=shutil.ignore_patterns("__pycache__")) + return DockerImageArtifact.put_context(f"proxy-task-image-{uid}", description="a task container", + context_path=str(context), dockerfile_path="Dockerfile", platform=None) + + +def _replaced(uid: str) -> A2AAgent: + with tempfile.TemporaryDirectory() as build_dir: + context = Path(build_dir) + (context / "Dockerfile").write_text(REPLACED_DOCKERFILE) + image = DockerImageArtifact.put_context(f"replaced-agent-image-{uid}", description="a replaced agent", + context_path=str(context), dockerfile_path="Dockerfile", platform=None) + return A2AAgent.put(id=f"replaced-agent-{uid}", docker_image_artifact=image) diff --git a/tst/integration/cli/run_bundle_task_image_test.py b/tst/integration/cli/run_bundle_task_image_test.py index f1a32e024..3a6ea17ab 100644 --- a/tst/integration/cli/run_bundle_task_image_test.py +++ b/tst/integration/cli/run_bundle_task_image_test.py @@ -5,6 +5,7 @@ import json import logging +import subprocess import pytest from click.testing import CliRunner @@ -58,12 +59,15 @@ def test_a_task_container_from_an_image_folder_is_built_by_the_first_run_alone(s builds, build = [], host_images._build monkeypatch.setattr(host_images, "_build", lambda *args: builds.append(args[2]) or build(*args)) - first = CliRunner().invoke(cli, ["run", str(root)]) + try: + first = CliRunner().invoke(cli, ["run", str(root)]) - assert first.exit_code == 0 and "tasks/grade.json v1: passed" in first.output, first.output - assert len(builds) == 1, (first.output, builds) + assert first.exit_code == 0 and "tasks/grade.json v1: passed" in first.output, first.output + assert len(builds) == 1, (first.output, builds) - again = CliRunner().invoke(cli, ["run", str(root)]) + again = CliRunner().invoke(cli, ["run", str(root)]) - assert again.exit_code == 0 and "tasks/grade.json v1: passed" in again.output, again.output - assert "building with docker" not in again.output and len(builds) == 1, (again.output, builds) + assert again.exit_code == 0 and "tasks/grade.json v1: passed" in again.output, again.output + assert "building with docker" not in again.output and len(builds) == 1, (again.output, builds) + finally: + subprocess.run(["docker", "rmi", "-f", *builds], capture_output=True) # teardown keeps it, for a next run diff --git a/tst/integration/cli/run_bundle_test.py b/tst/integration/cli/run_bundle_test.py index 4df05a23d..caa94e5e8 100644 --- a/tst/integration/cli/run_bundle_test.py +++ b/tst/integration/cli/run_bundle_test.py @@ -241,7 +241,8 @@ def test_a_signal_mid_run_marks_the_run_cancelled_tears_it_down_and_stops_its_co "bash_cmd": 'sleep 30 & echo $! > "$HOME/nap.pid"; wait'}]}, ])) - exit_code, output, held = _signalled([napping], "step 2/2 nap (verify_sandbox)", signum, group) + nap = tmp_path / "nap.pid" + exit_code, output, held = _signalled([napping], "step 2/2 nap (verify_sandbox)", signum, group, ready=nap.exists) assert exit_code == code, output assert len(held) == 1 @@ -250,7 +251,7 @@ def test_a_signal_mid_run_marks_the_run_cancelled_tears_it_down_and_stops_its_co (instance_id,) = re.findall(r"instance (\S+)", output) with namespace_routing(): assert Task.get_instance(instance_id).status == "cancelled" - assert not _alive(int((tmp_path / "nap.pid").read_text())) + assert not _alive(int(nap.read_text())) def test_a_run_killed_outright_leaves_its_sandboxes_under_its_root_for_a_supervisor_to_terminate(bundle, tmp_path): diff --git a/tst/unit/agents/proxy/test_proxy_agent.py b/tst/unit/agents/proxy/test_proxy_agent.py index 39a5fcdc2..ad9c2e479 100644 --- a/tst/unit/agents/proxy/test_proxy_agent.py +++ b/tst/unit/agents/proxy/test_proxy_agent.py @@ -2,8 +2,14 @@ from __future__ import annotations +import hashlib +import io +import os import re +import shutil import subprocess +import tarfile +from pathlib import Path import pytest @@ -13,6 +19,9 @@ from agent_env.task_step.task_steps.install_agent import _build_param_resolvers from tst.unit.agents.proxy.agent import AGENT_DIR, a2a_server +from tst.util.proxy_bake import BAKE_RECIPE, VERSIONS + +README = Path(__file__).resolve().parents[4] / "README.md" def _extensions() -> dict: @@ -45,9 +54,17 @@ def test_each_install_asks_only_for_params_install_agent_supplies_and_runs_the_f def test_the_install_script_parses_and_the_image_keeps_it_in_its_build_context(): subprocess.run(["bash", "-n", str(AGENT_DIR / "install.sh")], check=True) + for script in ("bake.sh", "versions.sh"): + subprocess.run(["sh", "-n", str(AGENT_DIR / script)], check=True) assert "COPY a2a_server.py install.sh ./" in (AGENT_DIR / "Dockerfile").read_text() +def test_the_readme_and_bake_sh_give_the_bake_recipe_for_the_versions_the_install_brings(): + assert re.fullmatch(r"sha256:[0-9a-f]{64}", VERSIONS["UV_IMAGE_DIGEST"]) + assert "\n".join(BAKE_RECIPE) in README.read_text() + assert "\n".join(f"# {line}" for line in BAKE_RECIPE) in (AGENT_DIR / "bake.sh").read_text() + + def test_the_image_works_in_app_as_agent(): lines = (AGENT_DIR / "Dockerfile").read_text().splitlines() last = {key: next(line.split(maxsplit=1)[1] for line in reversed(lines) if line.startswith(f"{key} ")) @@ -62,3 +79,93 @@ def test_the_episode_writes_it_from_the_bundle_taking_no_credentials(local_store assert (agent.version, agent.metadata) == (version, {"takes_credentials": False}) assert agent.docker_image_artifact.context_only assert write_installed(PROXY_AGENT_ID) == version + + +@pytest.mark.parametrize("baked", [None, "0" * 64], ids=["no-proxy-baked", "other-requirements-baked"]) +def test_a_container_without_this_proxy_baked_in_gets_uv_and_a_venv(tmp_path, baked): + calls, exit_code = _install(tmp_path, baked) + + assert exit_code != 0 and calls[-1].startswith("curl -fsSL --retry 3 https://github.com/astral-sh/uv/releases/") + assert not [call for call in calls if call.startswith("docker exec -d")] + + +def test_a_container_with_this_proxy_baked_in_only_starts_it(tmp_path): + requirements = hashlib.sha256((AGENT_DIR / "requirements.txt").read_bytes()).hexdigest() + + calls, exit_code = _install(tmp_path, requirements) + + assert exit_code == 0 and not [call for call in calls if call.startswith("curl")] + assert [call.split()[3] for call in calls if call.startswith("docker cp")] == [ + "task:/opt/agentenv-proxy-8000/a2a_server.py"] + assert calls[-1].endswith("exec /opt/agentenv-proxy/venv/bin/python /opt/agentenv-proxy-8000/a2a_server.py " + "> /opt/agentenv-proxy-8000/proxy.log 2>&1") + + +@pytest.mark.parametrize(("cached", "downloads"), [("good", 0), (b"not uv", 1), (None, 1)], + ids=["a-good-copy", "a-corrupt-copy", "no-copy"]) +def test_a_full_install_takes_uv_from_the_hosts_copy_unless_it_isnt_good(tmp_path, cached, downloads): + archive = _uv_archive(tmp_path) + + calls, exit_code = _install(tmp_path, None, archive=archive, cached=archive.read_bytes() if cached == "good" else cached) + + assert exit_code == 0 and len([call for call in calls if call.startswith("curl")]) == downloads + assert _cached(tmp_path).read_bytes() == archive.read_bytes() + assert [call.split()[3] for call in calls if call.startswith("docker cp")] == [ + "task:/opt/agentenv-proxy-8000/bin/uv", "task:/opt/agentenv-proxy-8000/requirements.txt", + "task:/opt/agentenv-proxy-8000/a2a_server.py"] + assert calls[-1].endswith("exec /opt/agentenv-proxy-8000/venv/bin/python /opt/agentenv-proxy-8000/a2a_server.py " + "> /opt/agentenv-proxy-8000/proxy.log 2>&1") + + +def _install(tmp_path, baked: str | None, *, archive: Path | None = None, + cached: bytes | None = None) -> tuple[list[str], int]: + """``install.sh container task 8000`` against a fake docker whose container's image baked a proxy holding the + requirements digest ``baked``, or none, with the host holding ``cached`` as its copy of uv, and a curl that serves + ``archive`` as uv, pinned to its checksum, or fails without one: every docker and curl call, and the exit code.""" + agent, bin_dir, log = tmp_path / "agent", tmp_path / "bin", tmp_path / "calls.log" + shutil.copytree(AGENT_DIR, agent, ignore=shutil.ignore_patterns("__pycache__")) + if archive is not None: + versions = agent / "versions.sh" + versions.write_text(re.sub(r"^UV_SHA256_X86_64=\S+$", f"UV_SHA256_X86_64={_sha256(archive.read_bytes())}", + versions.read_text(), flags=re.M)) + if cached is not None: + _cached(tmp_path).parent.mkdir(parents=True) + _cached(tmp_path).write_bytes(cached) + bin_dir.mkdir() + (bin_dir / "docker").write_text(f"""#!/bin/sh +printf 'docker %s\\n' "$*" >> {log} +case "$*" in + *"cat /opt/agentenv-proxy/baked"*) [ -n "$FAKE_BAKED" ] && echo "$FAKE_BAKED" ;; + *"uname -m"*) echo x86_64 ;; +esac +exit 0 +""") + (bin_dir / "curl").write_text(f"""#!/bin/sh +printf 'curl %s\\n' "$*" >> {log} +[ -n "$FAKE_UV" ] || exit 22 +while [ $# -gt 1 ]; do [ "$1" = -o ] && cp "$FAKE_UV" "$2"; shift; done +""") + for stub in ("docker", "curl"): + (bin_dir / stub).chmod(0o755) + env = {**os.environ, "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}", "FAKE_BAKED": baked or "", + "FAKE_UV": str(archive or ""), "HOME": str(tmp_path), "XDG_CACHE_HOME": str(tmp_path / "cache")} + done = subprocess.run(["bash", str(agent / "install.sh"), "container", "task", "8000"], capture_output=True, env=env) + return log.read_text().splitlines(), done.returncode + + +def _uv_archive(tmp_path) -> Path: + """A uv release archive whose uv is a stub.""" + archive = tmp_path / "uv.tar.gz" + with tarfile.open(archive, "w:gz") as tar: + uv = tarfile.TarInfo("uv-x86_64-unknown-linux-musl/uv") + uv.size, uv.mode = 10, 0o755 + tar.addfile(uv, io.BytesIO(b"#!/bin/sh\n")) + return archive + + +def _cached(tmp_path) -> Path: + return tmp_path / "cache" / "agentenv-proxy" / f"{VERSIONS['UV_VERSION']}-uv-x86_64-unknown-linux-musl.tar.gz" + + +def _sha256(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() diff --git a/tst/unit/task_step/install_agent_test.py b/tst/unit/task_step/install_agent_test.py index 981362a91..22f55a2fa 100644 --- a/tst/unit/task_step/install_agent_test.py +++ b/tst/unit/task_step/install_agent_test.py @@ -3,7 +3,9 @@ from __future__ import annotations import asyncio +import json import logging +import uuid from types import SimpleNamespace import pytest @@ -336,7 +338,8 @@ def _installing_in_place(monkeypatch, *, codex=CODEX_INSTALL, sandbox=None, cont step = InstallAgentTaskStep(id="i", version=None, sandbox_name="vm", a2a_agent_id="agent-x", agent_name="solver", container_name=container_name) cards = {"agent-x-image": codex, f"{PROXY}-image": PROXY_INSTALL} - monkeypatch.setattr(step, "_extract_install_extension", lambda sandbox, image: _resolved(cards[image])()) + monkeypatch.setattr(step, "_extract_install_extension", + lambda sandbox, image: _resolved(cards[image.image_name])()) monkeypatch.setattr(step, "_wait_for_agent_card", _resolved({})) monkeypatch.setattr(step, "_register_instance", lambda *args: SimpleNamespace(instance_id="solver-1")) context = TaskStepContext( @@ -401,3 +404,59 @@ def test_only_an_install_in_another_agents_place_gives_its_workdir_and_user(): agent_name="solver", replaced=HostPlace(workdir="/opt/agent-solver/workspace", user="agent-solver"), ) assert (placed["host_workdir"](), placed["host_user"]()) == ("/opt/agent-solver/workspace", "agent-solver") + + +class _CardSandbox: + """Runs an agent image's card dump, answering after ``delay`` with a card offering ``INSTALL``, or failing first + ``failures`` times; each run recorded.""" + + def __init__(self, delay: float = 0, failures: int = 0): + self.runs, self.delay, self.failures = [], delay, failures + + async def exec_with_output(self, *args): + self.runs.append(args[-3]) + await asyncio.sleep(self.delay) + if len(self.runs) <= self.failures: + return 1, "", "the image's Python exited" + return 0, json.dumps({"capabilities": {"extensions": [INSTALL]}}), "" + + +INSTALL = {"uri": A2AAgent.EXT_INSTALL, "params": {"a2a_port": 8000}} + + +def _card_image(version: int, name: str | None = None): + return SimpleNamespace(id=f"agent-image-{uuid.uuid4().hex}" if name is None else name, version=version, + image_name="agent:latest") + + +def _card_step(): + return InstallAgentTaskStep(id="i", version=None, sandbox_name="vm", a2a_agent_id="agent-x", agent_name="solver") + + +@pytest.mark.asyncio +async def test_an_images_install_extension_is_read_once_for_each_version_of_its_artifact(): + """A tarball's images carry no digest, so two versions under one name differ only as versions.""" + sandbox, name = _CardSandbox(), f"agent-image-{uuid.uuid4().hex}" + + read = [await _card_step()._extract_install_extension(sandbox, _card_image(version, name)) for version in (1, 1, 2)] + + assert read == [INSTALL] * 3 and sandbox.runs == ["agent:latest", "agent:latest"] + + +@pytest.mark.asyncio +async def test_installs_at_once_share_one_read_of_an_images_install_extension(): + sandbox, image = _CardSandbox(delay=0.05), _card_image(1) + + read = await asyncio.gather(*(_card_step()._extract_install_extension(sandbox, image) for _ in range(8))) + + assert read == [INSTALL] * 8 and len(sandbox.runs) == 1 + + +@pytest.mark.asyncio +async def test_a_failed_read_of_an_images_install_extension_is_made_again(): + sandbox, image = _CardSandbox(failures=1), _card_image(1) + + with pytest.raises(RuntimeError, match="the image's Python exited"): + await _card_step()._extract_install_extension(sandbox, image) + + assert await _card_step()._extract_install_extension(sandbox, image) == INSTALL and len(sandbox.runs) == 2 diff --git a/tst/util/proxy_bake.py b/tst/util/proxy_bake.py new file mode 100644 index 000000000..2fc30ad6f --- /dev/null +++ b/tst/util/proxy_bake.py @@ -0,0 +1,16 @@ +"""The proxy's shared versions and paths (its ``versions.sh``), and the Dockerfile lines that bake it into a task image, +as the README and ``bake.sh`` give them.""" + +import re +from pathlib import Path + +import agent_env + +PROXY_DIR = Path(agent_env.__file__).parent / "agents" / "agents" / "proxy" +VERSIONS = dict(re.findall(r"^(\w+)=(\S+)", (PROXY_DIR / "versions.sh").read_text(), re.M)) +BAKE_RECIPE = ( + f"COPY --from=ghcr.io/astral-sh/uv:{VERSIONS['UV_VERSION']}@{VERSIONS['UV_IMAGE_DIGEST']} /uv " + f"{VERSIONS['BAKED']}/bin/uv", + f"COPY agentenv-proxy {VERSIONS['BAKED']}/src", + f"RUN sh {VERSIONS['BAKED']}/src/bake.sh", +)