From f49a3e7880b37d33e344e7a170a120e98adfb420 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sun, 15 Nov 2026 20:22:10 -0800 Subject: [PATCH 1/2] feat(computers): a release publishes chromium composed, and a compose of the same inputs pulls it (M8.60) Each release composes chromium from its own wheels on an amd64 and an arm64 runner, probes it, and joins the two images by their digests as one index in ghcr.io/scaleapi/agentenv-computers; the wheel records each arch's key, image and the lock it was composed from. A compose with that wheel takes the release's lock rather than resolving one, finds its key in the record, and records the released image by digest, pulled rather than built; anything that changes the key composes as before. The release's computer jobs never hold a release back. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/infra_build.py | 169 +++++++++++++++++- .github/scripts/infra_images.py | 44 +++-- .github/scripts/release_computers.py | 75 ++++++++ .github/workflows/publish-pypi.yml | 142 ++++++++++++++- AGENTS.md | 6 + src/agent_env/browsers/README.md | 5 +- src/agent_env/computers/README.md | 8 + .../env/envs/computer/utils/composer.py | 84 ++++++++- src/agent_env/env/published_images.py | 58 +++++- .../computer/browser_computer_test.py | 72 +++++++- tst/unit/env/envs/test_computer_env.py | 106 +++++++++++ tst/unit/env/published_images_test.py | 42 +++++ tst/unit/test_infra_build_script.py | 138 +++++++++++++- tst/unit/test_infra_images_script.py | 18 +- 14 files changed, 919 insertions(+), 48 deletions(-) create mode 100644 .github/scripts/release_computers.py diff --git a/.github/scripts/infra_build.py b/.github/scripts/infra_build.py index 36ae8ef82..58798f6d3 100644 --- a/.github/scripts/infra_build.py +++ b/.github/scripts/infra_build.py @@ -14,15 +14,20 @@ build of NAME, runs: a disk form boots in the QEMU launcher image ``--launcher`` names until every program its base's supervisord.conf declares runs. ``publish MISSING`` signs in to GHCR and tags the two platform builds of each entry of MISSING, a JSON list, as one multi-platform image under its tag, once the entry is checked, unless the registry has -that tag. ``verify-record PATH`` fails unless PATH records each image agent-env ships once, with the digest the -registry serves under its tag now. ``retry COMMAND...`` runs COMMAND, again while a registry rate-limits it or PyPI -doesn't serve yet the agentenv-framework-protocol an agent image's build installs. +that tag. ``publish-computers FOLDER`` checks the computer apps FOLDER's amd64.json and arm64.json say were composed +(``release_computers.py compose``), signs in to GHCR, joins each app's two images into one index in +``COMPUTERS_REPOSITORY`` from the local registry they were composed into, keeping their digests, and writes their record +as FOLDER/computers.json. ``verify-record PATH`` fails unless PATH records each image agent-env ships once, with the +digest the registry serves under its tag now, and each computer app it records once, its index served as recorded. +``retry COMMAND...`` runs COMMAND, again while a registry rate-limits it or PyPI doesn't serve yet the +agentenv-framework-protocol an agent image's build installs. """ from __future__ import annotations import argparse import base64 +import hashlib import json import os import platform @@ -65,6 +70,16 @@ # Each disk form's build args, by the images agent-env ships that they name; its SIZE_GB is its base's [forms.disk] # min_disk_gb, read from the artifact.toml beside the base's Dockerfile. DISK_ARGS = {"agentenv-ubuntu-24.04-disk": {"ROOTFS": "agentenv-ubuntu-24.04", "TOOLS": "agentenv-disk-tools"}} +# Each computer app a release publishes composed, by its installed env id (``release_computers.py``), into one +# repository, an index of each one's arches. +COMPUTERS = ("@local/agentenv-framework/browsers/chromium",) +COMPUTERS_REPOSITORY = f"{REGISTRY}/agentenv-computers" +COMPUTER_ARCHES = ("amd64", "arm64") +# A composed image in the local registry it was composed into, tagged by its key. +_COMPOSED = re.compile(r"^localhost:5000/agentenv-computers:([0-9a-f]{64})@(sha256:[0-9a-f]{64})$") +_COMPUTER_IMAGE = re.compile(rf"^{re.escape(COMPUTERS_REPOSITORY)}@sha256:[0-9a-f]{{64}}$") +_COMPUTER_INDEX = re.compile(rf"^{re.escape(COMPUTERS_REPOSITORY)}:index-[0-9a-f]{{64}}@sha256:[0-9a-f]{{64}}$") +_INDEX_TYPES = ("application/vnd.oci.image.index.v1+json", "application/vnd.docker.distribution.manifest.list.v2+json") _TAG = re.compile(r"^src-[0-9a-f]{64}$") _DIGEST = re.compile(r"^sha256:[0-9a-f]{64}$") _HEALTHY_WITHIN_SECONDS = 120 @@ -156,6 +171,7 @@ def main() -> int: smoke.add_argument("image") smoke.add_argument("--launcher") commands.add_parser("publish").add_argument("missing") + commands.add_parser("publish-computers").add_argument("folder", type=Path) commands.add_parser("verify-record").add_argument("path", type=Path) commands.add_parser("retry").add_argument("argv", nargs=argparse.REMAINDER, metavar="COMMAND") args = parser.parse_args() @@ -176,6 +192,8 @@ def main() -> int: return _smoke(args.name, args.image, args.launcher) if args.command == "publish": return _publish(args.missing) + if args.command == "publish-computers": + return _publish_computers(args.folder) problems = _check(args.entry) if args.command == "check" else _verify_record(args.path) for problem in problems: print(f"::error::{problem}") @@ -238,6 +256,110 @@ def _publish(missing: str) -> int: return 0 +def _publish_computers(folder: Path) -> int: + """Push each computer app the ARCH.json files in ``folder`` name to ``COMPUTERS_REPOSITORY`` as one index of its + arches' images, from the local registry they were composed into, each by its platform manifest's digest, which the + copy keeps, so each is the image its compose probed; and write ``folder``/computers.json, the record of them a + release wheel carries, each app's index and each arch's key, image and lock.""" + composed, problems = {}, [] + for arch in COMPUTER_ARCHES: + try: + entries = json.loads((folder / f"{arch}.json").read_text()) + except (OSError, ValueError) as e: + problems.append(f"{folder / f'{arch}.json'} isn't a list of composed apps: {type(e).__name__}: {e}") + continue + problems += _check_composed(arch, entries) + composed[arch] = {entry.get("env"): entry for entry in entries if isinstance(entry, dict)} + if not problems and sorted(composed[arch]) != sorted(COMPUTERS): + problems.append(f"{folder / f'{arch}.json'} composes {sorted(composed[arch])}, not {list(COMPUTERS)}") + for problem in problems: + print(f"::error::{problem}") + if problems: + return 1 + _sign_in() + computers = [] + for env in COMPUTERS: + built = {arch: composed[arch][env] for arch in COMPUTER_ARCHES} + digests = {arch: _platform_manifest(_COMPOSED.match(entry["image"]).group(2), arch) + for arch, entry in built.items()} + if missing := [arch for arch, digest in digests.items() if digest is None]: + print(f"::error::{env}'s composed {', '.join(missing)} has no linux manifest of its arch") + return 1 + images = {arch: f"{COMPUTERS_REPOSITORY}@{digest}" for arch, digest in digests.items()} + index = f"{COMPUTERS_REPOSITORY}:index-" + hashlib.sha256( + ",".join(images[arch] for arch in COMPUTER_ARCHES).encode()).hexdigest() + if (code := _exists(index)) == 2: + return 1 + if code == 1: + source = f"https://github.com/{os.environ['GITHUB_REPOSITORY']}" + created = _tried_again(lambda: _streamed([ + "docker", "buildx", "imagetools", "create", "-t", index, + "--annotation", f"index:org.opencontainers.image.source={source}", + "--annotation", f"index:org.opencontainers.image.revision={os.environ['GITHUB_SHA']}", + *(f"localhost:5000/agentenv-computers@{digests[arch]}" for arch in COMPUTER_ARCHES)])) + if created.returncode: + return created.returncode + served, error = _served(index) + if served is None or _children(index) != set(digests.values()): + print(f"::error::{index} serves {served or f'nothing ({error})'}, and not an index of exactly " + f"{sorted(digests.values())}") + return 1 + computers.append({"env": env, "form": "container", "index": f"{index}@{served}", "arches": { + arch: {"key": entry["key"], "image": images[arch], "lock": entry["lock"]} + for arch, entry in built.items()}}) + print(f"published {env} as {index}@{served}, {', '.join(f'{arch} {images[arch]}' for arch in COMPUTER_ARCHES)}") + (folder / "computers.json").write_text(json.dumps(computers, indent=2) + "\n") + return 0 + + +def _check_composed(arch: str, entries: object) -> list[str]: + """Each problem with ``entries``, as ``release_computers.py compose`` writes a list of the apps it composed for + ``arch``: an app a release publishes, its form, arch, key, image in the local registry under that key, and lock.""" + if not isinstance(entries, list): + return [f"{arch}: {entries!r} isn't a list of composed apps"] + problems = [] + for entry in entries: + if not isinstance(entry, dict) or set(entry) != {"env", "form", "arch", "key", "image", "lock"}: + problems.append(f"{arch}: {entry!r} isn't an app's env, form, arch, key, image and lock") + continue + image, lock = _COMPOSED.match(str(entry["image"])), entry["lock"] + if (entry["env"] not in COMPUTERS or entry["form"] != "container" or entry["arch"] != arch or image is None + or entry["key"] != f"sha256:{image.group(1)}" or not isinstance(lock, dict) or lock.get("arch") != arch + or not _DIGEST.match(str(lock.get("inputs")))): + problems.append(f"{arch}: {entry['env']!r} isn't an app a release publishes, composed for {arch} into the " + f"local registry under its key, with its lock: {entry['form']!r} {entry['arch']!r} " + f"{entry['key']!r} {entry['image']!r}") + return problems + + +def _raw(ref: str) -> dict | None: + """The manifest the registry serves under ``ref``, as it serves it, or None.""" + result = _tried_again(lambda: subprocess.run(["docker", "buildx", "imagetools", "inspect", ref, "--raw"], + capture_output=True, text=True)) + try: + return json.loads(result.stdout) if result.returncode == 0 else None + except ValueError: + return None + + +def _platform_manifest(digest: str, arch: str) -> str | None: + """The digest of the linux/``arch`` manifest of the local registry's composed image ``digest``: itself when it's a + manifest, else the one its index names for that platform, an attestation's aside.""" + manifest = _raw(f"localhost:5000/agentenv-computers@{digest}") + if manifest is None: + return None + if manifest.get("mediaType") not in _INDEX_TYPES and "manifests" not in manifest: + return digest + return next((child["digest"] for child in manifest.get("manifests", []) + if child.get("platform", {}).get("os") == "linux" and child["platform"].get("architecture") == arch), + None) + + +def _children(index: str) -> set[str]: + """The digests of the manifests the index under ``index`` names.""" + return {child["digest"] for child in (_raw(index) or {}).get("manifests", [])} + + def _checked(entry: str) -> bool: problems = _check(entry) for problem in problems: @@ -367,9 +489,9 @@ def _verify_record(path: Path) -> list[str]: images = record["images"] except (OSError, ValueError, KeyError, TypeError) as e: return [f"{path} isn't a record of published images: {type(e).__name__}: {e}"] - if set(record) != {"images"} or not isinstance(images, list): + if not {"images"} <= set(record) <= {"images", "computers"} or not isinstance(images, list): return [f"{path} isn't a record of published images: it holds {sorted(record)}"] - problems, seen = [], [] + problems, seen = _verify_computers(path, record.get("computers", [])), [] for image in images: if not isinstance(image, dict) or set(image) != {"repository", "tag", "digest"}: problems.append(f"{path}: {image!r} isn't an image's repository, tag and digest") @@ -394,6 +516,43 @@ def _verify_record(path: Path) -> list[str]: return problems +def _verify_computers(path: Path, computers: object) -> list[str]: + """Each problem with ``computers``, as a release wheel records the computer apps it published composed: an app of + ``COMPUTERS`` at most once, its index served under its tag with the digest recorded and naming exactly its arches' + images, each by digest, with the key it was composed under and its lock. A release that composed none records + none.""" + if not isinstance(computers, list): + return [f"{path}: {computers!r} isn't a list of composed computers"] + problems, seen = [], [] + for computer in computers: + if not isinstance(computer, dict) or set(computer) != {"env", "form", "index", "arches"}: + problems.append(f"{path}: {computer!r} isn't a composed computer's env, form, index and arches") + continue + env, arches, index = computer["env"], computer["arches"], str(computer["index"]) + seen.append(env) + if env not in COMPUTERS or computer["form"] != "container" or not _COMPUTER_INDEX.match(index): + problems.append(f"{path}: {env!r} isn't a computer app a release publishes, as a container index in " + f"{COMPUTERS_REPOSITORY}: {computer['form']!r} {index!r}") + continue + if not isinstance(arches, dict) or sorted(arches) != sorted(COMPUTER_ARCHES) or not all( + isinstance(built, dict) and set(built) == {"key", "image", "lock"} and _DIGEST.match(str(built["key"])) + and _COMPUTER_IMAGE.match(str(built["image"])) and isinstance(built["lock"], dict) + and built["lock"].get("arch") == arch for arch, built in arches.items()): + problems.append(f"{path}: {env}'s arches aren't {list(COMPUTER_ARCHES)}, each a key, an image by digest in " + f"{COMPUTERS_REPOSITORY} and its lock") + continue + tagged, _, digest = index.partition("@") + served, error = _served(tagged) + if served != digest: + problems.append(f"{path}: {tagged} is recorded as {digest}, and the registry serves " + f"{served or f'nothing ({error})'}") + elif _children(tagged) != {built["image"].partition("@")[2] for built in arches.values()}: + problems.append(f"{path}: {tagged} doesn't name exactly the images recorded for {env}'s arches") + if repeated := sorted({env for env in seen if seen.count(env) > 1}): + problems.append(f"{path} records {', '.join(repeated)} more than once") + return problems + + def _config(image: str) -> str: """The config object of ``image``'s OCI config, a local image's, as base64'd JSON.""" inspected = subprocess.run(["docker", "image", "inspect", "--format", "{{json .Config}}", image], diff --git a/.github/scripts/infra_images.py b/.github/scripts/infra_images.py index 8052fe044..bd312a975 100644 --- a/.github/scripts/infra_images.py +++ b/.github/scripts/infra_images.py @@ -9,9 +9,10 @@ only the images that install the protocol, the agents' and the website browser's, which a release builds after it publishes the protocol, as their protocol floor may name it, and ``--only disks`` the base computers' disk forms, built over their published images. ``write PATH`` records the digest the registry serves under every -image's tag as a release wheel carries them, and fails if any is missing. ``public`` warns about each image a user, who -pulls anonymously, can't. The registry is read with ``GHCR_TOKEN`` (as ``GITHUB_ACTOR``) when it's set, anonymously -otherwise, and always anonymously by ``public``. +image's tag as a release wheel carries them, and fails if any is missing; with ``--computers FILE``, the composed +computers ``infra_build.py publish-computers`` wrote there too. ``public`` warns about each image a user, who pulls +anonymously, can't, and each composed computer's index the record ``--record`` names. The registry is read with +``GHCR_TOKEN`` (as ``GITHUB_ACTOR``) when it's set, anonymously otherwise, and always anonymously by ``public``. """ from __future__ import annotations @@ -23,7 +24,9 @@ from pathlib import Path from agent_env.env.bootstrap import PLAYWRIGHT_MCP_DOCKERFILE, PUBLISHED_IMAGES -from agent_env.env.published_images import BUNDLE_IMAGES, DISK_IMAGES, REGISTRY, PublishedImage, write_release_digests +from agent_env.env.published_images import ( + BUNDLE_IMAGES, COMPUTERS_REPOSITORY, DISK_IMAGES, REGISTRY, PublishedImage, released_computers, + write_release_digests) from agent_env.store.image_store.oci_registry_credentials import RegistryAuth from agent_env.store.image_store.registry_api import NoSuchImageError, RegistryRefusedError, pin_digest @@ -42,8 +45,10 @@ def main() -> int: plan = commands.add_parser("plan") plan.add_argument("--missing", action="store_true") plan.add_argument("--only", choices=sorted(_GROUPS)) - commands.add_parser("write").add_argument("path", type=Path) - commands.add_parser("public") + write = commands.add_parser("write") + write.add_argument("path", type=Path) + write.add_argument("--computers", type=Path) + commands.add_parser("public").add_argument("--record", type=Path) args = parser.parse_args() if args.command == "plan": images = _GROUPS[args.only] if args.only else PUBLISHED_IMAGES @@ -51,14 +56,16 @@ def main() -> int: if not args.missing or _published_digest(image) is None])) return 0 if args.command == "public": - _warn_unless_public() + _warn_unless_public(released_computers(args.record) if args.record else []) return 0 digests = {image.ref(): _digest(image) for image in PUBLISHED_IMAGES} if missing := sorted(ref for ref, digest in digests.items() if digest is None): print(f"::error::the registry has no image under {', '.join(missing)}") return 1 - write_release_digests(digests, args.path) - print(f"recorded {len(digests)} published images in {args.path}") + computers = json.loads(args.computers.read_text()) if args.computers else [] + write_release_digests(digests, args.path, computers) + released_computers(args.path) + print(f"recorded {len(digests)} published images and {len(computers)} composed computers in {args.path}") return 0 @@ -88,17 +95,20 @@ def _published_digest(image: PublishedImage) -> str | None: return None -def _warn_unless_public() -> None: - """A warning for each image an anonymous pull doesn't get, which a new package's first push leaves it as: GHCR makes - a package private until its admin makes it public, and an install that can't pull it builds the image instead.""" - for image in PUBLISHED_IMAGES: +def _warn_unless_public(computers: list[dict]) -> None: + """A warning for each image, and each composed computer's index in ``computers``, an anonymous pull doesn't get, + which a new package's first push leaves it as: GHCR makes a package private until its admin makes it public, and an + install that can't pull it builds the image, or composes the computer, instead.""" + refs = [(image.name, image.ref()) for image in PUBLISHED_IMAGES] + refs += [(COMPUTERS_REPOSITORY.rpartition("/")[2], computer["index"]) for computer in computers] + for name, ref in refs: try: - pin_digest(image.ref(), None) + pin_digest(ref, None) except ValueError as e: why = str(e).split(": ", 1)[-1].split(";")[0] # what the registry answered, without the ref - settings = f"https://github.com/orgs/{REGISTRY.split('/')[1]}/packages/container/{image.name}/settings" - print(f"::warning title={image.name} isn't public::{image.ref()}: {why}. Make the package public at " - f"{settings}; until then an install can't pull it and builds the image instead.") + settings = f"https://github.com/orgs/{REGISTRY.split('/')[1]}/packages/container/{name}/settings" + print(f"::warning title={name} isn't public::{ref}: {why}. Make the package public at " + f"{settings}; until then an install can't pull it and builds it instead.") def _auth() -> RegistryAuth | None: diff --git a/.github/scripts/release_computers.py b/.github/scripts/release_computers.py new file mode 100644 index 000000000..086eb77fd --- /dev/null +++ b/.github/scripts/release_computers.py @@ -0,0 +1,75 @@ +"""The computer apps a release publishes composed (``APPS``, chromium), for the release workflow's jobs that only read: +this imports agent-env, so it runs where this release's wheels are installed, and no job that holds a token to push +images runs it (those run ``infra_build.py``). + +``compose ARCH OUT`` composes each app for linux/ARCH on this machine's Docker, emulating nothing, as an install of this +release composes it: from the wheels installed, over the base computers the release published, its packages resolved +from Ubuntu's archive, PyPI and download URLs. Each is composed in stores of its own, so into the local registry's +``agentenv-computers``, and probed at full (gate 3). It writes, as OUT/ARCH.json, each app's installed env id, form and +arch, the key it took, its image there by digest and the lock it was composed from, and fails unless each one is +ready. The local registry is the one at localhost:5000, and the stores are the configured local ones. +""" + +from __future__ import annotations + +import argparse +import json +import sys +import tempfile +from pathlib import Path + +from agent_env.bundle import parse_bundle +from agent_env.bundle.installed import installed_bundles +from agent_env.bundle.materialize import materialize +from agent_env.bundle.plan import plan_bundle +from agent_env.bundle.resolve import resolve_bundle +from agent_env.env import Env +from agent_env.env.envs.computer.utils import packages +from agent_env.env.envs.computer.utils.composer import Composing +from agent_env.providers.sandbox_providers.host_images import host_platform +from agent_env.store.routing import namespace_routing + +#: Each app a release publishes composed, by the alias a task names it by and its installed env id. +APPS = {"chromium": "@local/agentenv-framework/browsers/chromium"} + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + commands = parser.add_subparsers(dest="command", required=True) + compose = commands.add_parser("compose") + compose.add_argument("arch", choices=["amd64", "arm64"]) + compose.add_argument("out", type=Path) + args = parser.parse_args() + return _compose(args.arch, args.out) + + +def _compose(arch: str, out: Path) -> int: + if host_platform() != f"linux/{arch}": + print(f"::error::this runner is {host_platform()}, and a release composes linux/{arch} natively") + return 1 + with tempfile.TemporaryDirectory(prefix="agentenv-release-") as work, namespace_routing(): + tasks = Path(work) / "tasks" + tasks.mkdir() + for alias in APPS: + (tasks / f"{alias}.json").write_text(json.dumps([{"id": "env", "type": "deploy_env", "env_id": alias}])) + plan = plan_bundle(resolve_bundle(parse_bundle(Path(work))), installed=installed_bundles, default_agents=False) + materialize(plan, composing=Composing(None, emulate=False, on_progress=print)) + entries = [] + for env_id in APPS.values(): + env = Env.get(env_id) + built = env.metadata.get("composed", {}).get("container", {}).get(arch, {}) + if built.get("status") != "ready": + print(f"::error::{env_id}'s container/{arch} is {built.get('status', 'not composed')}: " + f"{built.get('reason', 'no reason recorded')}") + return 1 + lock = packages.stored(built["lock"], env.stored_id) + entries.append({"env": env_id, "form": "container", "arch": arch, "key": built["key"], + "image": built["image"], "lock": json.loads(lock.model_dump_json())}) + out.mkdir(parents=True, exist_ok=True) + (out / f"{arch}.json").write_text(json.dumps(entries, indent=2) + "\n") + print(f"composed {', '.join(APPS)} for linux/{arch}: {out / f'{arch}.json'}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml index a3b59d054..1271381dd 100644 --- a/.github/workflows/publish-pypi.yml +++ b/.github/workflows/publish-pypi.yml @@ -14,7 +14,11 @@ name: Publish to PyPI # variable PUBLISH_INFRA_IMAGES is "true". The infra images are published before agentenv-framework-protocol. The # bundles' images keep a protocol floor, so each is built only when its folder changes, once the protocol is on PyPI, # as a raised floor may name the protocol this release ships. A disk form is built over its base computer's published -# image with the published disk tools, so after both. agentenv-framework waits for every image. Every job that +# image with the published disk tools, so after both. A release also publishes the computer apps it ships composed +# (chromium), each a new image every release, composed and probed from this release's wheels once the protocol and the +# base computers are published, and records each one's keys, images and locks in the wheel, so an install's compose of +# the same inputs pulls it (published_images.released_computers); those never hold a release back. +# agentenv-framework waits for every other image. Every job that # reads the shipped bundles' folders first pins the protocol in their envs and artifacts as the wheel does # (.github/scripts/shipped_bundles.py pin), so an image's tag, its build and the wheel all see the same sources. Run # by hand on main, the workflow publishes the images alone, with nothing sent to PyPI; an agent's image then installs @@ -360,11 +364,113 @@ jobs: GHCR_TOKEN: ${{ github.token }} run: python3 .github/scripts/infra_build.py publish "$MISSING" + # Each computer app agent-env ships composed (release_computers.py: chromium), on a runner of each platform, as an + # install of this release composes it: from this release's wheels, over the base computers it published, its packages + # resolved, built, pushed to a local registry and probed at full. It holds no token that pushes images: it hands + # computer-publish the local registry's storage and the record of what it composed. A new image every release, since + # an app's venv pins the protocol the release ships. + computer-build: + needs: [main-ci, infra-plan, protocol, agent-publish] + if: >- + !cancelled() && needs.main-ci.result == 'success' && needs.infra-plan.result == 'success' + && needs.protocol.result == 'success' && contains(fromJSON('["success", "skipped"]'), needs.agent-publish.result) + strategy: + fail-fast: true + matrix: + arch: [amd64, arm64] + runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} + timeout-minutes: 90 + permissions: + contents: read + env: + ARCH: ${{ matrix.arch }} + XDG_STATE_HOME: ${{ runner.temp }}/state + AGENT_ENV_DOCUMENT_STORE: local + AGENT_ENV_OBJECT_STORE: local + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + - name: Install uv from public PyPI + run: python -m pip install --quiet --index-url https://pypi.org/simple -r .github/ci-requirements.txt + - name: Pin agentenv-framework-protocol in the framework and the shipped bundles, as the release does + run: | + python3 .github/scripts/shipped_bundles.py pin-pyproject + python3 .github/scripts/shipped_bundles.py pin + # A wheel, not this checkout: a checkout composes with the protocol's source in each venv, under another key. + - name: Install agentenv-framework, built as the release builds it, with the protocol it published + run: | + uv build --no-sources --wheel --out-dir dist + protocol=$(sed -n 's/^version = "\(.*\)"/\1/p' packages/agentenv-protocol/pyproject.toml) + python -m venv "$RUNNER_TEMP/venv" + python3 .github/scripts/infra_build.py retry "$RUNNER_TEMP/venv/bin/python" -m pip install --quiet \ + --index-url https://pypi.org/simple dist/agentenv_framework-*.whl \ + "agentenv-framework-protocol[app]==${protocol}" + - name: Start the local registry, its storage in a folder handed on + run: | + docker run --detach --name registry --publish 5000:5000 \ + --volume "$RUNNER_TEMP/computers/registry:/var/lib/registry" mirror.gcr.io/library/registry:2 + timeout 30 sh -c 'until curl -sf http://localhost:5000/v2/ >/dev/null; do sleep 1; done' + - name: Compose and probe + run: | + "$RUNNER_TEMP/venv/bin/python" .github/scripts/release_computers.py compose "$ARCH" \ + "$RUNNER_TEMP/computers/composed" + - name: Stop the local registry + run: docker stop registry + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: computer-${{ matrix.arch }} + path: ${{ runner.temp }}/computers + if-no-files-found: error + retention-days: 1 + + # Each composed app's two images, from the local registries computer-build composed them into, joined as one index + # in agentenv-computers by their digests, so each is the image its compose probed; and the record of them for the + # wheel. It installs nothing, holding the token that pushes. A release whose composed computers fail ships without + # them, and an install composes them as before, so these two jobs never hold a release back. + computer-publish: + needs: [main-ci, computer-build] + if: >- + !cancelled() && needs.main-ci.result == 'success' && needs.computer-build.result == 'success' + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: computer-* + path: ${{ runner.temp }}/computers + merge-multiple: true + - name: Serve both registries' storage, which are content-addressed and so merge, as one local registry + run: | + docker run --detach --name registry --publish 5000:5000 \ + --volume "$RUNNER_TEMP/computers/registry:/var/lib/registry" mirror.gcr.io/library/registry:2 + timeout 30 sh -c 'until curl -sf http://localhost:5000/v2/ >/dev/null; do sleep 1; done' + - name: Join each app's images as one index in agentenv-computers + env: + GHCR_TOKEN: ${{ github.token }} + run: python3 .github/scripts/infra_build.py publish-computers "$RUNNER_TEMP/computers/composed" + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: computers-record + path: ${{ runner.temp }}/computers/composed/computers.json + if-no-files-found: error + # The digest the registry serves under every image's tag, for the wheel and the release notes, which the framework - # job checks against the registry before the wheel carries it; and a warning for each image an anonymous pull, as an - # install makes, doesn't get: GHCR keeps a new package private until its admin makes it public. + # job checks against the registry before the wheel carries it, with the composed computers computer-publish joined; + # and a warning for each image an anonymous pull, as an install makes, doesn't get: GHCR keeps a new package private + # until its admin makes it public. infra-record: - needs: [infra-plan, infra-build, infra-publish, agent-build, agent-publish, disk-build, disk-publish] + needs: [infra-plan, infra-build, infra-publish, agent-build, agent-publish, disk-build, disk-publish, + computer-build, computer-publish] if: >- !cancelled() && needs.infra-plan.result == 'success' && contains(fromJSON('["success", "skipped"]'), needs.infra-build.result) @@ -391,16 +497,34 @@ jobs: run: python -m pip install --quiet --index-url https://pypi.org/simple -r .github/ci-requirements.txt - name: Pin agentenv-framework-protocol in the shipped bundles, as the release does run: python3 .github/scripts/shipped_bundles.py pin - - name: Record the published images' digests + - name: Warn when the composed computers aren't published + if: github.event_name == 'push' && needs.computer-publish.result != 'success' + run: | + echo "::warning title=No composed computers::computer-build or computer-publish didn't pass, so this" \ + "release ships no composed chromium and an install composes it" + - name: Take the composed computers' record + if: needs.computer-publish.result == 'success' + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: computers-record + path: ${{ runner.temp }}/computers + - name: Record the published images' digests, and the composed computers id: record env: GHCR_TOKEN: ${{ github.token }} run: | uv sync --locked --no-dev - uv run --no-sync python .github/scripts/infra_images.py write "$RUNNER_TEMP/published_images.json" + computers=() + if [ -f "$RUNNER_TEMP/computers/computers.json" ]; then + computers=(--computers "$RUNNER_TEMP/computers/computers.json") + fi + uv run --no-sync python .github/scripts/infra_images.py write "$RUNNER_TEMP/published_images.json" \ + "${computers[@]}" echo "record=$(jq -c . "$RUNNER_TEMP/published_images.json")" >> "$GITHUB_OUTPUT" - name: Warn about images an anonymous pull doesn't get - run: uv run --no-sync python .github/scripts/infra_images.py public + run: | + uv run --no-sync python .github/scripts/infra_images.py public \ + --record "$RUNNER_TEMP/published_images.json" # A release's wheel names only images the registry serves under their tags. # The explorer's built UI, which the framework job ships in the wheel: built here, so the job publishing to PyPI runs @@ -513,6 +637,10 @@ jobs: if [ -n "$RECORD" ]; then header+=$'\n\n'"Images published for this release, by digest:"$'\n' header+=$(jq -r '.images[] | "- `\(.repository):\(.tag)@\(.digest)`"' <<< "$RECORD") + if jq -e '(.computers // []) | length > 0' <<< "$RECORD" >/dev/null; then + header+=$'\n\n'"Computer apps published composed, which a compose of the same inputs pulls:"$'\n' + header+=$(jq -r '.computers[] | "- \(.env | split("/") | last): `\(.index)`"' <<< "$RECORD") + fi fi args=(--verify-tag --title "$tag" --generate-notes --notes "$header") # From the newest earlier tag with a release, not the previous tag: a tag main's CI kept off PyPI has no diff --git a/AGENTS.md b/AGENTS.md index aa27493f9..f06c6c66f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -486,6 +486,12 @@ to PyPI. An agent's `requirements.txt`, and that env's, keeps its protocol floor only when its folder changes (raise the floor when it needs a newer protocol), once the protocol is on PyPI, and a failed agent image job keeps `agentenv-framework` off it. The wheel and the release notes record each image's digest. Run by hand on `main`, the workflow publishes the images alone. +A release also publishes `chromium` composed (`.github/scripts/release_computers.py`), a new image every release since +its venv pins the protocol the release ships: once the protocol and the base computers are published, it's composed +from the release's own wheels on a runner of each platform, probed, and joined as one index in +`ghcr.io/scaleapi/agentenv-computers` by the digests the probe booted; the wheel records each arch's key, image and +lock (`published_images.released_computers`), so an install's compose of the same inputs pulls it rather than +resolving and building. A failure there ships the release without it, and an install composes it as before. Its jobs that run agent-env (`.github/scripts/infra_images.py`) hold no token that writes; the jobs that push images or publish to PyPI install nothing and check what they're handed (`.github/scripts/infra_build.py`, standard library and the docker CLI only). diff --git a/src/agent_env/browsers/README.md b/src/agent_env/browsers/README.md index 20d59398d..91cf6e182 100644 --- a/src/agent_env/browsers/README.md +++ b/src/agent_env/browsers/README.md @@ -20,7 +20,10 @@ alone only; `locale`; `time_zone`; and (`apps = [{ env = "chromium", settings = { start_urls = ["https://example.test/"] } }]`), and a deploy overrides any of them (`env_params.app_settings = {chromium = {start_urls = [...]}}`). Its data is its profile, `~/.config/chromium`, which the image has none of, so each deploy and each reset starts with a clean one. Name it from another bundle as `chromium`, -the alias `aliases.toml` gives it, or by its id, `@local/agentenv-framework/browsers/chromium`. A multi env names it as +the alias `aliases.toml` gives it, or by its id, `@local/agentenv-framework/browsers/chromium`. Each release publishes +it composed, so a compose of it from a released wheel, on any machine, pulls that image by digest from +`ghcr.io/scaleapi/agentenv-computers` rather than resolving its packages and building it (its record says +`published`); a changed `chromium`, a checkout or another base composes it as before. A multi env names it as its website browser with `browser_envs = ["chromium"]`, in place of the run's `playwright-mcp`: it then deploys on a sandbox of its own beside the multi env's computers, sized by the deploy step's `child_envs.chromium`, its tools served under `chromium_`, routing the multi env's synthetic sites through the gateway, its network its own env's, as any diff --git a/src/agent_env/computers/README.md b/src/agent_env/computers/README.md index 7ff759198..e40ff7bcd 100644 --- a/src/agent_env/computers/README.md +++ b/src/agent_env/computers/README.md @@ -89,6 +89,14 @@ records named it, isn't deployed: a deploy that boots its form composes it again a put do, and its record says `by another composer` until then. So upgrading agent-env never boots a computer composed the old way. +A computer app a release publishes composed, `chromium`, needs no build: the wheel records the lock each arch was +composed from and the key it took (`published_images.released_computers`), so a compose of the same inputs takes that +lock rather than resolving one, finds its key in the record, and records the image the release published, pulled by +its digest from `ghcr.io/scaleapi/agentenv-computers`, as `reused` and `published`, then probes it as any reuse is +probed. The computer is written over the index the release published of its arches; an arch it composed here beside +one the release published is joined in the env's own store, the released one copied there first. A record of another +key, an image the registry won't give, or a checkout's compose, which stages the protocol's source, composes as before. + Its settings reach the computer as environment variables, `AGENTENV_SETTING_`, read as it starts: | Setting | Default | What it does | diff --git a/src/agent_env/env/envs/computer/utils/composer.py b/src/agent_env/env/envs/computer/utils/composer.py index f3863976e..4be257ab5 100644 --- a/src/agent_env/env/envs/computer/utils/composer.py +++ b/src/agent_env/env/envs/computer/utils/composer.py @@ -1,9 +1,11 @@ """The composer: a computer env's base, driver and apps built into one image for each form and arch, on a builder the put's sandbox provider gives (``builder``), tagged in the env's image store by a key of everything that goes into it, so -an image of that key, built for any env and on any builder, is reused. A disk form is made from its arch's container -once that has built: its root filesystem as the disk's, on the base disk's EFI system partition, by a build that runs -only on the builder's own platform. A VM bundle, macOS's, is composed on a Mac: its base's booted, given each layer -over ssh and kept on that Mac by a name of its key (``vm_bundle``). +an image of that key, built for any env and on any builder, is reused. An app agent-env's release published composed, +chromium, is pulled from where it published it instead: its wheel records the lock it composed from and the key it +took, so a build of the same inputs finds it, without resolving or building (``_released``). A disk form is made from +its arch's container once that has built: its root filesystem as the disk's, on the base disk's EFI system partition, +by a build that runs only on the builder's own platform. A VM bundle, macOS's, is composed on a Mac: its base's +booted, given each layer over ssh and kept on that Mac by a name of its key (``vm_bundle``). Each result is recorded in the env's metadata as it finishes (``composed``), and the arches that built are written as the env's computer, a ``computer`` artifact over an index of their images.""" @@ -52,7 +54,8 @@ ) from agent_env.env.envs.computer.utils.composed import BUILT, REPOSITORY, Record, failed, mirror_tag, now, stale from agent_env.env.envs.computer.utils.install import Install, for_os, platform_of -from agent_env.env.published_images import DISK, DISK_TOOLS_IMAGE, published_disk, published_pinned +from agent_env.env.published_images import ( + DISK, DISK_TOOLS_IMAGE, published_disk, published_pinned, released_image, released_index, released_lock) from agent_env.providers.sandbox_providers.host_images import platform_tag from agent_env.store.ids import derive_id from agent_env.store.image_store.image_contents import image_config, labels @@ -633,6 +636,17 @@ def on() -> builder.Builder: return held[0] inputs = packages.inputs_digest(wants, context, base_identity, arch) + settings, context_digest = _build_settings(env, base), _digest(context) + + def keyed(lock: packages.Lock | None) -> tuple[str, str]: + dockerfile = _dockerfile(source, layers, base, settings, lock) + return dockerfile, _key(base=base_identity, dockerfile=dockerfile, context=context_digest, + lock=lock.digest() if lock is not None else None, form=form, arch=arch, + format=FORMAT) + + if inputs is not None and (published := _released(inputs, keyed, arch, pair, progress)) is not None: + attempt.key = published["key"] + return published lock = packages.stored(inputs, env.stored_id) if inputs is not None else None if inputs is not None and lock is None: attempt.stage = "resolve" @@ -641,9 +655,8 @@ def on() -> builder.Builder: lock, fetched = packages.resolve(inputs, wants, resolving, source, arch, env.stored_id) progress.at(pair, "resolved", f"resolved {len(fetched)} files and stored them") attempt.stage = "build" - dockerfile = _dockerfile(source, layers, base, _build_settings(env, base), lock) - key = attempt.key = _key(base=base_identity, dockerfile=dockerfile, context=_digest(context), - lock=lock.digest() if lock is not None else None, form=form, arch=arch, format=FORMAT) + dockerfile, key = keyed(lock) + attempt.key = key store, ref, pinned = _stored(env, key) if pinned is not None: progress.at(pair, "reusing", "reusing the image of the same key, built earlier") @@ -792,6 +805,31 @@ def _stored(env: ComputerEnv, key: str) -> tuple[ImageStore, str, str | None]: return store, ref, pin_if_served(ref, store.auth(ref)) +def _released(inputs: str, keyed: Callable[[packages.Lock], tuple[str, str]], arch: str, pair: str, + progress: _Progress) -> dict | None: + """The image agent-env's release published of a build of ``inputs``, which it resolved to the lock its wheel + records: found by the key ``keyed`` gives a build under that lock, so it's what this build would make, and recorded + there by its digest, which a sandbox pulls anonymously. None when the release published none, a build here differs, + or the registry won't give it, so it's composed as before.""" + if (shipped := released_lock(inputs)) is None: + return None + _, key = keyed(packages.Lock.model_validate(shipped)) + if (image := released_image(key)) is None: + logger.info(f"{pair}: agent-env's release resolved these packages, but a build here differs from its build, " + "so it's composed") + return None + try: + kept = labels(image, arch, None) + except (ValueError, OSError) as e: + logger.warning(f"{pair}: agent-env's release published it as {image}, which can't be pulled, so it's composed: " + f"{e}") + return None + progress.at(pair, "reusing", f"reusing the image agent-env's release published, {image}") + return {"status": "unverified", "key": key, "image": image, "reused": True, "published": True, + "emulated": kept.get(EMULATED_LABEL) == "true", "lock": inputs, + "installed": json.loads(kept.get(INSTALLED_LABEL, "{}")), "built_at": now()} + + def _image_store(env: ComputerEnv) -> ImageStore: """The env's image store, with the repository its computers' images are kept in.""" store = get_config().get_image_store_for(env.stored_id) @@ -1128,8 +1166,18 @@ def _written(env: ComputerEnv, base: ComputerArtifact, arches: list[str], config def _joined(env: ComputerEnv, name: str, what: str, images: dict[str, str], arches: list[str]) -> DockerImageArtifact: - """``images`` of ``arches`` joined into one index in the env's image store, signed in to it, as an image artifact.""" + """``images`` of ``arches`` joined into one index in the env's image store, signed in to it, as an image artifact: + the index agent-env's release published when those are the images it published, and an image of it among others + copied into the store first, since an index names images in its own repository.""" + if (released := released_index({arch: images[arch] for arch in arches})) is not None: + try: + return DockerImageArtifact.put_ref(derive_id(env.stored_id, name), + description=f"{env.stored_id}'s {what}", image_name=released) + except (ValueError, OSError) as e: + logger.warning(f"{env.stored_id}: agent-env's release published its index as {released}, which can't be " + f"read, so one is joined in its store: {e}") store = get_config().get_image_store_for(env.stored_id) + images = {arch: _in_store(store, images[arch]) for arch in arches} index = store.image_ref(REPOSITORY, "index-" + hashlib.sha256( ",".join(images[arch] for arch in arches).encode()).hexdigest()) write_index(index, {f"linux/{arch}": images[arch] for arch in arches}, store.auth(index)) @@ -1137,6 +1185,24 @@ def _joined(env: ComputerEnv, name: str, what: str, images: dict[str, str], arch image_name=index) +def _in_store(store: ImageStore, image: str) -> str: + """``image``, pinned, as the store holds it: itself when it's in the store's repository, else copied there under a + tag of its digest, which the copy keeps.""" + digest = image.rpartition("@")[2] + ref = store.image_ref(REPOSITORY, f"released-{digest.removeprefix('sha256:')}") + if _repository(image) == _repository(ref): + return image + if pin_if_served(ref, store.auth(ref)) is None: + copy_image(image, ref, None, store.auth(ref)) + return f"{ref}@{digest}" + + +def _repository(ref: str) -> str: + """``ref``'s registry and repository, without its tag or digest; a registry's port isn't a tag.""" + name = ref.partition("@")[0] + return name[:name.rfind(":")] if name.rfind(":") > name.rfind("/") else name + + def _digest(folder: Path) -> str: """A digest of the files under ``folder``, by their paths, bytes and whether each is executable, which the composer alone decides (umask never sets it).""" diff --git a/src/agent_env/env/published_images.py b/src/agent_env/env/published_images.py index e65744d74..d5388df8b 100644 --- a/src/agent_env/env/published_images.py +++ b/src/agent_env/env/published_images.py @@ -2,7 +2,8 @@ disk forms, for linux/amd64 and linux/arm64: one per Dockerfile, built from the Dockerfile's folder. Each is tagged by its build context's content, and an image built over other published images by theirs too, so an install holding the same sources finds the image built from them; a release wheel also records the digest its release published under each -tag.""" +tag. A release also publishes the computer apps it ships composed, chromium's, in ``COMPUTERS_REPOSITORY``, and its +wheel records each pair's key, image and lock (``released_computers``), so a compose of the same inputs pulls it.""" from __future__ import annotations @@ -11,7 +12,7 @@ import logging import re import tomllib -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from dataclasses import dataclass from pathlib import Path @@ -21,9 +22,12 @@ logger = logging.getLogger(__name__) REGISTRY = "ghcr.io/scaleapi" +#: Where a release publishes the computer apps it ships, composed: one index each, a platform per arch. +COMPUTERS_REPOSITORY = f"{REGISTRY}/agentenv-computers" # The release workflow writes this into the wheels it publishes; a checkout, or a wheel built elsewhere, has none. RELEASE_FILE = Path(__file__).with_name("published_images.json") _DIGEST = re.compile(r"^sha256:[a-f0-9]{64}$") +_PINNED = re.compile(r"^[a-z0-9./_-]+(:[A-Za-z0-9._-]+)?@sha256:[a-f0-9]{64}$") # Short, so a network that drops requests doesn't hold up a run's checks for long. _LOOKUP_TIMEOUT_SECONDS = 10 _SERVED: dict[str, str] = {} # the digest the registry served each ref under, which never changes: no tag is moved @@ -153,10 +157,54 @@ def release_digests(path: Path = RELEASE_FILE) -> dict[str, str]: return digests -def write_release_digests(digests: Mapping[str, str], path: Path = RELEASE_FILE) -> None: - """Record ``digests`` (``repository:tag`` to digest) as ``release_digests`` reads them.""" +def write_release_digests(digests: Mapping[str, str], path: Path = RELEASE_FILE, + computers: Sequence[dict] = ()) -> None: + """Record ``digests`` (``repository:tag`` to digest) as ``release_digests`` reads them, and ``computers`` as + ``released_computers`` reads them.""" images = [] for ref, digest in sorted(digests.items()): repository, _, tag = ref.rpartition(":") images.append({"repository": repository, "tag": tag, "digest": digest}) - path.write_text(json.dumps({"images": images}, indent=2) + "\n") + path.write_text(json.dumps({"images": images, **({"computers": list(computers)} if computers else {})}, + indent=2) + "\n") + + +def released_computers(path: Path | None = None) -> list[dict]: + """The computer apps the release published composed, as its wheel records them: each one's installed env id, its + form, its index in ``COMPUTERS_REPOSITORY``, and for each arch, the key it was composed under, its image there by + digest and the lock it was composed from. Empty when the install records none. Raises ValueError when the record of + them isn't such a list. ``path`` is the record, by default the one the install carries.""" + path = path or RELEASE_FILE + if not path.is_file(): + return [] + try: + computers = json.loads(path.read_text()).get("computers", []) + for computer in computers: + if not (isinstance(computer["env"], str) and computer["form"] == "container" + and _PINNED.match(computer["index"]) and computer["arches"]): + raise ValueError(f"{computer!r} isn't a composed computer's env, form, index and arches") + for arch, built in computer["arches"].items(): + if not (_DIGEST.match(built["key"]) and _PINNED.match(built["image"]) + and isinstance(built["lock"].get("inputs"), str) and built["lock"].get("arch") == arch): + raise ValueError(f"{computer['env']}'s {arch} isn't a key, an image by digest and its lock") + except (ValueError, KeyError, TypeError, AttributeError) as e: + raise ValueError(f"{path} isn't a record of released computers: {type(e).__name__}: {e}") from e + return computers + + +def released_lock(inputs: str, path: Path | None = None) -> dict | None: + """The lock the release composed a computer app's pair from when it was resolved from ``inputs``, or None.""" + return next((built["lock"] for computer in released_computers(path) for built in computer["arches"].values() + if built["lock"]["inputs"] == inputs), None) + + +def released_image(key: str, path: Path | None = None) -> str | None: + """The image, by digest, the release published composed under ``key``, or None.""" + return next((built["image"] for computer in released_computers(path) for built in computer["arches"].values() + if built["key"] == key), None) + + +def released_index(images: Mapping[str, str], path: Path | None = None) -> str | None: + """The index the release published of exactly ``images``, by arch, or None.""" + return next((computer["index"] for computer in released_computers(path) + if {arch: built["image"] for arch, built in computer["arches"].items()} == dict(images)), None) diff --git a/tst/integration/computer/browser_computer_test.py b/tst/integration/computer/browser_computer_test.py index cdc9973c0..4d2665020 100644 --- a/tst/integration/computer/browser_computer_test.py +++ b/tst/integration/computer/browser_computer_test.py @@ -74,12 +74,21 @@ from agent_env.a2a_agent import conversation_store from agent_env.artifact import Artifact from agent_env.artifact.artifacts.file import FileArtifact -from agent_env.config import get_config +from agent_env.artifact.store import reset_artifact_store +from agent_env.bundle import parse_bundle +from agent_env.bundle.installed import installed_bundles +from agent_env.bundle.materialize import materialize +from agent_env.bundle.plan import plan_bundle +from agent_env.bundle.resolve import resolve_bundle +from agent_env.config import configure, get_config from agent_env.env import Env from agent_env.env.env import DeployedEnv -from agent_env.env.envs.computer.utils import inside +from agent_env.env import published_images +from agent_env.env.envs.computer.utils import inside, packages +from agent_env.env.envs.computer.utils.composer import Composing from agent_env.env.store import get_env_store from agent_env.providers.sandbox_providers.local_docker.provider import LocalSandboxProvider +from agent_env.store.image_store.registry_api import copy_image, pin_digest, write_index from agent_env.store.object_store.local.grant_server import grant_server from agent_env.store.object_store.local.tls import LocalCA, local_ca from agent_env.store.routing import namespace_routing @@ -1723,3 +1732,62 @@ async def _browser_op(browser: DeployedEnv, op: str, params: dict) -> dict: assert find_extension_method(app, BROWSER, op) is not None, op return await invoke_extension(browser.environment_url, app, BROWSER, params, 60, method=op, headers=browser.control_headers) + + +#: Where the local registry stands in for the repository a release publishes composed computers in. +RELEASED = "localhost:5000/agentenv-released" + + +@pytest.mark.asyncio +async def test_a_compose_of_chromium_from_a_release_record_pulls_its_image_and_the_browser_boots_from_it( + browser_computer, tmp_path): + """A release publishes chromium composed, and its wheel records each arch's lock, key and image, here in the local + registry: chromium composed again in fresh stores, with that record as the install's, takes the image by digest + rather than resolving its packages or building it, and a deploy of it boots that image.""" + with namespace_routing(): + pairs = {arch: built for arch, built in browser_computer.metadata["composed"]["container"].items() + if built.get("status") in ("ready", "unverified")} + locks = {arch: json.loads(packages.stored(built["lock"], browser_computer.stored_id).model_dump_json()) + for arch, built in pairs.items()} + released = {} + for arch, built in pairs.items(): + copy_image(built["image"], f"{RELEASED}:{arch}", None, None) + released[arch] = f"{RELEASED}@{built['image'].rpartition('@')[2]}" + write_index(f"{RELEASED}:index", {f"linux/{arch}": image for arch, image in released.items()}, None) + record = tmp_path / "published_images.json" + published_images.write_release_digests({}, record, [{ + "env": browser_computer.id, "form": "container", "index": pin_digest(f"{RELEASED}:index", None), + "arches": {arch: {"key": pairs[arch]["key"], "image": released[arch], "lock": locks[arch]} for arch in pairs}}]) + (tmp_path / "bundle" / "tasks").mkdir(parents=True) + (tmp_path / "bundle" / "tasks" / "chromium.json").write_text( + json.dumps([{"id": "env", "type": "deploy_env", "env_id": "chromium"}])) + told = [] + with pytest.MonkeyPatch.context() as patch: + patch.setattr(published_images, "RELEASE_FILE", record) + patch.setenv("XDG_STATE_HOME", str(tmp_path / "state")) + configure() + reset_artifact_store() + try: + with namespace_routing(): + materialize(plan_bundle(resolve_bundle(parse_bundle(tmp_path / "bundle")), installed=installed_bundles, + default_agents=False), composing=Composing(on_progress=told.append)) + composed_now = Env.get(browser_computer.id).metadata["composed"]["container"] + deployed = await Env.get(browser_computer.id).deploy() + context = TaskStepContext(deployed_envs=[deployed]) + try: + booted = deployed.metadata["computer"]["image"] + tabs = (await _record(deployed))["tabs"] + finally: + report = await teardown_run(context) + finally: + reset_artifact_store() + configure() + reset_artifact_store() + + print(json.dumps({"told": told, "composed": {arch: {k: v for k, v in built.items() if k != "probe"} + for arch, built in composed_now.items()}}, indent=1, default=str)) + assert {arch: (built["image"], built["published"], built["reused"]) for arch, built in composed_now.items() + if arch in released} == {arch: (image, True, True) for arch, image in released.items()} + assert composed_now[ARCH]["status"] == "ready" + assert not [line for line in told if "resolving" in line or "building the image" in line], told + assert booted == released[ARCH] and tabs and not report.failed diff --git a/tst/unit/env/envs/test_computer_env.py b/tst/unit/env/envs/test_computer_env.py index 210b86240..0c8055f34 100644 --- a/tst/unit/env/envs/test_computer_env.py +++ b/tst/unit/env/envs/test_computer_env.py @@ -30,6 +30,7 @@ from agentenv_protocol.website_browser import BROWSER from click.testing import CliRunner +from agent_env.artifact.artifacts import docker_image from agent_env.artifact.artifacts.computer import ComputerArtifact from agent_env.artifact.artifacts.docker_image import DockerImageArtifact from agent_env.artifact.artifacts.environment import EnvironmentArtifact @@ -41,11 +42,13 @@ from agent_env.config import get_config from agent_env.env import Env from agent_env.env import env as env_module +from agent_env.env import published_images from agent_env.env.env import DeployedEnv, EnvCapabilityUnsupported from agent_env.env.envs.computer.utils import (boot, builder, composed, composer, gate, inside, packages, params, probing, resolve_in_base) from agent_env.env.envs.computer.utils import sites as computer_sites from agent_env.env.envs.computer.utils.composer import Composing +from agent_env.store.image_store import registry_api from agent_env.store.image_store.registry_api import RegistryRefusedError from agent_env.env.envs.computer.utils.probing import _probe from agent_env.env.envs.computer.env import COMPUTER_TOOLS, ComputerEnv @@ -821,6 +824,109 @@ def refused(ref): assert f"the computer image mirror {MIRROR} couldn't give" in caplog.text +RELEASED = "ghcr.io/scaleapi/agentenv-computers" + + +def _released(env, docker, tmp_path, monkeypatch, arches=("amd64", "arm64"), key=None, digest=None): + """Record ``env``'s composed ``arches`` as a release wheel records the computers it published, each image in the + released repository by the digest it was composed under (``digest`` in its place when given), with its key + (``key``) and the lock it was composed from, joined as one index the registry serves; and take that record as the + install's. Each arch's released image, and the index.""" + pairs = Env.get(env.id).metadata["composed"]["container"] + record = {arch: {"key": key or pairs[arch]["key"], + "image": f"{RELEASED}@{digest or pairs[arch]['image'].rpartition('@')[2]}", + "lock": json.loads(packages.stored(pairs[arch]["lock"], env.id).model_dump_json())} + for arch in arches} + index = f"{RELEASED}:index-{'a' * 64}@" + docker.registry.index(**{ + arch: pairs[arch]["image"].rpartition("@")[2] for arch in arches}) + path = tmp_path / "published_images.json" + path.write_text(json.dumps({"images": [], "computers": [ + {"env": env.id, "form": "container", "index": index, "arches": record}]})) + monkeypatch.setattr(published_images, "RELEASE_FILE", path) + serving = docker.registry.pin_digest + monkeypatch.setattr(docker_image, "pin_digest", lambda ref, auth: ref if ref.startswith(RELEASED) and "@" in ref + else serving(ref, auth)) + return {arch: built["image"] for arch, built in record.items()}, index + + +def test_a_compose_of_what_the_release_published_pulls_its_image_without_resolving_or_building(base, driver, docker, + tmp_path, monkeypatch): + released, index = _released(_put(base, driver), docker, tmp_path, monkeypatch) + builds, resolved, told = len(docker.builds), len(docker.resolved), [] + + second = _put(base, driver, id="@local/~/kit/other", environment_name="other", + composing=Composing(on_progress=told.append)) + + pairs = Env.get(second.id).metadata["composed"]["container"] + assert len(docker.builds) == builds and len(docker.resolved) == resolved + assert {arch: (r["image"], r["published"], r["reused"], r["status"]) for arch, r in pairs.items()} == { + arch: (image, True, True, "ready" if arch == "arm64" else "unverified") for arch, image in released.items()} + assert f"container/arm64: reusing the image agent-env's release published, {released['arm64']}" in told + assert not [line for line in told if "resolving" in line or "building" in line] + assert ComputerArtifact.get(f"{second.id}__computer").image().image_name == index + + +def test_a_release_record_of_another_key_or_an_image_the_registry_wont_give_leaves_the_compose_as_before( + base, driver, docker, tmp_path, monkeypatch, caplog): + first = _put(base, driver) + _released(first, docker, tmp_path, monkeypatch, key="sha256:" + "c" * 64) + other = _put(base, driver, id="@local/~/kit/other", environment_name="other") + unserved, _ = _released(first, docker, tmp_path, monkeypatch, digest="sha256:" + "d" * 64) + + with caplog.at_level(logging.WARNING, logger=composer.__name__): + third = _put(base, driver, id="@local/~/kit/third", environment_name="third") + + for env in (other, third): + pairs = Env.get(env.id).metadata["composed"]["container"] + assert all(r["reused"] and not r.get("published") for r in pairs.values()) + assert f"agent-env's release published it as {unserved['arm64']}, which can't be pulled" in caplog.text + + +def test_a_released_index_the_registry_wont_give_is_joined_in_the_store_instead(base, driver, docker, tmp_path, + monkeypatch, caplog): + released, index = _released(_put(base, driver), docker, tmp_path, monkeypatch) + serving = docker.registry.pin_digest + + def pin(ref, auth): + if ref == index: + raise registry_api.NoSuchImageError(f"{ref}: ghcr.io has no such image") + return ref if ref.startswith(RELEASED) and "@" in ref else serving(ref, auth) + + monkeypatch.setattr(docker_image, "pin_digest", pin) + monkeypatch.setattr(composer, "copy_image", lambda source, target, source_auth, target_auth: ( + docker.registry.tags.__setitem__(target, source.rpartition("@")[2]))) + + with caplog.at_level(logging.WARNING, logger=composer.__name__): + second = _put(base, driver, id="@local/~/kit/other", environment_name="other") + + pairs = Env.get(second.id).metadata["composed"]["container"] + assert all(pairs[arch]["image"] == image and pairs[arch]["published"] for arch, image in released.items()) + assert ComputerArtifact.get(f"{second.id}__computer").image().image_name.startswith( + f"localhost:5000/{composed.REPOSITORY}:index-") + assert f"agent-env's release published its index as {index}, which can't be read" in caplog.text + + +def test_an_arch_the_release_published_beside_one_composed_here_joins_an_index_in_the_store(base, driver, docker, + tmp_path, monkeypatch): + released, _ = _released(_put(base, driver), docker, tmp_path, monkeypatch, arches=("arm64",)) + copied, joined = [], {} + monkeypatch.setattr(composer, "copy_image", lambda source, target, source_auth, target_auth: ( + copied.append((source, target)), docker.registry.tags.__setitem__(target, source.rpartition("@")[2]))) + monkeypatch.setattr(composer, "write_index", lambda index, images, auth: ( + joined.update({index: images}), docker.joined(index, images, auth))) + + second = _put(base, driver, id="@local/~/kit/other", environment_name="other") + + pairs = Env.get(second.id).metadata["composed"]["container"] + digest = released["arm64"].rpartition("@")[2] + in_store = f"localhost:5000/{composed.REPOSITORY}:released-{digest.removeprefix('sha256:')}" + assert pairs["arm64"]["image"] == released["arm64"] and pairs["arm64"]["published"] + assert not pairs["amd64"].get("published") and copied == [(released["arm64"], in_store)] + index = ComputerArtifact.get(f"{second.id}__computer").image().image_name + assert joined == {index.partition("@")[0]: {"linux/amd64": pairs["amd64"]["image"], + "linux/arm64": f"{in_store}@{digest}"}} + + def test_builds_of_one_env_at_once_keep_each_others_pairs_and_a_failure_doesnt_replace_a_built_one(base, driver, docker, monkeypatch): diff --git a/tst/unit/env/published_images_test.py b/tst/unit/env/published_images_test.py index 3b36b9484..8aa7c20d2 100644 --- a/tst/unit/env/published_images_test.py +++ b/tst/unit/env/published_images_test.py @@ -24,6 +24,10 @@ published_digest, published_disk, release_digests, + released_computers, + released_image, + released_index, + released_lock, write_release_digests, ) from agent_env.store.image_store.registry_api import NoSuchImageError @@ -198,3 +202,41 @@ def test_a_disk_forms_tag_is_of_its_boot_recipe_its_bases_sources_and_the_disk_t assert published_disk(sources) is None served[disk.ref()] = _DIGEST assert published_disk(sources) == f"{disk.ref()}@{_DIGEST}" and published_disk("sha256:" + "0" * 64) is None + + +def _computer(arch: str, digest: str) -> dict: + return {"key": "sha256:" + digest[-1] * 64, "image": f"{published_images.COMPUTERS_REPOSITORY}@{digest}", + "lock": {"inputs": "sha256:" + digest[-1] * 63 + "f", "arch": arch, "apt": [], "wheels": {}, "downloads": {}}} + + +def test_the_record_of_released_computers_finds_each_lock_image_and_index_and_an_install_without_one_finds_none( + tmp_path): + arches = {"amd64": _computer("amd64", "sha256:" + "1" * 64), "arm64": _computer("arm64", "sha256:" + "2" * 64)} + index = f"{published_images.COMPUTERS_REPOSITORY}:index-{'a' * 64}@sha256:{'b' * 64}" + path = tmp_path / "published_images.json" + write_release_digests({}, path, [{"env": "@local/agentenv-framework/browsers/chromium", "form": "container", + "index": index, "arches": arches}]) + + assert released_lock(arches["arm64"]["lock"]["inputs"], path) == arches["arm64"]["lock"] + assert released_image(arches["amd64"]["key"], path) == arches["amd64"]["image"] + assert released_index({arch: built["image"] for arch, built in arches.items()}, path) == index + assert released_index({"amd64": arches["amd64"]["image"]}, path) is None + assert released_lock("sha256:" + "0" * 64, path) is None and released_image("sha256:" + "0" * 64, path) is None + assert release_digests(path) == {} + assert released_computers(tmp_path / "none.json") == [] + + +@pytest.mark.parametrize("computers", [ + [{"env": "chromium", "form": "disk", "index": "x", "arches": {}}], + [{"env": "chromium", "form": "container", "index": f"ghcr.io/scaleapi/agentenv-computers:i@sha256:{'b' * 64}", + "arches": {"amd64": {**_computer("amd64", "sha256:" + "1" * 64), "image": "ghcr.io/x:latest"}}}], + [{"env": "chromium", "form": "container", "index": f"ghcr.io/scaleapi/agentenv-computers:i@sha256:{'b' * 64}", + "arches": {"arm64": _computer("amd64", "sha256:" + "1" * 64)}}], + "not a list", +], ids=["not-a-container", "an-image-by-tag", "another-arch-lock", "not-a-list"]) +def test_a_record_of_released_computers_that_isnt_one_is_refused(tmp_path, computers): + path = tmp_path / "published_images.json" + path.write_text(json.dumps({"images": [], "computers": computers})) + + with pytest.raises(ValueError, match="isn't a record of released computers"): + released_computers(path) diff --git a/tst/unit/test_infra_build_script.py b/tst/unit/test_infra_build_script.py index 359084bb8..85cebb680 100644 --- a/tst/unit/test_infra_build_script.py +++ b/tst/unit/test_infra_build_script.py @@ -6,6 +6,7 @@ import ast import base64 import configparser +import hashlib import json import subprocess import sys @@ -14,7 +15,8 @@ import pytest from agent_env.env import bootstrap -from agent_env.env.published_images import DISK_IMAGES, REGISTRY, write_release_digests +from agent_env.env.published_images import ( + COMPUTERS_REPOSITORY, DISK_IMAGES, REGISTRY, released_computers, write_release_digests) from tst.util.scripts import SCRIPTS, load_script REPO = SCRIPTS.parents[1] @@ -287,6 +289,140 @@ def test_verify_record_refuses_an_image_recorded_twice(script, monkeypatch, tmp_ assert f"records {record['images'][0]['repository'].split('/')[-1]} more than once" in capsys.readouterr().out +CHROMIUM = "@local/agentenv-framework/browsers/chromium" +_AMD64, _ARM64, _ARM64_COMPOSED, _INDEX = ("sha256:" + c * 64 for c in "1234") +_LOCAL = "localhost:5000/agentenv-computers" +_IMAGES = {"amd64": f"{COMPUTERS_REPOSITORY}@{_AMD64}", "arm64": f"{COMPUTERS_REPOSITORY}@{_ARM64}"} +_COMPUTER_INDEX = f"{COMPUTERS_REPOSITORY}:index-" + hashlib.sha256( + ",".join(_IMAGES[arch] for arch in ("amd64", "arm64")).encode()).hexdigest() + + +def _lock(arch: str) -> dict: + return {"inputs": "sha256:" + "c" * 64, "arch": arch, "apt": [], "wheels": {}, "downloads": {}} + + +def _composed(folder: Path, changes: dict | None = None, arches=("amd64", "arm64")) -> Path: + """What release_computers.py compose writes for each arch: chromium in the local registry under its key, amd64's a + manifest, arm64's an index of its platform's manifest and an attestation, with ``changes`` by arch.""" + for arch in arches: + key, digest = ("a" if arch == "amd64" else "b") * 64, _AMD64 if arch == "amd64" else _ARM64_COMPOSED + entry = {"env": CHROMIUM, "form": "container", "arch": arch, "key": f"sha256:{key}", + "image": f"{_LOCAL}:{key}@{digest}", "lock": _lock(arch)} + (folder / f"{arch}.json").write_text(json.dumps([{**entry, **(changes or {}).get(arch, {})}])) + return folder + + +def _computer_registry(served: dict[str, str], created: list, children: set[str] | None = None): + """Docker as the registries answer: ``served`` as ``_docker`` serves it, the local registry's composed images, and + the agentenv-computers index, served once created, of ``children``, by default the two images it was made of.""" + published = _docker(served) + raws = {f"{_LOCAL}@{_AMD64}": {"mediaType": "application/vnd.oci.image.manifest.v1+json"}, + f"{_LOCAL}@{_ARM64_COMPOSED}": {"mediaType": "application/vnd.oci.image.index.v1+json", "manifests": [ + {"digest": _ARM64, "platform": {"os": "linux", "architecture": "arm64"}}, + {"digest": "sha256:" + "9" * 64, "platform": {"os": "unknown", "architecture": "unknown"}}]}} + + def run(command, **kw): + if command[:5] != ["docker", "buildx", "imagetools", "inspect", _COMPUTER_INDEX] and command[5:] != ["--raw"]: + return published(command, **kw) + ref = command[4] + if ref == _COMPUTER_INDEX and not created: + return subprocess.CompletedProcess(command, 1, "", f"ERROR: {ref}: not found\n") + if command[5:] == ["--raw"]: + raw = raws.get(ref) or (ref == _COMPUTER_INDEX and {"manifests": [ + {"digest": digest} for digest in sorted(children or {_AMD64, _ARM64})]}) + return subprocess.CompletedProcess(command, 0 if raw else 1, json.dumps(raw) if raw else "", "") + return subprocess.CompletedProcess(command, 0, json.dumps({"digest": _INDEX}), "") + return run + + +def test_publish_computers_joins_each_apps_platform_manifests_into_one_index_and_records_it(script, monkeypatch, + tmp_path, workflow): + created = [] + monkeypatch.setattr(script.subprocess, "run", _computer_registry({}, created)) + monkeypatch.setattr(script.subprocess, "Popen", lambda argv, **kw: (created.append(argv), _Process("", 0))[1]) + folder = _composed(tmp_path) + + assert _run(script, monkeypatch, "publish-computers", str(folder)) == 0 + + assert created == [["docker", "buildx", "imagetools", "create", "-t", _COMPUTER_INDEX, + "--annotation", "index:org.opencontainers.image.source=https://github.com/scaleapi/agentenv-framework", + "--annotation", "index:org.opencontainers.image.revision=c0ffee", + f"{_LOCAL}@{_AMD64}", f"{_LOCAL}@{_ARM64}"]] + computers = json.loads((folder / "computers.json").read_text()) + assert computers == [{"env": CHROMIUM, "form": "container", "index": f"{_COMPUTER_INDEX}@{_INDEX}", "arches": { + arch: {"key": f"sha256:{('a' if arch == 'amd64' else 'b') * 64}", "image": _IMAGES[arch], "lock": _lock(arch)} + for arch in ("amd64", "arm64")}}] + record = tmp_path / "published_images.json" + write_release_digests({}, record, computers) + assert released_computers(record) == computers + + +@pytest.mark.parametrize("changes, arches, problem", [ + ({"arm64": {"env": "@local/agentenv-framework/browsers/other"}}, ("amd64", "arm64"), + "isn't an app a release publishes"), + ({"amd64": {"key": "sha256:" + "f" * 64}}, ("amd64", "arm64"), "isn't an app a release publishes"), + ({"arm64": {"lock": _lock("amd64")}}, ("amd64", "arm64"), "isn't an app a release publishes"), + ({"amd64": {"more": 1}}, ("amd64", "arm64"), "isn't an app's env, form, arch, key, image and lock"), + ({}, ("amd64",), "arm64.json isn't a list of composed apps"), +], ids=["another-app", "not-under-its-key", "another-arch-lock", "more", "an-arch-missing"]) +def test_publish_computers_refuses_what_a_compose_didnt_write_and_pushes_nothing(script, monkeypatch, tmp_path, + workflow, capsys, changes, arches, + problem): + calls = [] + monkeypatch.setattr(script.subprocess, "run", _docker({}, calls)) + + assert _run(script, monkeypatch, "publish-computers", str(_composed(tmp_path, changes, arches))) == 1 + + assert problem in capsys.readouterr().out and calls == [] + + +def test_publish_computers_refuses_an_index_that_doesnt_name_exactly_its_images(script, monkeypatch, tmp_path, + workflow, capsys): + created = [] + monkeypatch.setattr(script.subprocess, "run", _computer_registry({}, created, children={_AMD64})) + monkeypatch.setattr(script.subprocess, "Popen", lambda argv, **kw: (created.append(argv), _Process("", 0))[1]) + folder = _composed(tmp_path) + + assert _run(script, monkeypatch, "publish-computers", str(folder)) == 1 + + assert f"and not an index of exactly ['{_AMD64}', '{_ARM64}']" in capsys.readouterr().out + assert not (folder / "computers.json").exists() + + +def _computers_record(tmp_path, arches=None) -> Path: + path = tmp_path / "published_images.json" + write_release_digests({image.ref(): _DIGEST for image in bootstrap.PUBLISHED_IMAGES}, path, [ + {"env": CHROMIUM, "form": "container", "index": f"{_COMPUTER_INDEX}@{_INDEX}", "arches": arches or { + arch: {"key": "sha256:" + "a" * 64, "image": image, "lock": _lock(arch)} for arch, image in _IMAGES.items()}}]) + return path + + +@pytest.mark.parametrize("children, problem", [ + (None, None), + ({_AMD64}, f"{_COMPUTER_INDEX} doesn't name exactly the images recorded for {CHROMIUM}'s arches"), +], ids=["served", "another-index"]) +def test_verify_record_checks_each_composed_computers_index_and_its_images(script, monkeypatch, tmp_path, capsys, + children, problem): + served = {image.ref(): _DIGEST for image in bootstrap.PUBLISHED_IMAGES} + monkeypatch.setattr(script.subprocess, "run", _computer_registry(served, [True], children)) + + code = _run(script, monkeypatch, "verify-record", str(_computers_record(tmp_path))) + + out = capsys.readouterr().out + assert (code, problem in out if problem else "::error::" not in out) == (1 if problem else 0, True), out + + +def test_verify_record_refuses_a_composed_computer_recorded_in_another_repository(script, monkeypatch, tmp_path, + capsys): + monkeypatch.setattr(script.subprocess, "run", _docker({image.ref(): _DIGEST for image in bootstrap.PUBLISHED_IMAGES})) + arches = {arch: {"key": "sha256:" + "a" * 64, "image": f"ghcr.io/someone/thing@{_AMD64}", "lock": _lock(arch)} + for arch in ("amd64", "arm64")} + + assert _run(script, monkeypatch, "verify-record", str(_computers_record(tmp_path, arches))) == 1 + + assert f"{CHROMIUM}'s arches aren't ['amd64', 'arm64'], each a key, an image by digest in" in capsys.readouterr().out + + @pytest.mark.parametrize("name, expected", [ ("agentenv-gateway", ["docker", "run", "--rm", "--entrypoint", "python", "img", "-c", "import gateway.entrypoint"]), ("agentenv-db-mcp", ["docker", "run", "--rm", "img", "--help"]), diff --git a/tst/unit/test_infra_images_script.py b/tst/unit/test_infra_images_script.py index 864e5dacf..041d40e39 100644 --- a/tst/unit/test_infra_images_script.py +++ b/tst/unit/test_infra_images_script.py @@ -9,7 +9,7 @@ import pytest from agent_env.env import bootstrap -from agent_env.env.published_images import BUNDLE_IMAGES, release_digests +from agent_env.env.published_images import BUNDLE_IMAGES, release_digests, released_computers from agent_env.store.image_store.registry_api import NoSuchImageError, RegistryRefusedError from tst.util.scripts import load_script @@ -105,6 +105,22 @@ def test_write_records_the_digest_served_under_every_tag(script, monkeypatch, tm assert release_digests(path) == {image.ref(): _DIGEST for image in bootstrap.PUBLISHED_IMAGES} +def test_write_records_the_composed_computers_it_is_handed_beside_the_images(script, monkeypatch, tmp_path): + _registry(script, monkeypatch, {}) + computers = [{"env": "@local/agentenv-framework/browsers/chromium", "form": "container", + "index": f"ghcr.io/scaleapi/agentenv-computers:index-{'a' * 64}@sha256:{'b' * 64}", "arches": { + arch: {"key": "sha256:" + "c" * 64, "image": f"ghcr.io/scaleapi/agentenv-computers@sha256:{c * 64}", + "lock": {"inputs": "sha256:" + "d" * 64, "arch": arch}} + for arch, c in (("amd64", "1"), ("arm64", "2"))}}] + handed, path = tmp_path / "computers.json", tmp_path / "published_images.json" + handed.write_text(json.dumps(computers)) + + assert _run(script, monkeypatch, "write", str(path), "--computers", str(handed)) == 0 + + assert released_computers(path) == computers + assert release_digests(path) == {image.ref(): _DIGEST for image in bootstrap.PUBLISHED_IMAGES} + + def test_write_records_nothing_when_an_image_is_missing(script, monkeypatch, tmp_path, capsys): db_mcp = next(image for image in bootstrap.PUBLISHED_IMAGES if image.name == "agentenv-db-mcp") _registry(script, monkeypatch, {db_mcp.ref(): NoSuchImageError("no such image")}) From 51a11aa5f5ddcc3470ad1cc19abae9ce9b2bb7d2 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sun, 15 Nov 2026 20:34:20 -0800 Subject: [PATCH 2/2] fix(computers): a release record may name its registry by host and port Co-Authored-By: Claude Opus 5.5 --- src/agent_env/env/published_images.py | 3 ++- tst/unit/env/published_images_test.py | 10 ++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/agent_env/env/published_images.py b/src/agent_env/env/published_images.py index d5388df8b..652f5b2b9 100644 --- a/src/agent_env/env/published_images.py +++ b/src/agent_env/env/published_images.py @@ -27,7 +27,8 @@ # The release workflow writes this into the wheels it publishes; a checkout, or a wheel built elsewhere, has none. RELEASE_FILE = Path(__file__).with_name("published_images.json") _DIGEST = re.compile(r"^sha256:[a-f0-9]{64}$") -_PINNED = re.compile(r"^[a-z0-9./_-]+(:[A-Za-z0-9._-]+)?@sha256:[a-f0-9]{64}$") +# An image by digest, tagged or not, its registry's port allowed. +_PINNED = re.compile(r"^[a-z0-9.-]+(:[0-9]+)?(/[a-z0-9._-]+)+(:[A-Za-z0-9._-]+)?@sha256:[a-f0-9]{64}$") # Short, so a network that drops requests doesn't hold up a run's checks for long. _LOOKUP_TIMEOUT_SECONDS = 10 _SERVED: dict[str, str] = {} # the digest the registry served each ref under, which never changes: no tag is moved diff --git a/tst/unit/env/published_images_test.py b/tst/unit/env/published_images_test.py index 8aa7c20d2..a0e5cee41 100644 --- a/tst/unit/env/published_images_test.py +++ b/tst/unit/env/published_images_test.py @@ -226,6 +226,16 @@ def test_the_record_of_released_computers_finds_each_lock_image_and_index_and_an assert released_computers(tmp_path / "none.json") == [] +def test_a_record_of_released_computers_may_name_a_registry_by_its_port(tmp_path): + repository = "localhost:5001/agentenv-computers" + arches = {"amd64": {**_computer("amd64", "sha256:" + "1" * 64), "image": f"{repository}@sha256:{'1' * 64}"}} + path = tmp_path / "published_images.json" + write_release_digests({}, path, [{"env": "chromium", "form": "container", "arches": arches, + "index": f"{repository}:index-{'a' * 64}@sha256:{'b' * 64}"}]) + + assert released_image(arches["amd64"]["key"], path) == f"{repository}@sha256:{'1' * 64}" + + @pytest.mark.parametrize("computers", [ [{"env": "chromium", "form": "disk", "index": "x", "arches": {}}], [{"env": "chromium", "form": "container", "index": f"ghcr.io/scaleapi/agentenv-computers:i@sha256:{'b' * 64}",