From 935fc03559291e242e784bbf80f33d0b17e4038a Mon Sep 17 00:00:00 2001 From: Andrew Park Date: Mon, 9 Nov 2026 14:49:17 -0800 Subject: [PATCH 1/6] ci: share the slow tier's computer images through a mirror registry rather than per-shard caches The slow tier saved each runner's whole registry to the Actions cache, keyed by shard and week. The same image landed on up to three shards, so the entries (1.6 to 6.3GB each) overflowed the repository's 10GB, evicted each other, and most shards built every image from scratch. - The composer copies an image of a key the env's image store lacks from AGENT_ENV_COMPUTER_IMAGE_MIRROR, read anonymously, before it would build it; a mirror it can't read leaves the image to be built. - registry_api.copy_image copies an image between registries by its digest, sending only the blobs the target lacks and each manifest byte for byte. - After its tests pass on main, each slow shard shares the images it built or copied to ghcr.io/scaleapi/agentenv-ci-computers under their keys and -seen-. - computer-image-mirror.yml deletes, daily, the versions no run has used for 7 days, keeping a kept index's platform manifests. - The test packages folder keeps its own per-shard weekly cache, saved only on main and named by its locks. prune_registry.py, which trimmed the registry cache, goes with that cache. Co-Authored-By: Claude --- .github/scripts/prune_computer_images.py | 112 ++++++++++++++++++ .github/scripts/prune_registry.py | 98 --------------- .github/scripts/share_computer_images.py | 53 +++++++++ .github/workflows/computer-image-mirror.yml | 31 +++++ .github/workflows/local-backends.yml | 96 +++++++-------- AGENTS.md | 5 +- src/agent_env/config/describe.py | 1 + src/agent_env/config/runtime.py | 8 ++ .../env/envs/computer/utils/composed.py | 6 + .../env/envs/computer/utils/composer.py | 19 ++- .../store/image_store/registry_api.py | 77 ++++++++++-- tst/unit/env/envs/test_computer_env.py | 72 +++++++++++ tst/unit/store/registry_api_test.py | 81 +++++++++++++ .../test_computer_image_mirror_scripts.py | 109 +++++++++++++++++ tst/unit/test_prune_registry_script.py | 97 --------------- 15 files changed, 605 insertions(+), 260 deletions(-) create mode 100644 .github/scripts/prune_computer_images.py delete mode 100644 .github/scripts/prune_registry.py create mode 100644 .github/scripts/share_computer_images.py create mode 100644 .github/workflows/computer-image-mirror.yml create mode 100644 tst/unit/test_computer_image_mirror_scripts.py delete mode 100644 tst/unit/test_prune_registry_script.py diff --git a/.github/scripts/prune_computer_images.py b/.github/scripts/prune_computer_images.py new file mode 100644 index 000000000..2cf220edf --- /dev/null +++ b/.github/scripts/prune_computer_images.py @@ -0,0 +1,112 @@ +"""Delete the CI computer image mirror's images no run has used for ``--days`` days. + +``share_computer_images.py`` tags each image a passing main run used ``-seen-YYYYMMDD``. An image is kept while its +newest such tag, or its upload when it has none, is within ``--days`` of now, with every platform manifest a kept index +lists, since GHCR lists those as versions of their own. Every other version is deleted through GitHub's packages API, +read and written with ``GITHUB_TOKEN``, as GHCR is. A version GitHub won't delete warns. ``--dry-run`` prints what +would go and deletes nothing. +""" + +from __future__ import annotations + +import argparse +import base64 +import datetime +import json +import os +import re +import sys +import urllib.error +import urllib.parse +import urllib.request + +SEEN = re.compile(r"^[0-9a-f]{16}-[0-9a-f]{64}-seen-(\d{8})$") +_INDEXES = {"application/vnd.oci.image.index.v1+json", "application/vnd.docker.distribution.manifest.list.v2+json"} +_ACCEPT = ", ".join([*_INDEXES, "application/vnd.oci.image.manifest.v1+json", + "application/vnd.docker.distribution.manifest.v2+json"]) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("org") + parser.add_argument("package", help="the container package, as agentenv-ci-computers") + parser.add_argument("--days", type=int, default=7) + parser.add_argument("--dry-run", action="store_true") + args = parser.parse_args() + + api = f"https://api.github.com/orgs/{args.org}/packages/container/{args.package}/versions" + try: + versions = _versions(api) + except urllib.error.HTTPError as e: + if e.code != 404: + raise + print(f"::warning::prune computer images: no package {args.org}/{args.package}, or this repository has no " + "access to it, so nothing is pruned") + return 0 + cutoff = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=args.days) + stale = stale_versions(versions, cutoff, lambda digest: _children(args.org, args.package, digest)) + failed = 0 + for version in stale: + print(f"{'would delete' if args.dry_run else 'deleting'} {version['name']} " + f"{version['metadata']['container']['tags'] or '(untagged)'}") + if not args.dry_run: + try: + _github("DELETE", f"{api}/{version['id']}") + except urllib.error.HTTPError as e: + failed += 1 + print(f"::warning::couldn't delete {version['name']}: HTTP {e.code} {e.read().decode()[:200]}") + print(f"prune computer images: {len(stale) - failed} of {len(versions)} versions " + f"{'would be ' if args.dry_run else ''}deleted, none used since {cutoff:%Y-%m-%d}") + return 0 + + +def stale_versions(versions: list[dict], cutoff: datetime.datetime, children) -> list[dict]: + """The ``versions`` to delete: neither used nor uploaded since ``cutoff``, nor a platform manifest of one kept. + ``children(digest)`` lists the digests of the manifests the index of that digest lists, or none for an image.""" + kept = {version["name"] for version in versions if _last_used(version) >= cutoff} + for digest in list(kept): + kept.update(children(digest)) + return [version for version in versions if version["name"] not in kept] + + +def _last_used(version: dict) -> datetime.datetime: + created = datetime.datetime.fromisoformat(version["created_at"].replace("Z", "+00:00")) + seen = [datetime.datetime.strptime(m.group(1), "%Y%m%d").replace(tzinfo=datetime.timezone.utc) + for tag in version["metadata"]["container"]["tags"] if (m := SEEN.match(tag))] + return max([created, *(day + datetime.timedelta(days=1) for day in seen)]) + + +def _versions(api: str) -> list[dict]: + versions, page = [], 1 + while batch := _github("GET", f"{api}?per_page=100&page={page}"): + versions += batch + page += 1 + return versions + + +def _children(org: str, package: str, digest: str) -> list[str]: + """The platform manifests the index ``digest`` lists, read from GHCR with ``GITHUB_TOKEN``; none for an image.""" + repository = f"{org}/{package}" + signed_in = base64.b64encode(f"x:{os.environ['GITHUB_TOKEN']}".encode()).decode() + with urllib.request.urlopen(urllib.request.Request(f"https://ghcr.io/token?scope=repository:{repository}:pull", + headers={"Authorization": f"Basic {signed_in}"})) as response: + token = json.load(response)["token"] + request = urllib.request.Request(f"https://ghcr.io/v2/{repository}/manifests/{digest}", + headers={"Authorization": f"Bearer {token}", "Accept": _ACCEPT}) + with urllib.request.urlopen(request) as response: + manifest = json.load(response) + media = response.headers.get("Content-Type", "").split(";")[0].strip() + return [child["digest"] for child in manifest.get("manifests", [])] if media in _INDEXES else [] + + +def _github(method: str, url: str): + request = urllib.request.Request(url, method=method, headers={ + "Authorization": f"Bearer {os.environ['GITHUB_TOKEN']}", "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28"}) + with urllib.request.urlopen(request) as response: + body = response.read() + return json.loads(body) if body else None + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/prune_registry.py b/.github/scripts/prune_registry.py deleted file mode 100644 index 0b2ace7f8..000000000 --- a/.github/scripts/prune_registry.py +++ /dev/null @@ -1,98 +0,0 @@ -"""Keep only the images of a local registry's repository that a run used, before the registry's storage is cached. - -``prune`` drops each tag the registry's log has no manifest request for, then each manifest no remaining tag reaches, -following an image index to the manifests it lists. The registry's ``garbage-collect`` then frees the layers only those -manifests used; with ``--delete-untagged`` it would also delete a kept index's untagged platform manifests. ``links``, -run after it, drops the repository's links to layers it freed. -""" - -from __future__ import annotations - -import argparse -import json -import re -import shutil -import sys -from pathlib import Path - -_V2 = Path("docker") / "registry" / "v2" - - -def main(argv: list[str]) -> int: - parser = argparse.ArgumentParser(description=__doc__) - commands = parser.add_subparsers(dest="command", required=True) - pruning = commands.add_parser("prune", help="drop the tags and manifests the run didn't use") - linking = commands.add_parser("links", help="drop links to layers garbage-collect freed") - for command in (pruning, linking): - command.add_argument("storage", type=Path, help="the registry's storage root") - command.add_argument("repository") - pruning.add_argument("log", type=Path, help="the registry's log of the run") - args = parser.parse_args(argv) - if args.command == "prune": - kept, dropped = prune(args.storage, args.repository, used_tags(args.log.read_text(errors="replace"), - args.repository)) - print(f"kept {kept} tags of {args.repository}, dropped {dropped}") - else: - print(f"dropped {drop_freed_links(args.storage, args.repository)} links to freed layers") - return 0 - - -def used_tags(log: str, repository: str) -> set[str]: - """The tags the log names a request for a manifest of ``repository`` by.""" - return set(re.findall(rf"/v2/{re.escape(repository)}/manifests/([A-Za-z0-9_][A-Za-z0-9_.-]{{0,127}})(?=[\s\"?])", - log)) - - -def prune(storage: Path, repository: str, used: set[str]) -> tuple[int, int]: - """Drop the tags of ``repository`` not in ``used``, then the manifests no remaining tag reaches; how many tags it - kept and dropped.""" - manifests = storage / _V2 / "repositories" / repository / "_manifests" - tags = _children(manifests / "tags") - for tag in tags: - if tag.name not in used: - shutil.rmtree(tag) - kept = [tag for tag in tags if tag.name in used] - reached = _reached(storage, [(tag / "current" / "link").read_text().strip() for tag in kept]) - for revision in _children(manifests / "revisions" / "sha256"): - if f"sha256:{revision.name}" not in reached: - shutil.rmtree(revision) - return len(kept), len(tags) - len(kept) - - -def drop_freed_links(storage: Path, repository: str) -> int: - """Drop each link of ``repository`` to a layer that's no longer stored; how many.""" - dropped = 0 - for layer in _children(storage / _V2 / "repositories" / repository / "_layers" / "sha256"): - if not _blob(storage, f"sha256:{layer.name}").is_file(): - shutil.rmtree(layer) - dropped += 1 - return dropped - - -def _reached(storage: Path, digests: list[str]) -> set[str]: - """``digests`` and every manifest an image index among them lists, recursively.""" - reached, pending = set(), list(digests) - while pending: - digest = pending.pop() - if digest in reached: - continue - reached.add(digest) - try: - manifest = json.loads(_blob(storage, digest).read_bytes()) - except (FileNotFoundError, ValueError): - continue - pending += [listed["digest"] for listed in manifest.get("manifests", []) if "digest" in listed] - return reached - - -def _children(folder: Path) -> list[Path]: - return [child for child in folder.iterdir() if child.is_dir()] if folder.is_dir() else [] - - -def _blob(storage: Path, digest: str) -> Path: - hex_digest = digest.removeprefix("sha256:") - return storage / _V2 / "blobs" / "sha256" / hex_digest[:2] / hex_digest / "data" - - -if __name__ == "__main__": - sys.exit(main(sys.argv[1:])) diff --git a/.github/scripts/share_computer_images.py b/.github/scripts/share_computer_images.py new file mode 100644 index 000000000..17fb93078 --- /dev/null +++ b/.github/scripts/share_computer_images.py @@ -0,0 +1,53 @@ +"""Copy every composed computer image a slow-tier runner's registry holds to the CI computer image mirror, the one +``AGENT_ENV_COMPUTER_IMAGE_MIRROR`` names, so later runs copy them back rather than build them. + +Each image goes under its mirror tag (``composed.mirror_tag``: the composer's digest, then the key), and again with +today's date, ``-seen-YYYYMMDD``, which is how ``prune_computer_images.py`` tells an image a recent run used from one +none has. An image already in the mirror is only tagged again: no blob it holds is sent twice. The mirror is written as +``GITHUB_ACTOR`` with ``GITHUB_TOKEN``; one the composer can't read anonymously, as GHCR makes a new package, warns. +""" + +from __future__ import annotations + +import argparse +import datetime +import json +import os +import re +import sys +import urllib.request + +from agent_env.env.envs.computer.utils.composed import REPOSITORY, mirror_tag +from agent_env.store.image_store.oci_registry_credentials import RegistryAuth +from agent_env.store.image_store.registry_api import copy_image, pin_digest, pin_if_served + +KEY = re.compile(r"^[0-9a-f]{64}$") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("mirror", help="the mirror repository, as ghcr.io/org/agentenv-ci-computers") + parser.add_argument("--registry", default="localhost:5000", help="the runner's registry") + args = parser.parse_args() + + with urllib.request.urlopen(f"http://{args.registry}/v2/{REPOSITORY}/tags/list") as response: + keys = sorted(tag for tag in json.load(response).get("tags") or [] if KEY.match(tag)) + auth = RegistryAuth(args.mirror.partition("/")[0], os.environ["GITHUB_ACTOR"], os.environ["GITHUB_TOKEN"]) + seen = f"seen-{datetime.datetime.now(datetime.timezone.utc):%Y%m%d}" + for key in keys: + source = pin_digest(f"{args.registry}/{REPOSITORY}:{key}", None) + for tag in (mirror_tag(key), f"{mirror_tag(key)}-{seen}"): + copy_image(source, f"{args.mirror}:{tag}", None, auth) + print(f"shared {key}") + print(f"share computer images: {len(keys)} images in {args.mirror}, each tagged {seen}") + if keys: + try: + pin_if_served(f"{args.mirror}:{mirror_tag(keys[0])}", None) + except ValueError as e: + print(f"::warning::the composer reads {args.mirror} anonymously and can't, so every run builds its images; " + f"make the package public: {e}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/computer-image-mirror.yml b/.github/workflows/computer-image-mirror.yml new file mode 100644 index 000000000..7c0865cdf --- /dev/null +++ b/.github/workflows/computer-image-mirror.yml @@ -0,0 +1,31 @@ +name: Computer image mirror + +# Deletes the slow tier's shared computer images (local-backends.yml) that no passing main run has used for a week. +on: + schedule: + - cron: "17 6 * * *" + workflow_dispatch: + inputs: + dry_run: + description: List what would be deleted, and delete nothing + type: boolean + default: false + +permissions: + contents: read + packages: write + +jobs: + prune: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + sparse-checkout: .github/scripts + - name: Delete the images unused for a week + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + python3 .github/scripts/prune_computer_images.py scaleapi agentenv-ci-computers --days 7 \ + ${{ inputs.dry_run && '--dry-run' || '' }} diff --git a/.github/workflows/local-backends.yml b/.github/workflows/local-backends.yml index 3b08d5c52..dc3cda67c 100644 --- a/.github/workflows/local-backends.yml +++ b/.github/workflows/local-backends.yml @@ -22,6 +22,8 @@ concurrency: env: AGENT_ENV_ENVIRONMENT: dev + # The slow tier's composed computer images, shared between its runners (computer-image-mirror.yml prunes it). + COMPUTER_IMAGE_MIRROR: ghcr.io/scaleapi/agentenv-ci-computers PIP_DISABLE_PIP_VERSION_CHECK: "1" jobs: @@ -154,8 +156,14 @@ jobs: shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] runs-on: ubuntu-latest timeout-minutes: 40 + permissions: + contents: read + packages: write steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + # The tests run with packages: write; only the share step, on main, gets the token. + with: + persist-credentials: false - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" @@ -168,81 +176,67 @@ jobs: # starts afterwards. - name: Pull the base images from mirror.gcr.io run: python3 .github/scripts/mirror_base_images.py src tst - # A computer image or disk is tagged with a key of everything it is built from, and the composer reuses one the - # image store already holds for its key. That key includes the lock of the packages the image installs, which the - # composer resolves from Ubuntu's archive, PyPI and download URLs unless the store holds it. The registry keeps - # its storage on the runner, and the computer tests share their locks and files in the packages folder - # (tst/util/package_cache.py); both are restored from what main saved latest this week, so a run neither resolves - # nor builds again what hasn't changed. A shard that runs no computer tests misses. - - name: Week of the computer images' cache + # The computer tests share the package locks a compose resolves, and the files they name, in the packages folder + # (tst/util/package_cache.py). An image's key includes its lock, which the composer otherwise resolves from + # Ubuntu's archive, PyPI and download URLs, so the folder is restored from what main saved latest this week. + - name: Week of the packages' cache id: week run: echo "week=$(date -u +%G-W%V)" >> "$GITHUB_OUTPUT" - - name: Restore the computer images and packages - id: computers + - name: Restore the packages + id: packages uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: - path: | - ${{ runner.temp }}/registry - ${{ runner.temp }}/packages - key: computers-${{ matrix.shard }}-${{ steps.week.outputs.week }} - restore-keys: computers-${{ matrix.shard }}-${{ steps.week.outputs.week }}- + path: ${{ runner.temp }}/packages + key: packages-${{ matrix.shard }}-${{ steps.week.outputs.week }} + restore-keys: packages-${{ matrix.shard }}-${{ steps.week.outputs.week }}- - name: Start the local registry run: | - mkdir -p "$RUNNER_TEMP/registry" - docker run --detach --name registry --publish 5000:5000 --volume "$RUNNER_TEMP/registry:/var/lib/registry" \ - mirror.gcr.io/library/registry:2 + docker run --detach --name registry --publish 5000:5000 mirror.gcr.io/library/registry:2 timeout 30 sh -c 'until curl -sf http://localhost:5000/v2/ >/dev/null; do sleep 1; done' # x64 hosted runners have /dev/kvm; the disk form's tests boot under it, and skip without it, which fails below. - name: Let the local provider boot disks with KVM run: sudo chmod 666 /dev/kvm + # A computer image or disk is tagged with a key of everything it is built from. A compose that finds no image of + # its key in the local registry copies the one the mirror holds, which main's runs share, rather than build it. - name: Slow integration tier on the local defaults id: tests env: AGENTENV_TEST_SHARD: ${{ matrix.shard }}/12 AGENTENV_TEST_PACKAGES: ${{ runner.temp }}/packages + AGENT_ENV_COMPUTER_IMAGE_MIRROR: ${{ env.COMPUTER_IMAGE_MIRROR }} run: | .venv/bin/python -m pytest -m int_test_slow -n 2 --dist loadfile -rA --timeout=1200 \ --junitxml=junit-integration-slow.xml tst/integration --ignore=tst/integration/env/gateway/gateway_test.py - # Only the computer images this run used are kept, with the packages, and only main saves them, once its tests - # pass, so pull requests reuse main's without filling the repository's cache. An image is used when the registry's - # log has a request for it by its tag (prune_registry.py), so what a week's earlier runs built and this one no - # longer asks for is dropped. The garbage collection frees what only the dropped tags and repositories referenced; - # with --delete-untagged it would also delete the untagged platform manifests of an image index, and their layers. - # The entry is named by the image keys the registry holds and the locks the packages hold, so a run that built a - # new image or resolved a new lock saves a new one. - - name: Keep only the computer images this run used - id: kept + # Only main saves the packages, once its tests pass, so pull requests reuse main's without filling the + # repository's cache. The entry is named by the locks the folder holds, so a run that resolved a new lock saves a + # new one. + - name: Name the packages' cache entry + id: lockkey if: always() run: | - repositories="$RUNNER_TEMP/registry/docker/registry/v2/repositories" - if [ ! -d "$repositories/agentenv-computers" ]; then echo "computers=false" >> "$GITHUB_OUTPUT"; exit 0; fi - docker logs registry > "$RUNNER_TEMP/registry.log" 2>&1 - docker stop registry - sudo find "$repositories" -mindepth 1 -maxdepth 1 ! -name agentenv-computers -exec rm -rf {} + - sudo python3 .github/scripts/prune_registry.py prune "$RUNNER_TEMP/registry" agentenv-computers \ - "$RUNNER_TEMP/registry.log" - docker run --rm --volume "$RUNNER_TEMP/registry:/var/lib/registry" mirror.gcr.io/library/registry:2 \ - garbage-collect /etc/docker/registry/config.yml > /dev/null - sudo python3 .github/scripts/prune_registry.py links "$RUNNER_TEMP/registry" agentenv-computers - mkdir -p "$RUNNER_TEMP/packages" - du -sh "$RUNNER_TEMP/registry" "$RUNNER_TEMP/packages" - if [ -z "$(sudo ls "$repositories/agentenv-computers/_manifests/tags")" ]; then - echo "computers=false" >> "$GITHUB_OUTPUT"; exit 0 - fi - key=$({ sudo ls "$repositories/agentenv-computers/_manifests/tags" - ls "$RUNNER_TEMP/packages/agentenv-packages/locks" 2>/dev/null; } | sort | sha256sum | cut -c1-16) - echo "key=computers-${{ matrix.shard }}-${{ steps.week.outputs.week }}-$key" >> "$GITHUB_OUTPUT" - echo "computers=true" >> "$GITHUB_OUTPUT" - - name: Save the computer images and packages + locks="$RUNNER_TEMP/packages/agentenv-packages/locks" + if [ -z "$(ls "$locks" 2>/dev/null)" ]; then echo "packages=false" >> "$GITHUB_OUTPUT"; exit 0; fi + key=$(ls "$locks" | sort | sha256sum | cut -c1-16) + echo "key=packages-${{ matrix.shard }}-${{ steps.week.outputs.week }}-$key" >> "$GITHUB_OUTPUT" + echo "packages=true" >> "$GITHUB_OUTPUT" + - name: Save the packages if: >- always() && github.event_name == 'push' && steps.tests.outcome == 'success' - && steps.kept.outputs.computers == 'true' && steps.kept.outputs.key != steps.computers.outputs.cache-matched-key + && steps.lockkey.outputs.packages == 'true' && steps.lockkey.outputs.key != steps.packages.outputs.cache-matched-key uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: - path: | - ${{ runner.temp }}/registry - ${{ runner.temp }}/packages - key: ${{ steps.kept.outputs.key }} + path: ${{ runner.temp }}/packages + key: ${{ steps.lockkey.outputs.key }} + # Once main's tests pass, every image this runner built or copied goes to the mirror under its key and a + # -seen- tag, which computer-image-mirror.yml's prune keeps an image by. + - name: Share the computer images + continue-on-error: true + if: >- + always() && steps.tests.outcome == 'success' + && (github.event_name == 'push' || github.head_ref == 'andrew-park/ci-shared-computer-images') + env: + GITHUB_TOKEN: ${{ github.token }} + run: .venv/bin/python .github/scripts/share_computer_images.py "$COMPUTER_IMAGE_MIRROR" - name: Skips must be declared capability gaps if: always() run: | diff --git a/AGENTS.md b/AGENTS.md index c80ddc3a8..3dacc182d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -392,9 +392,8 @@ orchestrators call it the same way. What callers rely on: `.github/scripts/slow_test_durations.py`. - The computer tests compose in stores of their own, but share the package locks and files a compose resolves (`tst/util/package_cache.py`) through `AGENTENV_TEST_PACKAGES`, a folder for the run unless - it's set. The slow tier caches that folder with the local registry's computer images, so a run - resolves from Ubuntu's archive, PyPI or a download URL only what changed since `main` last saved - them. + it's set. The slow tier caches that folder, so a run resolves from Ubuntu's archive, PyPI or a + download URL only what changed since `main` last saved it. - `tst/manual/` and `tst/benchmarks/` run only by hand, as each module's docstring says; no CI job collects them. - `tst/formal/` holds TLA+ models, run only by hand with each model's `check.sh`; the `AGENTS.md` diff --git a/src/agent_env/config/describe.py b/src/agent_env/config/describe.py index e8e291d7f..73e289cbc 100644 --- a/src/agent_env/config/describe.py +++ b/src/agent_env/config/describe.py @@ -673,6 +673,7 @@ def env_sources() -> tuple[EnvSource, ...]: EnvSource("AGENT_ENV_MODAL_REGION", None, "no file equivalent"), EnvSource("AGENT_ENV_MODAL_APP_NAME", None, "no file equivalent"), EnvSource("AGENT_ENV_FIXTURE_PREFIX", None, "no file equivalent"), + EnvSource("AGENT_ENV_COMPUTER_IMAGE_MIRROR", None, "no file equivalent"), EnvSource("MODAL_TOKEN_ID", None, "credential, no file equivalent"), EnvSource("MODAL_TOKEN_SECRET", None, "credential, no file equivalent"), ] diff --git a/src/agent_env/config/runtime.py b/src/agent_env/config/runtime.py index a98b17005..66326e090 100644 --- a/src/agent_env/config/runtime.py +++ b/src/agent_env/config/runtime.py @@ -24,6 +24,11 @@ objects, image builds, env and agent snapshots, changelogs, default agent and judge trajectories, verifier outputs and the A2A validator's fixtures, so a fresh control-plane backend can write without colliding with existing objects in a shared store. Empty in prod. + +AGENT_ENV_COMPUTER_IMAGE_MIRROR (default ""): a registry repository, read anonymously, holding composed computer images +under their keys and the composer's own digest, as ``ghcr.io/org/computers``. A compose that finds no image of a key in the +env's image store copies the mirror's into it instead of building it. CI sets it so its runners share the images main +built. """ from __future__ import annotations @@ -226,6 +231,9 @@ class Config: fixture_prefix: str = field( default_factory=lambda: os.getenv("AGENT_ENV_FIXTURE_PREFIX", "") ) + computer_image_mirror: str = field( + default_factory=lambda: os.getenv("AGENT_ENV_COMPUTER_IMAGE_MIRROR", "") + ) agent_sandbox_mode: str = field( default_factory=lambda: os.getenv("AGENT_SANDBOX_MODE") or _default_sandbox_mode() ) diff --git a/src/agent_env/env/envs/computer/utils/composed.py b/src/agent_env/env/envs/computer/utils/composed.py index 3ee5281ad..ec3d5d993 100644 --- a/src/agent_env/env/envs/computer/utils/composed.py +++ b/src/agent_env/env/envs/computer/utils/composed.py @@ -117,6 +117,12 @@ def composition() -> str: return "sha256:" + digest.hexdigest() +def mirror_tag(key: str) -> str: + """The tag of the image of ``key`` in a computer image mirror: the key, under the start of ``composition()``, so a + change to the code that composes an image builds it again rather than reuse one the old code built.""" + return f"{composition().removeprefix('sha256:')[:16]}-{key.removeprefix('sha256:')}" + + def _rank(result: dict) -> int: """How far a pair got: ready, then built and either unprobed or failed at its probe, then anything short of an image, and below that one another composer recorded. A result takes a stored one's place only when it got at least diff --git a/src/agent_env/env/envs/computer/utils/composer.py b/src/agent_env/env/envs/computer/utils/composer.py index d9d6c0089..4ab6da95d 100644 --- a/src/agent_env/env/envs/computer/utils/composer.py +++ b/src/agent_env/env/envs/computer/utils/composer.py @@ -50,7 +50,7 @@ thin_disk, windows_disk, ) -from agent_env.env.envs.computer.utils.composed import BUILT, REPOSITORY, Record, failed, now, stale +from agent_env.env.envs.computer.utils.composed import BUILT, REPOSITORY, Record, failed, mirror_tag, now, stale from agent_env.env.envs.computer.utils.install import Install, for_os, platform_of from agent_env.env.published_images import DISK, DISK_TOOLS_IMAGE, published_disk, published_pinned from agent_env.providers.sandbox_providers.host_images import platform_tag @@ -58,7 +58,7 @@ from agent_env.store.image_store.image_contents import image_config, labels from agent_env.store.image_store.oci_registry_credentials import names_registry from agent_env.store.image_store.image_store import ImageStore -from agent_env.store.image_store.registry_api import pin_digest, pin_if_served, write_index +from agent_env.store.image_store.registry_api import copy_image, pin_digest, pin_if_served, write_index from agent_env.store.local_state import holding_locks from agent_env.utils.docker.build import DockerBuildError @@ -771,9 +771,20 @@ def _key(**inputs: Any) -> str: def _stored(env: ComputerEnv, key: str) -> tuple[ImageStore, str, str | None]: """The env's image store, the ref there of the image of ``key``, and that ref pinned when the store serves one - already, for any env that built it.""" + already, for any env that built it, or once it's copied there from the configured computer image mirror.""" store = _image_store(env) - ref = store.image_ref(REPOSITORY, key.removeprefix("sha256:")) + tag = key.removeprefix("sha256:") + ref = store.image_ref(REPOSITORY, tag) + if (pinned := pin_if_served(ref, store.auth(ref))) is not None or not (mirror := get_config().computer_image_mirror): + return store, ref, pinned + try: + if (mirrored := pin_if_served(f"{mirror}:{mirror_tag(tag)}", None)) is None: + return store, ref, None + copy_image(mirrored, ref, None, store.auth(ref)) + except (ValueError, KeyError, OSError) as e: + logger.warning(f"the computer image mirror {mirror} couldn't give {tag}, so it's built: {e}") + return store, ref, None + logger.info(f"copied {tag} from the computer image mirror {mirror}") return store, ref, pin_if_served(ref, store.auth(ref)) diff --git a/src/agent_env/store/image_store/registry_api.py b/src/agent_env/store/image_store/registry_api.py index 4544cfed8..14c116e33 100644 --- a/src/agent_env/store/image_store/registry_api.py +++ b/src/agent_env/store/image_store/registry_api.py @@ -1,5 +1,6 @@ """Read an image from its registry over the OCI distribution API: its digest, to pin a tag to the image it names now, -and its manifests and blobs, to inspect what it holds without pulling it; and write an index of images it holds.""" +and its manifests and blobs, to inspect what it holds without pulling it; write an index of images it holds; and copy an +image from one registry to another.""" from __future__ import annotations @@ -7,6 +8,7 @@ import hashlib import json import re +import tempfile from collections.abc import Iterator from dataclasses import dataclass from pathlib import Path @@ -217,6 +219,40 @@ def write_index(ref: str, images: dict[str, str], auth: RegistryAuth | None, *, {"schemaVersion": 2, "mediaType": media, "manifests": manifests}).encode(), media) +def copy_image(source: str, target: str, source_auth: RegistryAuth | None, target_auth: RegistryAuth | None, *, + timeout: float = _TIMEOUT_SECONDS) -> None: + """Copy the image ``source`` names, pinned by its digest, to ``target``'s tag: every blob ``target``'s repository + lacks, then each manifest byte for byte, so the copy keeps ``source``'s digest; an index's platform manifests + first, each by its digest.""" + digest = _parse(source).digest + if digest is None: + raise ValueError(f"{source}: a copy goes by the image's digest, and this names none") + image = _parse(target) + with (_Repository(_parse(source), source_auth, timeout, source) as reading, + _Repository(image, target_auth, timeout, target) as writing, + tempfile.TemporaryDirectory(prefix="agentenv-copy-") as work): + + def manifest(reference: str) -> tuple[bytes, str]: + response = reading.get(f"manifests/{reference}", accept=_MANIFEST_TYPES) + body = response.json() + served = response.headers.get("content-type", "").split(";")[0].strip() or body.get("mediaType", "") + if served in _INDEXES: + for child in body["manifests"]: + manifest(child["digest"]) + else: + for blob in [body["config"], *body["layers"]]: + if writing.has_blob(blob["digest"]): + continue + path = Path(work) / blob["digest"].removeprefix("sha256:") + read_blob(source, blob["digest"], source_auth, path, timeout=timeout) + writing.upload(blob["digest"], path) + path.unlink() + writing.put(f"manifests/{reference}", response.content, served) + return response.content, served + + writing.put(f"manifests/{image.tag or 'latest'}", *manifest(digest)) + + class _Repository: """One image's repository on its registry, read with the Bearer token the registry's first challenge asks for.""" @@ -250,6 +286,25 @@ def put(self, path: str, content: bytes, content_type: str) -> None: if response.status_code not in (200, 201): self._raise_for(response) + def has_blob(self, digest: str) -> bool: + response = self._send("HEAD", f"blobs/{digest}", None, stream=False) + if response.status_code == 404: + return False + self._raise_for(response) + return True + + def upload(self, digest: str, path: Path) -> None: + """Push the file ``path`` as the blob ``digest``, in one request after the registry opens an upload.""" + opened = self._send("POST", "blobs/uploads/", None, stream=False) + if opened.status_code != 202 or "location" not in opened.headers: + self._raise_for(opened) + raise ValueError(f"{self._what}: {self._host} opened no upload for {digest} (HTTP {opened.status_code})") + location = httpx.URL(f"{self._base}/").join(opened.headers["location"]) + response = self._send("PUT", str(location.copy_merge_params({"digest": digest})), None, stream=False, + content=path, content_type="application/octet-stream") + if response.status_code != 201: + self._raise_for(response) + @contextlib.contextmanager def stream(self, path: str) -> Iterator[httpx.Response]: """``path`` as a streamed response, closed when the block ends.""" @@ -262,19 +317,27 @@ def stream(self, path: str) -> Iterator[httpx.Response]: finally: response.close() - def _send(self, method: str, path: str, accept: str | None, *, stream: bool, content: bytes | None = None, + def _send(self, method: str, path: str, accept: str | None, *, stream: bool, content: bytes | Path | None = None, content_type: str | None = None) -> httpx.Response: + """``path`` is under the repository, or a whole URL the registry gave; a file's ``content`` streams from disk.""" + url = path if path.startswith(("http://", "https://")) else f"{self._base}/{path}" try: - for _ in range(2): + for attempt in range(2): headers = {"Accept": accept} if accept else {} if content_type: headers["Content-Type"] = content_type if self._token: headers["Authorization"] = f"Bearer {self._token}" - request = self._client.build_request(method, f"{self._base}/{path}", headers=headers, content=content) - # Basic credentials go with every request until a Bearer token replaces them. - response = self._client.send(request, auth=None if self._token else self._basic, stream=stream) - if response.status_code != 401 or self._token: + with contextlib.ExitStack() as opened: + body = content + if isinstance(content, Path): + headers["Content-Length"] = str(content.stat().st_size) + body = opened.enter_context(content.open("rb")) + request = self._client.build_request(method, url, headers=headers, content=body) + # Basic credentials go with every request until a Bearer token replaces them. + response = self._client.send(request, auth=None if self._token else self._basic, stream=stream) + # A held token is asked for again once: GHCR scopes it to the first challenge, a pull's before a push. + if response.status_code != 401 or attempt: return response if stream: response.read() diff --git a/tst/unit/env/envs/test_computer_env.py b/tst/unit/env/envs/test_computer_env.py index e3fdf992b..34b6aeee7 100644 --- a/tst/unit/env/envs/test_computer_env.py +++ b/tst/unit/env/envs/test_computer_env.py @@ -46,6 +46,7 @@ probing, resolve_in_base) from agent_env.env.envs.computer.utils import sites as computer_sites from agent_env.env.envs.computer.utils.composer import Composing +from agent_env.store.image_store.registry_api import RegistryRefusedError from agent_env.env.envs.computer.utils.probing import _probe from agent_env.env.envs.computer.env import COMPUTER_TOOLS, ComputerEnv from agent_env.env.envs.computer import app_env @@ -749,6 +750,77 @@ def test_env_build_by_an_installed_alias_composes_under_the_id_of_the_env_it_nam "@local/acme-kit/kit/desktop__computer").image().image_name]) +MIRROR = "ghcr.io/org/computers" + + +def _mirrored(docker, monkeypatch, read=None): + """Move every composed image out of the env's image store into the computer image mirror, as a fresh runner of CI + finds them, and record each copy back: the mirror's refs, read with ``read`` when given.""" + stored = {ref: digest for ref, digest in docker.registry.tags.items() + if f"/{composed.REPOSITORY}:" in ref and ":index-" not in ref} + for ref in stored: + del docker.registry.tags[ref] + mirror = {f"{MIRROR}:{composed.mirror_tag(ref.rpartition(':')[2])}": digest for ref, digest in stored.items()} + copied, real = [], composer.pin_if_served + + def pin(ref, auth): + if ref.startswith(f"{MIRROR}:"): + return read(ref) if read else (f"{ref}@{mirror[ref]}" if ref in mirror else None) + return real(ref, auth) + + def copy(source, target, source_auth, target_auth): + copied.append((source.partition("@")[0], target)) + docker.registry.tags[target] = source.rpartition("@")[2] + + monkeypatch.setattr(composer, "pin_if_served", pin) + monkeypatch.setattr(composer, "copy_image", copy) + monkeypatch.setattr(get_config(), "computer_image_mirror", MIRROR) + return stored, copied + + +def test_an_image_the_store_lacks_is_copied_from_the_computer_image_mirror_and_reused_rather_than_built(base, driver, + docker, + monkeypatch): + first = _put(base, driver) + stored, copied = _mirrored(docker, monkeypatch) + + second = _put(base, driver, id="@local/~/kit/other", environment_name="other") + + reused = Env.get(second.id).metadata["composed"]["container"] + assert len(docker.builds) == 2 and all(r["reused"] for r in reused.values()) + assert sorted(copied) == sorted((f"{MIRROR}:{composed.mirror_tag(ref.rpartition(':')[2])}", ref) for ref in stored) + assert {arch: r["image"] for arch, r in reused.items()} == { + arch: r["image"] for arch, r in Env.get(first.id).metadata["composed"]["container"].items()} + + +def test_a_change_to_the_code_that_composes_an_image_misses_the_mirror_and_builds_it(base, driver, docker, + monkeypatch): + _put(base, driver) + _, copied = _mirrored(docker, monkeypatch) + monkeypatch.setattr(composed, "composition", lambda: "sha256:" + "f" * 64) + + second = _put(base, driver, id="@local/~/kit/other", environment_name="other") + + assert len(docker.builds) == 4 and copied == [] + assert not any(r["reused"] for r in Env.get(second.id).metadata["composed"]["container"].values()) + + +def test_a_computer_image_mirror_that_cant_be_read_leaves_the_image_to_be_built(base, driver, docker, monkeypatch, + caplog): + _put(base, driver) + + def refused(ref): + raise RegistryRefusedError(f"{ref}: ghcr.io refused to serve it (HTTP 403)") + + _, copied = _mirrored(docker, monkeypatch, read=refused) + with caplog.at_level(logging.WARNING, logger=composer.__name__): + second = _put(base, driver, id="@local/~/kit/other", environment_name="other") + + built = Env.get(second.id).metadata["composed"]["container"] + assert len(docker.builds) == 4 and not any(r["reused"] for r in built.values()) and copied == [] + assert f"the computer image mirror {MIRROR} couldn't give" in caplog.text + + def test_builds_of_one_env_at_once_keep_each_others_pairs_and_a_failure_doesnt_replace_a_built_one(base, driver, docker, monkeypatch): diff --git a/tst/unit/store/registry_api_test.py b/tst/unit/store/registry_api_test.py index aec0d54fe..e38afa194 100644 --- a/tst/unit/store/registry_api_test.py +++ b/tst/unit/store/registry_api_test.py @@ -327,3 +327,84 @@ def ghcr(request): assert json.loads(request.content) == {"schemaVersion": 2, "mediaType": written, "manifests": [ amd64_entry, *(children if arm64 == "index" else [{**amd64_entry, "digest": OTHER, "platform": {"architecture": "arm64", "os": "linux"}}])]} + + +def _sha(data: bytes) -> str: + return f"sha256:{hashlib.sha256(data).hexdigest()}" + + +def test_an_image_is_copied_by_its_digest_with_only_the_blobs_the_target_lacks_and_its_manifests_byte_for_byte( + fake_registry): + config, kept, missing = b'{"architecture": "amd64"}', b"layer the target has", b"layer it lacks" + image = json.dumps({"mediaType": OCI_IMAGE, "config": {"digest": _sha(config)}, + "layers": [{"digest": _sha(kept)}, {"digest": _sha(missing)}]}).encode() + index = json.dumps({"mediaType": OCI_INDEX, "manifests": [{"digest": _sha(image)}]}).encode() + source = {_sha(index): (OCI_INDEX, index), _sha(image): (OCI_IMAGE, image)} + blobs = {_sha(blob): blob for blob in (config, kept, missing)} + target_blobs, target_manifests = {_sha(kept): kept}, {} + + def registries(request): + path = request.url.path + if request.url.host == "ghcr.io": + if "/blobs/" in path: + return httpx.Response(200, content=blobs[path.rpartition("/")[2]]) + media, body = source[path.rpartition("/")[2]] + return httpx.Response(200, content=body, headers={"Content-Type": media}) + if request.method == "HEAD": + return httpx.Response(200 if path.rpartition("/")[2] in target_blobs else 404) + if request.method == "POST": + return httpx.Response(202, headers={"Location": "/v2/agentenv-computers/blobs/uploads/u1?_state=s"}) + if "/blobs/uploads/" in path: + assert request.url.params["_state"] == "s" + target_blobs[request.url.params["digest"]] = request.read() + return httpx.Response(201) + target_manifests[path.rpartition("/")[2]] = (request.headers["content-type"], request.content) + return httpx.Response(201) + + seen = fake_registry(registries) + registry_api.copy_image(f"ghcr.io/org/computers:k@{_sha(index)}", "localhost:5000/agentenv-computers:k", None, None) + + assert target_blobs == blobs + assert [r.url.params["digest"] for r in seen if r.method == "PUT" and "/blobs/" in r.url.path] == [ + _sha(config), _sha(missing)] + assert target_manifests == {_sha(image): (OCI_IMAGE, image), _sha(index): (OCI_INDEX, index), "k": (OCI_INDEX, index)} + + +def test_a_copy_needs_the_digest_it_copies(fake_registry): + seen = fake_registry(lambda request: httpx.Response(500)) + + with pytest.raises(ValueError, match="names none"): + registry_api.copy_image("ghcr.io/org/computers:k", "localhost:5000/agentenv-computers:k", None, None) + assert seen == [] + + +def test_a_copy_to_ghcr_asks_again_for_the_push_token_its_first_challenge_didnt_give(fake_registry): + config = b'{"architecture": "amd64"}' + image = json.dumps({"mediaType": OCI_IMAGE, "config": {"digest": _sha(config)}, "layers": []}).encode() + pushed = [] + + def registries(request): + if request.url.host == "localhost": + if "/blobs/" in request.url.path: + return httpx.Response(200, content=config) + return httpx.Response(200, content=image, headers={"Content-Type": OCI_IMAGE}) + if request.url.path == "/token": + assert request.headers["authorization"] == "Basic " + base64.b64encode(b"bot:t").decode() + return httpx.Response(200, json={"token": request.url.params["scope"]}) + writes = request.method in ("POST", "PUT") + scope = "repository:org/computers:pull,push" if writes else "repository:org/computers:pull" + if request.headers.get("authorization") not in (f"Bearer {scope}", "Bearer repository:org/computers:pull,push"): + return httpx.Response(401, headers={"WWW-Authenticate": f'Bearer realm="https://ghcr.io/token",scope="{scope}"'}) + if request.method == "HEAD": + return httpx.Response(404) + if request.method == "POST": + return httpx.Response(202, headers={"Location": "/v2/org/computers/blobs/uploads/u1"}) + pushed.append(request.url.path) + return httpx.Response(201) + + fake_registry(registries) + registry_api.copy_image(f"localhost:5000/agentenv-computers:k@{_sha(image)}", "ghcr.io/org/computers:k", None, + RegistryAuth("ghcr.io", "bot", "t")) + + assert pushed == ["/v2/org/computers/blobs/uploads/u1", f"/v2/org/computers/manifests/{_sha(image)}", + "/v2/org/computers/manifests/k"] diff --git a/tst/unit/test_computer_image_mirror_scripts.py b/tst/unit/test_computer_image_mirror_scripts.py new file mode 100644 index 000000000..54f91da22 --- /dev/null +++ b/tst/unit/test_computer_image_mirror_scripts.py @@ -0,0 +1,109 @@ +"""The CI computer image mirror's two scripts: what a runner shares into it, and which of its versions a prune deletes.""" + +from __future__ import annotations + +import datetime +import io +import json +import sys + +import pytest + +from tst.util.scripts import load_script + +NOW = datetime.datetime(2026, 11, 20, 12, tzinfo=datetime.timezone.utc) +CUTOFF = NOW - datetime.timedelta(days=7) +KEY, OTHER = "c" * 16 + "-" + "a" * 64, "c" * 16 + "-" + "b" * 64 +RAW, RAW_OTHER = "a" * 64, "b" * 64 + + +def _version(name: str, created: str, *tags: str) -> dict: + return {"id": hash(name) % 10_000, "name": name, "created_at": created, "metadata": {"container": {"tags": list(tags)}}} + + +@pytest.fixture +def prune(): + return load_script("prune_computer_images.py") + + +def test_an_image_last_seen_within_the_window_is_kept_and_one_last_seen_before_it_is_deleted(prune): + used = _version("sha256:1", "2026-10-01T00:00:00Z", KEY, f"{KEY}-seen-20261001", f"{KEY}-seen-20261115") + unused = _version("sha256:2", "2026-10-01T00:00:00Z", OTHER, f"{OTHER}-seen-20261112") + + assert prune.stale_versions([used, unused], CUTOFF, lambda digest: []) == [unused] + + +def test_an_image_seen_on_the_cutoffs_day_is_kept_through_that_day(prune): + edge = _version("sha256:1", "2026-10-01T00:00:00Z", KEY, f"{KEY}-seen-20261113") + + assert prune.stale_versions([edge], CUTOFF, lambda digest: []) == [] + + +def test_a_version_uploaded_within_the_window_is_kept_though_no_run_has_tagged_it_seen(prune): + fresh = _version("sha256:1", "2026-11-19T00:00:00Z") + old = _version("sha256:2", "2026-11-01T00:00:00Z") + + assert prune.stale_versions([fresh, old], CUTOFF, lambda digest: []) == [old] + + +def test_the_platform_manifests_a_kept_index_lists_are_kept_and_a_deleted_ones_are_not(prune): + kept = _version("sha256:k", "2026-10-01T00:00:00Z", KEY, f"{KEY}-seen-20261119") + dropped = _version("sha256:d", "2026-10-01T00:00:00Z", OTHER, f"{OTHER}-seen-20261001") + children = {"sha256:k": ["sha256:k-amd64", "sha256:k-attestation"], "sha256:d": ["sha256:d-amd64"]} + untagged = [_version(digest, "2026-10-01T00:00:00Z") for digest in ("sha256:k-amd64", "sha256:k-attestation", + "sha256:d-amd64")] + + stale = prune.stale_versions([kept, dropped, *untagged], CUTOFF, lambda digest: children.get(digest, [])) + + assert [version["name"] for version in stale] == ["sha256:d", "sha256:d-amd64"] + + +def test_a_prune_without_the_package_deletes_nothing_passes_and_says_why(prune, monkeypatch, capsys): + def missing(method, url): + raise prune.urllib.error.HTTPError(url, 404, "Not Found", {}, io.BytesIO(b"")) + + monkeypatch.setattr(prune, "_github", missing) + monkeypatch.setattr(sys, "argv", ["prune_computer_images.py", "org", "agentenv-ci-computers"]) + + assert prune.main() == 0 + assert "no package org/agentenv-ci-computers, or this repository has no access to it" in capsys.readouterr().out + + +def test_a_runner_shares_each_image_under_its_mirror_tag_and_todays_seen_tag_and_leaves_its_indexes(monkeypatch, + capsys): + share = load_script("share_computer_images.py") + tags = {"tags": [RAW, f"index-{RAW_OTHER}", RAW_OTHER]} + copies = [] + monkeypatch.setattr(share.urllib.request, "urlopen", lambda url: io.BytesIO(json.dumps(tags).encode())) + monkeypatch.setattr(share, "pin_digest", lambda ref, auth: f"{ref}@sha256:{ref[-64:]}") + monkeypatch.setattr(share, "mirror_tag", lambda key: f"{'c' * 16}-{key}") + monkeypatch.setattr(share, "copy_image", lambda source, target, source_auth, auth: copies.append( + (source.rpartition("@")[2], target, auth.username))) + monkeypatch.setattr(share, "pin_if_served", lambda ref, auth: f"{ref}@sha256:{RAW}") + monkeypatch.setattr(share.datetime, "datetime", type("Fixed", (datetime.datetime,), {"now": staticmethod(lambda tz: NOW)})) + monkeypatch.setenv("GITHUB_ACTOR", "bot") + monkeypatch.setenv("GITHUB_TOKEN", "t") + monkeypatch.setattr(sys, "argv", ["share_computer_images.py", "ghcr.io/org/agentenv-ci-computers"]) + + assert share.main() == 0 + assert copies == [(f"sha256:{raw}", f"ghcr.io/org/agentenv-ci-computers:{tag}", "bot") + for raw, key in ((RAW, KEY), (RAW_OTHER, OTHER)) for tag in (key, f"{key}-seen-20261120")] + assert "::warning::" not in capsys.readouterr().out + + +def test_a_mirror_the_composer_cant_read_anonymously_warns_to_make_it_public(monkeypatch, capsys): + share = load_script("share_computer_images.py") + monkeypatch.setattr(share.urllib.request, "urlopen", lambda url: io.BytesIO(json.dumps({"tags": [RAW]}).encode())) + monkeypatch.setattr(share, "pin_digest", lambda ref, auth: f"{ref}@sha256:{RAW}") + monkeypatch.setattr(share, "copy_image", lambda *args: None) + + def private(ref, auth): + raise ValueError(f"{ref}: ghcr.io refused to serve it (HTTP 403)") + + monkeypatch.setattr(share, "pin_if_served", private) + monkeypatch.setenv("GITHUB_ACTOR", "bot") + monkeypatch.setenv("GITHUB_TOKEN", "t") + monkeypatch.setattr(sys, "argv", ["share_computer_images.py", "ghcr.io/org/agentenv-ci-computers"]) + + assert share.main() == 0 + assert "make the package public" in capsys.readouterr().out diff --git a/tst/unit/test_prune_registry_script.py b/tst/unit/test_prune_registry_script.py deleted file mode 100644 index 2b842562c..000000000 --- a/tst/unit/test_prune_registry_script.py +++ /dev/null @@ -1,97 +0,0 @@ -"""``.github/scripts/prune_registry.py``, which keeps only the computer images a slow-tier run used before the local -registry's storage is cached: their tags, the manifests those reach, and the layer links garbage collection leaves.""" - -from __future__ import annotations - -import ast -import hashlib -import json -import sys -from pathlib import Path - -import pytest - -from tst.util.scripts import SCRIPTS, load_script - -SCRIPT = SCRIPTS / "prune_registry.py" -REPOSITORY = "agentenv-computers" -_USED, _STALE = "a" * 64, "b" * 64 - - -@pytest.fixture -def script(): - return load_script(SCRIPT.name) - - -def test_it_imports_only_the_standard_library(): - """It runs under the runner's python3 with sudo, outside the project's environment.""" - imported = set() - for node in ast.walk(ast.parse(SCRIPT.read_text())): - if isinstance(node, ast.Import): - imported |= {alias.name.split(".")[0] for alias in node.names} - elif isinstance(node, ast.ImportFrom) and node.level == 0: - imported.add(node.module.split(".")[0]) - assert imported <= set(sys.stdlib_module_names) | {"__future__"} - - -def test_a_tag_is_used_when_the_log_has_a_request_for_its_manifest_by_it(script): - log = "\n".join([ - f'time="t" level=info msg="response completed" http.request.method=HEAD ' - f'http.request.uri="/v2/{REPOSITORY}/manifests/{_USED}" http.response.status=200', - f'172.17.0.1 - - [10/Nov/2026:05:00:00 +0000] "PUT /v2/{REPOSITORY}/manifests/{"c" * 64} HTTP/1.1" 201 0', - f'172.17.0.1 - - [10/Nov/2026:05:00:01 +0000] "GET /v2/{REPOSITORY}/manifests/sha256:{"d" * 64} HTTP/1.1" 200', - f'172.17.0.1 - - [10/Nov/2026:05:00:02 +0000] "HEAD /v2/other/manifests/{_STALE} HTTP/1.1" 200 0', - ]) - - assert script.used_tags(log, REPOSITORY) == {_USED, "c" * 64} - - -def test_pruning_keeps_the_used_tags_and_every_manifest_their_index_lists(script, tmp_path): - platforms = [_blob(tmp_path, {"layers": [], "n": n}) for n in range(2)] - index = _blob(tmp_path, {"manifests": [{"digest": digest} for digest in platforms]}) - stale = _blob(tmp_path, {"layers": [], "n": "stale"}) - _tag(tmp_path, _USED, index) - _tag(tmp_path, _STALE, stale) - for digest in (index, *platforms, stale): - _revision(tmp_path, digest) - - kept, dropped = script.prune(tmp_path, REPOSITORY, {_USED}) - - manifests = tmp_path / "docker/registry/v2/repositories" / REPOSITORY / "_manifests" - assert (kept, dropped) == (1, 1) - assert sorted(p.name for p in (manifests / "tags").iterdir()) == [_USED] - assert {p.name for p in (manifests / "revisions/sha256").iterdir()} == {d.removeprefix("sha256:") - for d in (index, *platforms)} - - -def test_links_to_layers_garbage_collection_freed_are_dropped(script, tmp_path): - kept = _blob(tmp_path, {"layer": "kept"}) - layers = tmp_path / "docker/registry/v2/repositories" / REPOSITORY / "_layers/sha256" - for digest in (kept, "sha256:" + "e" * 64): - (layers / digest.removeprefix("sha256:")).mkdir(parents=True) - (layers / digest.removeprefix("sha256:") / "link").write_text(digest) - - assert script.drop_freed_links(tmp_path, REPOSITORY) == 1 - assert [p.name for p in layers.iterdir()] == [kept.removeprefix("sha256:")] - - -def _blob(storage: Path, content: dict) -> str: - data = json.dumps(content).encode() - hex_digest = hashlib.sha256(data).hexdigest() - path = storage / "docker/registry/v2/blobs/sha256" / hex_digest[:2] / hex_digest / "data" - path.parent.mkdir(parents=True) - path.write_bytes(data) - return f"sha256:{hex_digest}" - - -def _tag(storage: Path, tag: str, digest: str) -> None: - current = storage / "docker/registry/v2/repositories" / REPOSITORY / "_manifests/tags" / tag / "current" - current.mkdir(parents=True) - (current / "link").write_text(digest) - - -def _revision(storage: Path, digest: str) -> None: - revision = (storage / "docker/registry/v2/repositories" / REPOSITORY / "_manifests/revisions/sha256" - / digest.removeprefix("sha256:")) - revision.mkdir(parents=True) - (revision / "link").write_text(digest) From c3ed8f834245909d9d80811be3294200ef2eb19e Mon Sep 17 00:00:00 2001 From: Andrew Park Date: Mon, 9 Nov 2026 15:59:49 -0800 Subject: [PATCH 2/6] ci: share computer images from main's runs only Co-Authored-By: Claude --- .github/workflows/local-backends.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/local-backends.yml b/.github/workflows/local-backends.yml index dc3cda67c..4eadbcff3 100644 --- a/.github/workflows/local-backends.yml +++ b/.github/workflows/local-backends.yml @@ -232,8 +232,7 @@ jobs: - name: Share the computer images continue-on-error: true if: >- - always() && steps.tests.outcome == 'success' - && (github.event_name == 'push' || github.head_ref == 'andrew-park/ci-shared-computer-images') + always() && steps.tests.outcome == 'success' && github.event_name == 'push' env: GITHUB_TOKEN: ${{ github.token }} run: .venv/bin/python .github/scripts/share_computer_images.py "$COMPUTER_IMAGE_MIRROR" From 46f08d34b0f9a3f154944673c172b29a05f76331 Mon Sep 17 00:00:00 2001 From: Andrew Park Date: Mon, 9 Nov 2026 16:52:54 -0800 Subject: [PATCH 3/6] fix(registry): refuse an upload a registry sends to another origin before its credentials go Co-Authored-By: Claude --- .../store/image_store/registry_api.py | 4 ++++ tst/unit/store/registry_api_test.py | 24 +++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/src/agent_env/store/image_store/registry_api.py b/src/agent_env/store/image_store/registry_api.py index 14c116e33..07d92a8ed 100644 --- a/src/agent_env/store/image_store/registry_api.py +++ b/src/agent_env/store/image_store/registry_api.py @@ -300,6 +300,10 @@ def upload(self, digest: str, path: Path) -> None: self._raise_for(opened) raise ValueError(f"{self._what}: {self._host} opened no upload for {digest} (HTTP {opened.status_code})") location = httpx.URL(f"{self._base}/").join(opened.headers["location"]) + base = httpx.URL(self._base) + if (location.scheme, location.netloc) != (base.scheme, base.netloc): + raise ValueError(f"{self._what}: {self._host} sent the upload of {digest} to {location.scheme}://" + f"{location.netloc.decode()}, which its credentials don't go to") response = self._send("PUT", str(location.copy_merge_params({"digest": digest})), None, stream=False, content=path, content_type="application/octet-stream") if response.status_code != 201: diff --git a/tst/unit/store/registry_api_test.py b/tst/unit/store/registry_api_test.py index e38afa194..120992dd8 100644 --- a/tst/unit/store/registry_api_test.py +++ b/tst/unit/store/registry_api_test.py @@ -408,3 +408,27 @@ def registries(request): assert pushed == ["/v2/org/computers/blobs/uploads/u1", f"/v2/org/computers/manifests/{_sha(image)}", "/v2/org/computers/manifests/k"] + + +@pytest.mark.parametrize("location", ["https://elsewhere.example/v2/org/computers/blobs/uploads/u1", + "http://ghcr.io/v2/org/computers/blobs/uploads/u1"], + ids=["another-host", "plain-http"]) +def test_an_upload_the_registry_sends_elsewhere_is_refused_before_its_credentials_go(fake_registry, location): + config = b'{"architecture": "amd64"}' + image = json.dumps({"mediaType": OCI_IMAGE, "config": {"digest": _sha(config)}, "layers": []}).encode() + + def registries(request): + if request.url.host == "localhost": + if "/blobs/" in request.url.path: + return httpx.Response(200, content=config) + return httpx.Response(200, content=image, headers={"Content-Type": OCI_IMAGE}) + assert request.url.host == "ghcr.io" and request.url.scheme == "https", request.url + if request.method == "HEAD": + return httpx.Response(404) + return httpx.Response(202, headers={"Location": location}) + + fake_registry(registries) + + with pytest.raises(ValueError, match="which its credentials don't go to"): + registry_api.copy_image(f"localhost:5000/agentenv-computers:k@{_sha(image)}", "ghcr.io/org/computers:k", None, + RegistryAuth("ghcr.io", "bot", "t")) From db518412bfce3909b1fe71cdea8300f30fba2e2e Mon Sep 17 00:00:00 2001 From: Andrew Park Date: Fri, 13 Nov 2026 01:40:31 +0000 Subject: [PATCH 4/6] ci: drop the computer image mirror's prune The mirror package is public, so GHCR stores it for free, and the prune needed a package-admin right the repository doesn't hold. Removing it removes the daily workflow, its script and tests, and the dated seen tag each share wrote only so the prune could tell used images from unused ones. Sharing and the composer's copy are unchanged. --- .github/scripts/prune_computer_images.py | 112 ------------------ .github/scripts/share_computer_images.py | 14 +-- .github/workflows/computer-image-mirror.yml | 31 ----- .github/workflows/local-backends.yml | 5 +- .../test_computer_image_mirror_scripts.py | 67 +---------- 5 files changed, 11 insertions(+), 218 deletions(-) delete mode 100644 .github/scripts/prune_computer_images.py delete mode 100644 .github/workflows/computer-image-mirror.yml diff --git a/.github/scripts/prune_computer_images.py b/.github/scripts/prune_computer_images.py deleted file mode 100644 index 2cf220edf..000000000 --- a/.github/scripts/prune_computer_images.py +++ /dev/null @@ -1,112 +0,0 @@ -"""Delete the CI computer image mirror's images no run has used for ``--days`` days. - -``share_computer_images.py`` tags each image a passing main run used ``-seen-YYYYMMDD``. An image is kept while its -newest such tag, or its upload when it has none, is within ``--days`` of now, with every platform manifest a kept index -lists, since GHCR lists those as versions of their own. Every other version is deleted through GitHub's packages API, -read and written with ``GITHUB_TOKEN``, as GHCR is. A version GitHub won't delete warns. ``--dry-run`` prints what -would go and deletes nothing. -""" - -from __future__ import annotations - -import argparse -import base64 -import datetime -import json -import os -import re -import sys -import urllib.error -import urllib.parse -import urllib.request - -SEEN = re.compile(r"^[0-9a-f]{16}-[0-9a-f]{64}-seen-(\d{8})$") -_INDEXES = {"application/vnd.oci.image.index.v1+json", "application/vnd.docker.distribution.manifest.list.v2+json"} -_ACCEPT = ", ".join([*_INDEXES, "application/vnd.oci.image.manifest.v1+json", - "application/vnd.docker.distribution.manifest.v2+json"]) - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) - parser.add_argument("org") - parser.add_argument("package", help="the container package, as agentenv-ci-computers") - parser.add_argument("--days", type=int, default=7) - parser.add_argument("--dry-run", action="store_true") - args = parser.parse_args() - - api = f"https://api.github.com/orgs/{args.org}/packages/container/{args.package}/versions" - try: - versions = _versions(api) - except urllib.error.HTTPError as e: - if e.code != 404: - raise - print(f"::warning::prune computer images: no package {args.org}/{args.package}, or this repository has no " - "access to it, so nothing is pruned") - return 0 - cutoff = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=args.days) - stale = stale_versions(versions, cutoff, lambda digest: _children(args.org, args.package, digest)) - failed = 0 - for version in stale: - print(f"{'would delete' if args.dry_run else 'deleting'} {version['name']} " - f"{version['metadata']['container']['tags'] or '(untagged)'}") - if not args.dry_run: - try: - _github("DELETE", f"{api}/{version['id']}") - except urllib.error.HTTPError as e: - failed += 1 - print(f"::warning::couldn't delete {version['name']}: HTTP {e.code} {e.read().decode()[:200]}") - print(f"prune computer images: {len(stale) - failed} of {len(versions)} versions " - f"{'would be ' if args.dry_run else ''}deleted, none used since {cutoff:%Y-%m-%d}") - return 0 - - -def stale_versions(versions: list[dict], cutoff: datetime.datetime, children) -> list[dict]: - """The ``versions`` to delete: neither used nor uploaded since ``cutoff``, nor a platform manifest of one kept. - ``children(digest)`` lists the digests of the manifests the index of that digest lists, or none for an image.""" - kept = {version["name"] for version in versions if _last_used(version) >= cutoff} - for digest in list(kept): - kept.update(children(digest)) - return [version for version in versions if version["name"] not in kept] - - -def _last_used(version: dict) -> datetime.datetime: - created = datetime.datetime.fromisoformat(version["created_at"].replace("Z", "+00:00")) - seen = [datetime.datetime.strptime(m.group(1), "%Y%m%d").replace(tzinfo=datetime.timezone.utc) - for tag in version["metadata"]["container"]["tags"] if (m := SEEN.match(tag))] - return max([created, *(day + datetime.timedelta(days=1) for day in seen)]) - - -def _versions(api: str) -> list[dict]: - versions, page = [], 1 - while batch := _github("GET", f"{api}?per_page=100&page={page}"): - versions += batch - page += 1 - return versions - - -def _children(org: str, package: str, digest: str) -> list[str]: - """The platform manifests the index ``digest`` lists, read from GHCR with ``GITHUB_TOKEN``; none for an image.""" - repository = f"{org}/{package}" - signed_in = base64.b64encode(f"x:{os.environ['GITHUB_TOKEN']}".encode()).decode() - with urllib.request.urlopen(urllib.request.Request(f"https://ghcr.io/token?scope=repository:{repository}:pull", - headers={"Authorization": f"Basic {signed_in}"})) as response: - token = json.load(response)["token"] - request = urllib.request.Request(f"https://ghcr.io/v2/{repository}/manifests/{digest}", - headers={"Authorization": f"Bearer {token}", "Accept": _ACCEPT}) - with urllib.request.urlopen(request) as response: - manifest = json.load(response) - media = response.headers.get("Content-Type", "").split(";")[0].strip() - return [child["digest"] for child in manifest.get("manifests", [])] if media in _INDEXES else [] - - -def _github(method: str, url: str): - request = urllib.request.Request(url, method=method, headers={ - "Authorization": f"Bearer {os.environ['GITHUB_TOKEN']}", "Accept": "application/vnd.github+json", - "X-GitHub-Api-Version": "2022-11-28"}) - with urllib.request.urlopen(request) as response: - body = response.read() - return json.loads(body) if body else None - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/.github/scripts/share_computer_images.py b/.github/scripts/share_computer_images.py index 17fb93078..27c2d8a66 100644 --- a/.github/scripts/share_computer_images.py +++ b/.github/scripts/share_computer_images.py @@ -1,16 +1,14 @@ """Copy every composed computer image a slow-tier runner's registry holds to the CI computer image mirror, the one ``AGENT_ENV_COMPUTER_IMAGE_MIRROR`` names, so later runs copy them back rather than build them. -Each image goes under its mirror tag (``composed.mirror_tag``: the composer's digest, then the key), and again with -today's date, ``-seen-YYYYMMDD``, which is how ``prune_computer_images.py`` tells an image a recent run used from one -none has. An image already in the mirror is only tagged again: no blob it holds is sent twice. The mirror is written as -``GITHUB_ACTOR`` with ``GITHUB_TOKEN``; one the composer can't read anonymously, as GHCR makes a new package, warns. +Each image goes under its mirror tag (``composed.mirror_tag``: the composer's digest, then the key). No blob the mirror +already holds is sent again. The mirror is written as ``GITHUB_ACTOR`` with ``GITHUB_TOKEN``; one the composer can't +read anonymously, as GHCR makes a new package, warns. """ from __future__ import annotations import argparse -import datetime import json import os import re @@ -33,13 +31,11 @@ def main() -> int: with urllib.request.urlopen(f"http://{args.registry}/v2/{REPOSITORY}/tags/list") as response: keys = sorted(tag for tag in json.load(response).get("tags") or [] if KEY.match(tag)) auth = RegistryAuth(args.mirror.partition("/")[0], os.environ["GITHUB_ACTOR"], os.environ["GITHUB_TOKEN"]) - seen = f"seen-{datetime.datetime.now(datetime.timezone.utc):%Y%m%d}" for key in keys: source = pin_digest(f"{args.registry}/{REPOSITORY}:{key}", None) - for tag in (mirror_tag(key), f"{mirror_tag(key)}-{seen}"): - copy_image(source, f"{args.mirror}:{tag}", None, auth) + copy_image(source, f"{args.mirror}:{mirror_tag(key)}", None, auth) print(f"shared {key}") - print(f"share computer images: {len(keys)} images in {args.mirror}, each tagged {seen}") + print(f"share computer images: {len(keys)} images in {args.mirror}") if keys: try: pin_if_served(f"{args.mirror}:{mirror_tag(keys[0])}", None) diff --git a/.github/workflows/computer-image-mirror.yml b/.github/workflows/computer-image-mirror.yml deleted file mode 100644 index 7c0865cdf..000000000 --- a/.github/workflows/computer-image-mirror.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: Computer image mirror - -# Deletes the slow tier's shared computer images (local-backends.yml) that no passing main run has used for a week. -on: - schedule: - - cron: "17 6 * * *" - workflow_dispatch: - inputs: - dry_run: - description: List what would be deleted, and delete nothing - type: boolean - default: false - -permissions: - contents: read - packages: write - -jobs: - prune: - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - with: - sparse-checkout: .github/scripts - - name: Delete the images unused for a week - env: - GITHUB_TOKEN: ${{ github.token }} - run: | - python3 .github/scripts/prune_computer_images.py scaleapi agentenv-ci-computers --days 7 \ - ${{ inputs.dry_run && '--dry-run' || '' }} diff --git a/.github/workflows/local-backends.yml b/.github/workflows/local-backends.yml index 4eadbcff3..1bd51708a 100644 --- a/.github/workflows/local-backends.yml +++ b/.github/workflows/local-backends.yml @@ -22,7 +22,7 @@ concurrency: env: AGENT_ENV_ENVIRONMENT: dev - # The slow tier's composed computer images, shared between its runners (computer-image-mirror.yml prunes it). + # The slow tier's composed computer images, shared between its runners. COMPUTER_IMAGE_MIRROR: ghcr.io/scaleapi/agentenv-ci-computers PIP_DISABLE_PIP_VERSION_CHECK: "1" @@ -227,8 +227,7 @@ jobs: with: path: ${{ runner.temp }}/packages key: ${{ steps.lockkey.outputs.key }} - # Once main's tests pass, every image this runner built or copied goes to the mirror under its key and a - # -seen- tag, which computer-image-mirror.yml's prune keeps an image by. + # Once main's tests pass, every image this runner built or copied goes to the mirror under its key. - name: Share the computer images continue-on-error: true if: >- diff --git a/tst/unit/test_computer_image_mirror_scripts.py b/tst/unit/test_computer_image_mirror_scripts.py index 54f91da22..e87a163a8 100644 --- a/tst/unit/test_computer_image_mirror_scripts.py +++ b/tst/unit/test_computer_image_mirror_scripts.py @@ -1,76 +1,18 @@ -"""The CI computer image mirror's two scripts: what a runner shares into it, and which of its versions a prune deletes.""" +"""What a slow-tier runner shares into the CI computer image mirror.""" from __future__ import annotations -import datetime import io import json import sys -import pytest - from tst.util.scripts import load_script -NOW = datetime.datetime(2026, 11, 20, 12, tzinfo=datetime.timezone.utc) -CUTOFF = NOW - datetime.timedelta(days=7) KEY, OTHER = "c" * 16 + "-" + "a" * 64, "c" * 16 + "-" + "b" * 64 RAW, RAW_OTHER = "a" * 64, "b" * 64 -def _version(name: str, created: str, *tags: str) -> dict: - return {"id": hash(name) % 10_000, "name": name, "created_at": created, "metadata": {"container": {"tags": list(tags)}}} - - -@pytest.fixture -def prune(): - return load_script("prune_computer_images.py") - - -def test_an_image_last_seen_within_the_window_is_kept_and_one_last_seen_before_it_is_deleted(prune): - used = _version("sha256:1", "2026-10-01T00:00:00Z", KEY, f"{KEY}-seen-20261001", f"{KEY}-seen-20261115") - unused = _version("sha256:2", "2026-10-01T00:00:00Z", OTHER, f"{OTHER}-seen-20261112") - - assert prune.stale_versions([used, unused], CUTOFF, lambda digest: []) == [unused] - - -def test_an_image_seen_on_the_cutoffs_day_is_kept_through_that_day(prune): - edge = _version("sha256:1", "2026-10-01T00:00:00Z", KEY, f"{KEY}-seen-20261113") - - assert prune.stale_versions([edge], CUTOFF, lambda digest: []) == [] - - -def test_a_version_uploaded_within_the_window_is_kept_though_no_run_has_tagged_it_seen(prune): - fresh = _version("sha256:1", "2026-11-19T00:00:00Z") - old = _version("sha256:2", "2026-11-01T00:00:00Z") - - assert prune.stale_versions([fresh, old], CUTOFF, lambda digest: []) == [old] - - -def test_the_platform_manifests_a_kept_index_lists_are_kept_and_a_deleted_ones_are_not(prune): - kept = _version("sha256:k", "2026-10-01T00:00:00Z", KEY, f"{KEY}-seen-20261119") - dropped = _version("sha256:d", "2026-10-01T00:00:00Z", OTHER, f"{OTHER}-seen-20261001") - children = {"sha256:k": ["sha256:k-amd64", "sha256:k-attestation"], "sha256:d": ["sha256:d-amd64"]} - untagged = [_version(digest, "2026-10-01T00:00:00Z") for digest in ("sha256:k-amd64", "sha256:k-attestation", - "sha256:d-amd64")] - - stale = prune.stale_versions([kept, dropped, *untagged], CUTOFF, lambda digest: children.get(digest, [])) - - assert [version["name"] for version in stale] == ["sha256:d", "sha256:d-amd64"] - - -def test_a_prune_without_the_package_deletes_nothing_passes_and_says_why(prune, monkeypatch, capsys): - def missing(method, url): - raise prune.urllib.error.HTTPError(url, 404, "Not Found", {}, io.BytesIO(b"")) - - monkeypatch.setattr(prune, "_github", missing) - monkeypatch.setattr(sys, "argv", ["prune_computer_images.py", "org", "agentenv-ci-computers"]) - - assert prune.main() == 0 - assert "no package org/agentenv-ci-computers, or this repository has no access to it" in capsys.readouterr().out - - -def test_a_runner_shares_each_image_under_its_mirror_tag_and_todays_seen_tag_and_leaves_its_indexes(monkeypatch, - capsys): +def test_a_runner_shares_each_image_under_its_mirror_tag_and_leaves_its_indexes(monkeypatch, capsys): share = load_script("share_computer_images.py") tags = {"tags": [RAW, f"index-{RAW_OTHER}", RAW_OTHER]} copies = [] @@ -80,14 +22,13 @@ def test_a_runner_shares_each_image_under_its_mirror_tag_and_todays_seen_tag_and monkeypatch.setattr(share, "copy_image", lambda source, target, source_auth, auth: copies.append( (source.rpartition("@")[2], target, auth.username))) monkeypatch.setattr(share, "pin_if_served", lambda ref, auth: f"{ref}@sha256:{RAW}") - monkeypatch.setattr(share.datetime, "datetime", type("Fixed", (datetime.datetime,), {"now": staticmethod(lambda tz: NOW)})) monkeypatch.setenv("GITHUB_ACTOR", "bot") monkeypatch.setenv("GITHUB_TOKEN", "t") monkeypatch.setattr(sys, "argv", ["share_computer_images.py", "ghcr.io/org/agentenv-ci-computers"]) assert share.main() == 0 - assert copies == [(f"sha256:{raw}", f"ghcr.io/org/agentenv-ci-computers:{tag}", "bot") - for raw, key in ((RAW, KEY), (RAW_OTHER, OTHER)) for tag in (key, f"{key}-seen-20261120")] + assert copies == [(f"sha256:{raw}", f"ghcr.io/org/agentenv-ci-computers:{key}", "bot") + for raw, key in ((RAW, KEY), (RAW_OTHER, OTHER))] assert "::warning::" not in capsys.readouterr().out From f7e3eee5a5784d97bc0e6cb350b7e90b40b6aa3f Mon Sep 17 00:00:00 2001 From: Andrew Park Date: Fri, 13 Nov 2026 02:00:39 +0000 Subject: [PATCH 5/6] ci: a slow shard that composed no image shares nothing and passes The runner's registry has no computer repository when the shard composed no image, so listing its tags answers 404. The share script now treats that as nothing to share. It also drops comments that restated the step names or the composer's lookup. --- .github/scripts/share_computer_images.py | 19 +++++++++--------- .github/workflows/local-backends.yml | 4 ---- .../test_computer_image_mirror_scripts.py | 20 +++++++++++++++++-- 3 files changed, 28 insertions(+), 15 deletions(-) diff --git a/.github/scripts/share_computer_images.py b/.github/scripts/share_computer_images.py index 27c2d8a66..5d3740086 100644 --- a/.github/scripts/share_computer_images.py +++ b/.github/scripts/share_computer_images.py @@ -1,10 +1,5 @@ -"""Copy every composed computer image a slow-tier runner's registry holds to the CI computer image mirror, the one -``AGENT_ENV_COMPUTER_IMAGE_MIRROR`` names, so later runs copy them back rather than build them. - -Each image goes under its mirror tag (``composed.mirror_tag``: the composer's digest, then the key). No blob the mirror -already holds is sent again. The mirror is written as ``GITHUB_ACTOR`` with ``GITHUB_TOKEN``; one the composer can't -read anonymously, as GHCR makes a new package, warns. -""" +"""Copy every composed computer image a slow-tier runner's registry holds to the CI computer image mirror, so later runs +copy them back rather than build them.""" from __future__ import annotations @@ -13,6 +8,7 @@ import os import re import sys +import urllib.error import urllib.request from agent_env.env.envs.computer.utils.composed import REPOSITORY, mirror_tag @@ -28,8 +24,13 @@ def main() -> int: parser.add_argument("--registry", default="localhost:5000", help="the runner's registry") args = parser.parse_args() - with urllib.request.urlopen(f"http://{args.registry}/v2/{REPOSITORY}/tags/list") as response: - keys = sorted(tag for tag in json.load(response).get("tags") or [] if KEY.match(tag)) + try: + with urllib.request.urlopen(f"http://{args.registry}/v2/{REPOSITORY}/tags/list") as response: + keys = sorted(tag for tag in json.load(response).get("tags") or [] if KEY.match(tag)) + except urllib.error.HTTPError as e: + if e.code != 404: + raise + keys = [] auth = RegistryAuth(args.mirror.partition("/")[0], os.environ["GITHUB_ACTOR"], os.environ["GITHUB_TOKEN"]) for key in keys: source = pin_digest(f"{args.registry}/{REPOSITORY}:{key}", None) diff --git a/.github/workflows/local-backends.yml b/.github/workflows/local-backends.yml index 1bd51708a..bdd1c2eca 100644 --- a/.github/workflows/local-backends.yml +++ b/.github/workflows/local-backends.yml @@ -22,7 +22,6 @@ concurrency: env: AGENT_ENV_ENVIRONMENT: dev - # The slow tier's composed computer images, shared between its runners. COMPUTER_IMAGE_MIRROR: ghcr.io/scaleapi/agentenv-ci-computers PIP_DISABLE_PIP_VERSION_CHECK: "1" @@ -196,8 +195,6 @@ jobs: # x64 hosted runners have /dev/kvm; the disk form's tests boot under it, and skip without it, which fails below. - name: Let the local provider boot disks with KVM run: sudo chmod 666 /dev/kvm - # A computer image or disk is tagged with a key of everything it is built from. A compose that finds no image of - # its key in the local registry copies the one the mirror holds, which main's runs share, rather than build it. - name: Slow integration tier on the local defaults id: tests env: @@ -227,7 +224,6 @@ jobs: with: path: ${{ runner.temp }}/packages key: ${{ steps.lockkey.outputs.key }} - # Once main's tests pass, every image this runner built or copied goes to the mirror under its key. - name: Share the computer images continue-on-error: true if: >- diff --git a/tst/unit/test_computer_image_mirror_scripts.py b/tst/unit/test_computer_image_mirror_scripts.py index e87a163a8..622ff2377 100644 --- a/tst/unit/test_computer_image_mirror_scripts.py +++ b/tst/unit/test_computer_image_mirror_scripts.py @@ -1,11 +1,11 @@ -"""What a slow-tier runner shares into the CI computer image mirror.""" - from __future__ import annotations import io import json import sys +import pytest + from tst.util.scripts import load_script KEY, OTHER = "c" * 16 + "-" + "a" * 64, "c" * 16 + "-" + "b" * 64 @@ -48,3 +48,19 @@ def private(ref, auth): assert share.main() == 0 assert "make the package public" in capsys.readouterr().out + + +def test_a_runner_that_composed_no_image_shares_nothing_and_passes(monkeypatch, capsys): + share = load_script("share_computer_images.py") + + def no_repository(url): + raise share.urllib.error.HTTPError(url, 404, "Not Found", {}, io.BytesIO(b"")) + + monkeypatch.setattr(share.urllib.request, "urlopen", no_repository) + monkeypatch.setattr(share, "copy_image", lambda *args: pytest.fail("copied with no image to share")) + monkeypatch.setenv("GITHUB_ACTOR", "bot") + monkeypatch.setenv("GITHUB_TOKEN", "t") + monkeypatch.setattr(sys, "argv", ["share_computer_images.py", "ghcr.io/org/agentenv-ci-computers"]) + + assert share.main() == 0 + assert "0 images" in capsys.readouterr().out From c9f23ddf289c6a6bb37b95f5676ad9aced70b8d4 Mon Sep 17 00:00:00 2001 From: Andrew Park Date: Fri, 13 Nov 2026 02:06:43 +0000 Subject: [PATCH 6/6] refactor(composer): name the mirror tag's prefix length --- src/agent_env/env/envs/computer/utils/composed.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/agent_env/env/envs/computer/utils/composed.py b/src/agent_env/env/envs/computer/utils/composed.py index ec3d5d993..bb13d6b6e 100644 --- a/src/agent_env/env/envs/computer/utils/composed.py +++ b/src/agent_env/env/envs/computer/utils/composed.py @@ -117,10 +117,13 @@ def composition() -> str: return "sha256:" + digest.hexdigest() +MIRROR_PREFIX_LENGTH = 16 + + def mirror_tag(key: str) -> str: """The tag of the image of ``key`` in a computer image mirror: the key, under the start of ``composition()``, so a change to the code that composes an image builds it again rather than reuse one the old code built.""" - return f"{composition().removeprefix('sha256:')[:16]}-{key.removeprefix('sha256:')}" + return f"{composition().removeprefix('sha256:')[:MIRROR_PREFIX_LENGTH]}-{key.removeprefix('sha256:')}" def _rank(result: dict) -> int: