Skip to content

Commit d1fcdf8

Browse files
ci: share the slow tier's computer images through a mirror registry rather than per-shard caches (#580)
Co-authored-by: Claude <noreply@anthropic.com>
1 parent eac7660 commit d1fcdf8

13 files changed

Lines changed: 441 additions & 260 deletions

File tree

‎.github/scripts/prune_registry.py‎

Lines changed: 0 additions & 98 deletions
This file was deleted.
Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
"""Copy every composed computer image a slow-tier runner's registry holds to the CI computer image mirror, so later runs
2+
copy them back rather than build them."""
3+
4+
from __future__ import annotations
5+
6+
import argparse
7+
import json
8+
import os
9+
import re
10+
import sys
11+
import urllib.error
12+
import urllib.request
13+
14+
from agent_env.env.envs.computer.utils.composed import REPOSITORY, mirror_tag
15+
from agent_env.store.image_store.oci_registry_credentials import RegistryAuth
16+
from agent_env.store.image_store.registry_api import copy_image, pin_digest, pin_if_served
17+
18+
KEY = re.compile(r"^[0-9a-f]{64}$")
19+
20+
21+
def main() -> int:
22+
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
23+
parser.add_argument("mirror", help="the mirror repository, as ghcr.io/org/agentenv-ci-computers")
24+
parser.add_argument("--registry", default="localhost:5000", help="the runner's registry")
25+
args = parser.parse_args()
26+
27+
try:
28+
with urllib.request.urlopen(f"http://{args.registry}/v2/{REPOSITORY}/tags/list") as response:
29+
keys = sorted(tag for tag in json.load(response).get("tags") or [] if KEY.match(tag))
30+
except urllib.error.HTTPError as e:
31+
if e.code != 404:
32+
raise
33+
keys = []
34+
auth = RegistryAuth(args.mirror.partition("/")[0], os.environ["GITHUB_ACTOR"], os.environ["GITHUB_TOKEN"])
35+
for key in keys:
36+
source = pin_digest(f"{args.registry}/{REPOSITORY}:{key}", None)
37+
copy_image(source, f"{args.mirror}:{mirror_tag(key)}", None, auth)
38+
print(f"shared {key}")
39+
print(f"share computer images: {len(keys)} images in {args.mirror}")
40+
if keys:
41+
try:
42+
pin_if_served(f"{args.mirror}:{mirror_tag(keys[0])}", None)
43+
except ValueError as e:
44+
print(f"::warning::the composer reads {args.mirror} anonymously and can't, so every run builds its images; "
45+
f"make the package public: {e}")
46+
return 0
47+
48+
49+
if __name__ == "__main__":
50+
sys.exit(main())

‎.github/workflows/local-backends.yml‎

Lines changed: 39 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ concurrency:
2222

2323
env:
2424
AGENT_ENV_ENVIRONMENT: dev
25+
COMPUTER_IMAGE_MIRROR: ghcr.io/scaleapi/agentenv-ci-computers
2526
PIP_DISABLE_PIP_VERSION_CHECK: "1"
2627

2728
jobs:
@@ -169,8 +170,14 @@ jobs:
169170
shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
170171
runs-on: ubuntu-latest
171172
timeout-minutes: 40
173+
permissions:
174+
contents: read
175+
packages: write
172176
steps:
173177
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
178+
# The tests run with packages: write; only the share step, on main, gets the token.
179+
with:
180+
persist-credentials: false
174181
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
175182
with:
176183
python-version: "3.12"
@@ -183,29 +190,22 @@ jobs:
183190
# starts afterwards.
184191
- name: Pull the base images from mirror.gcr.io
185192
run: python3 .github/scripts/mirror_base_images.py src tst
186-
# A computer image or disk is tagged with a key of everything it is built from, and the composer reuses one the
187-
# image store already holds for its key. That key includes the lock of the packages the image installs, which the
188-
# composer resolves from Ubuntu's archive, PyPI and download URLs unless the store holds it. The registry keeps
189-
# its storage on the runner, and the computer tests share their locks and files in the packages folder
190-
# (tst/util/package_cache.py); both are restored from what main saved latest this week, so a run neither resolves
191-
# nor builds again what hasn't changed. A shard that runs no computer tests misses.
192-
- name: Week of the computer images' cache
193+
# The computer tests share the package locks a compose resolves, and the files they name, in the packages folder
194+
# (tst/util/package_cache.py). An image's key includes its lock, which the composer otherwise resolves from
195+
# Ubuntu's archive, PyPI and download URLs, so the folder is restored from what main saved latest this week.
196+
- name: Week of the packages' cache
193197
id: week
194198
run: echo "week=$(date -u +%G-W%V)" >> "$GITHUB_OUTPUT"
195-
- name: Restore the computer images and packages
196-
id: computers
199+
- name: Restore the packages
200+
id: packages
197201
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
198202
with:
199-
path: |
200-
${{ runner.temp }}/registry
201-
${{ runner.temp }}/packages
202-
key: computers-${{ matrix.shard }}-${{ steps.week.outputs.week }}
203-
restore-keys: computers-${{ matrix.shard }}-${{ steps.week.outputs.week }}-
203+
path: ${{ runner.temp }}/packages
204+
key: packages-${{ matrix.shard }}-${{ steps.week.outputs.week }}
205+
restore-keys: packages-${{ matrix.shard }}-${{ steps.week.outputs.week }}-
204206
- name: Start the local registry
205207
run: |
206-
mkdir -p "$RUNNER_TEMP/registry"
207-
docker run --detach --name registry --publish 5000:5000 --volume "$RUNNER_TEMP/registry:/var/lib/registry" \
208-
mirror.gcr.io/library/registry:2
208+
docker run --detach --name registry --publish 5000:5000 mirror.gcr.io/library/registry:2
209209
timeout 30 sh -c 'until curl -sf http://localhost:5000/v2/ >/dev/null; do sleep 1; done'
210210
# x64 hosted runners have /dev/kvm; the disk form's tests boot under it, and skip without it, which fails below.
211211
- name: Let the local provider boot disks with KVM
@@ -215,49 +215,37 @@ jobs:
215215
env:
216216
AGENTENV_TEST_SHARD: ${{ matrix.shard }}/12
217217
AGENTENV_TEST_PACKAGES: ${{ runner.temp }}/packages
218+
AGENT_ENV_COMPUTER_IMAGE_MIRROR: ${{ env.COMPUTER_IMAGE_MIRROR }}
218219
run: |
219220
.venv/bin/python -m pytest -m int_test_slow -n 2 --dist loadfile -rA --timeout=1200 \
220221
--junitxml=junit-integration-slow.xml tst/integration --ignore=tst/integration/env/gateway/gateway_test.py
221-
# Only the computer images this run used are kept, with the packages, and only main saves them, once its tests
222-
# pass, so pull requests reuse main's without filling the repository's cache. An image is used when the registry's
223-
# log has a request for it by its tag (prune_registry.py), so what a week's earlier runs built and this one no
224-
# longer asks for is dropped. The garbage collection frees what only the dropped tags and repositories referenced;
225-
# with --delete-untagged it would also delete the untagged platform manifests of an image index, and their layers.
226-
# The entry is named by the image keys the registry holds and the locks the packages hold, so a run that built a
227-
# new image or resolved a new lock saves a new one.
228-
- name: Keep only the computer images this run used
229-
id: kept
222+
# Only main saves the packages, once its tests pass, so pull requests reuse main's without filling the
223+
# repository's cache. The entry is named by the locks the folder holds, so a run that resolved a new lock saves a
224+
# new one.
225+
- name: Name the packages' cache entry
226+
id: lockkey
230227
if: always()
231228
run: |
232-
repositories="$RUNNER_TEMP/registry/docker/registry/v2/repositories"
233-
if [ ! -d "$repositories/agentenv-computers" ]; then echo "computers=false" >> "$GITHUB_OUTPUT"; exit 0; fi
234-
docker logs registry > "$RUNNER_TEMP/registry.log" 2>&1
235-
docker stop registry
236-
sudo find "$repositories" -mindepth 1 -maxdepth 1 ! -name agentenv-computers -exec rm -rf {} +
237-
sudo python3 .github/scripts/prune_registry.py prune "$RUNNER_TEMP/registry" agentenv-computers \
238-
"$RUNNER_TEMP/registry.log"
239-
docker run --rm --volume "$RUNNER_TEMP/registry:/var/lib/registry" mirror.gcr.io/library/registry:2 \
240-
garbage-collect /etc/docker/registry/config.yml > /dev/null
241-
sudo python3 .github/scripts/prune_registry.py links "$RUNNER_TEMP/registry" agentenv-computers
242-
mkdir -p "$RUNNER_TEMP/packages"
243-
du -sh "$RUNNER_TEMP/registry" "$RUNNER_TEMP/packages"
244-
if [ -z "$(sudo ls "$repositories/agentenv-computers/_manifests/tags")" ]; then
245-
echo "computers=false" >> "$GITHUB_OUTPUT"; exit 0
246-
fi
247-
key=$({ sudo ls "$repositories/agentenv-computers/_manifests/tags"
248-
ls "$RUNNER_TEMP/packages/agentenv-packages/locks" 2>/dev/null; } | sort | sha256sum | cut -c1-16)
249-
echo "key=computers-${{ matrix.shard }}-${{ steps.week.outputs.week }}-$key" >> "$GITHUB_OUTPUT"
250-
echo "computers=true" >> "$GITHUB_OUTPUT"
251-
- name: Save the computer images and packages
229+
locks="$RUNNER_TEMP/packages/agentenv-packages/locks"
230+
if [ -z "$(ls "$locks" 2>/dev/null)" ]; then echo "packages=false" >> "$GITHUB_OUTPUT"; exit 0; fi
231+
key=$(ls "$locks" | sort | sha256sum | cut -c1-16)
232+
echo "key=packages-${{ matrix.shard }}-${{ steps.week.outputs.week }}-$key" >> "$GITHUB_OUTPUT"
233+
echo "packages=true" >> "$GITHUB_OUTPUT"
234+
- name: Save the packages
252235
if: >-
253236
always() && github.event_name == 'push' && steps.tests.outcome == 'success'
254-
&& steps.kept.outputs.computers == 'true' && steps.kept.outputs.key != steps.computers.outputs.cache-matched-key
237+
&& steps.lockkey.outputs.packages == 'true' && steps.lockkey.outputs.key != steps.packages.outputs.cache-matched-key
255238
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
256239
with:
257-
path: |
258-
${{ runner.temp }}/registry
259-
${{ runner.temp }}/packages
260-
key: ${{ steps.kept.outputs.key }}
240+
path: ${{ runner.temp }}/packages
241+
key: ${{ steps.lockkey.outputs.key }}
242+
- name: Share the computer images
243+
continue-on-error: true
244+
if: >-
245+
always() && steps.tests.outcome == 'success' && github.event_name == 'push'
246+
env:
247+
GITHUB_TOKEN: ${{ github.token }}
248+
run: .venv/bin/python .github/scripts/share_computer_images.py "$COMPUTER_IMAGE_MIRROR"
261249
- name: Skips must be declared capability gaps
262250
if: always()
263251
run: |

‎AGENTS.md‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -393,9 +393,8 @@ orchestrators call it the same way. What callers rely on:
393393
`.github/scripts/slow_test_durations.py`.
394394
- The computer tests compose in stores of their own, but share the package locks and files a compose
395395
resolves (`tst/util/package_cache.py`) through `AGENTENV_TEST_PACKAGES`, a folder for the run unless
396-
it's set. The slow tier caches that folder with the local registry's computer images, so a run
397-
resolves from Ubuntu's archive, PyPI or a download URL only what changed since `main` last saved
398-
them.
396+
it's set. The slow tier caches that folder, so a run resolves from Ubuntu's archive, PyPI or a
397+
download URL only what changed since `main` last saved it.
399398
- `tst/manual/` and `tst/benchmarks/` run only by hand, as each module's docstring says; no CI job
400399
collects them.
401400
- `tst/formal/` holds TLA+ models, run only by hand with each model's `check.sh`; the `AGENTS.md`

‎src/agent_env/config/describe.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -673,6 +673,7 @@ def env_sources() -> tuple[EnvSource, ...]:
673673
EnvSource("AGENT_ENV_MODAL_REGION", None, "no file equivalent"),
674674
EnvSource("AGENT_ENV_MODAL_APP_NAME", None, "no file equivalent"),
675675
EnvSource("AGENT_ENV_FIXTURE_PREFIX", None, "no file equivalent"),
676+
EnvSource("AGENT_ENV_COMPUTER_IMAGE_MIRROR", None, "no file equivalent"),
676677
EnvSource("MODAL_TOKEN_ID", None, "credential, no file equivalent"),
677678
EnvSource("MODAL_TOKEN_SECRET", None, "credential, no file equivalent"),
678679
]

‎src/agent_env/config/runtime.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,11 @@
2424
objects, image builds, env and agent snapshots, changelogs, default agent and judge
2525
trajectories, verifier outputs and the A2A validator's fixtures, so a fresh control-plane
2626
backend can write without colliding with existing objects in a shared store. Empty in prod.
27+
28+
AGENT_ENV_COMPUTER_IMAGE_MIRROR (default ""): a registry repository, read anonymously, holding composed computer images
29+
under their keys and the composer's own digest, as ``ghcr.io/org/computers``. A compose that finds no image of a key in the
30+
env's image store copies the mirror's into it instead of building it. CI sets it so its runners share the images main
31+
built.
2732
"""
2833

2934
from __future__ import annotations
@@ -226,6 +231,9 @@ class Config:
226231
fixture_prefix: str = field(
227232
default_factory=lambda: os.getenv("AGENT_ENV_FIXTURE_PREFIX", "")
228233
)
234+
computer_image_mirror: str = field(
235+
default_factory=lambda: os.getenv("AGENT_ENV_COMPUTER_IMAGE_MIRROR", "")
236+
)
229237
agent_sandbox_mode: str = field(
230238
default_factory=lambda: os.getenv("AGENT_SANDBOX_MODE") or _default_sandbox_mode()
231239
)

‎src/agent_env/env/envs/computer/utils/composed.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,15 @@ def composition() -> str:
117117
return "sha256:" + digest.hexdigest()
118118

119119

120+
MIRROR_PREFIX_LENGTH = 16
121+
122+
123+
def mirror_tag(key: str) -> str:
124+
"""The tag of the image of ``key`` in a computer image mirror: the key, under the start of ``composition()``, so a
125+
change to the code that composes an image builds it again rather than reuse one the old code built."""
126+
return f"{composition().removeprefix('sha256:')[:MIRROR_PREFIX_LENGTH]}-{key.removeprefix('sha256:')}"
127+
128+
120129
def _rank(result: dict) -> int:
121130
"""How far a pair got: ready, then built and either unprobed or failed at its probe, then anything short of an
122131
image, and below that one another composer recorded. A result takes a stored one's place only when it got at least

‎src/agent_env/env/envs/computer/utils/composer.py‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -50,15 +50,15 @@
5050
thin_disk,
5151
windows_disk,
5252
)
53-
from agent_env.env.envs.computer.utils.composed import BUILT, REPOSITORY, Record, failed, now, stale
53+
from agent_env.env.envs.computer.utils.composed import BUILT, REPOSITORY, Record, failed, mirror_tag, now, stale
5454
from agent_env.env.envs.computer.utils.install import Install, for_os, platform_of
5555
from agent_env.env.published_images import DISK, DISK_TOOLS_IMAGE, published_disk, published_pinned
5656
from agent_env.providers.sandbox_providers.host_images import platform_tag
5757
from agent_env.store.ids import derive_id
5858
from agent_env.store.image_store.image_contents import image_config, labels
5959
from agent_env.store.image_store.oci_registry_credentials import names_registry
6060
from agent_env.store.image_store.image_store import ImageStore
61-
from agent_env.store.image_store.registry_api import pin_digest, pin_if_served, write_index
61+
from agent_env.store.image_store.registry_api import copy_image, pin_digest, pin_if_served, write_index
6262
from agent_env.store.local_state import holding_locks
6363
from agent_env.utils.docker.build import DockerBuildError
6464

@@ -771,9 +771,20 @@ def _key(**inputs: Any) -> str:
771771

772772
def _stored(env: ComputerEnv, key: str) -> tuple[ImageStore, str, str | None]:
773773
"""The env's image store, the ref there of the image of ``key``, and that ref pinned when the store serves one
774-
already, for any env that built it."""
774+
already, for any env that built it, or once it's copied there from the configured computer image mirror."""
775775
store = _image_store(env)
776-
ref = store.image_ref(REPOSITORY, key.removeprefix("sha256:"))
776+
tag = key.removeprefix("sha256:")
777+
ref = store.image_ref(REPOSITORY, tag)
778+
if (pinned := pin_if_served(ref, store.auth(ref))) is not None or not (mirror := get_config().computer_image_mirror):
779+
return store, ref, pinned
780+
try:
781+
if (mirrored := pin_if_served(f"{mirror}:{mirror_tag(tag)}", None)) is None:
782+
return store, ref, None
783+
copy_image(mirrored, ref, None, store.auth(ref))
784+
except (ValueError, KeyError, OSError) as e:
785+
logger.warning(f"the computer image mirror {mirror} couldn't give {tag}, so it's built: {e}")
786+
return store, ref, None
787+
logger.info(f"copied {tag} from the computer image mirror {mirror}")
777788
return store, ref, pin_if_served(ref, store.auth(ref))
778789

779790

0 commit comments

Comments
 (0)