Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
169 changes: 164 additions & 5 deletions .github/scripts/infra_build.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,15 +14,20 @@
build of NAME, runs: a disk form boots in the QEMU launcher image ``--launcher`` names until every program its base's
supervisord.conf declares runs. ``publish MISSING`` signs in to GHCR and tags the two platform builds of each entry of
MISSING, a JSON list, as one multi-platform image under its tag, once the entry is checked, unless the registry has
that tag. ``verify-record PATH`` fails unless PATH records each image agent-env ships once, with the digest the
registry serves under its tag now. ``retry COMMAND...`` runs COMMAND, again while a registry rate-limits it or PyPI
doesn't serve yet the agentenv-framework-protocol an agent image's build installs.
that tag. ``publish-computers FOLDER`` checks the computer apps FOLDER's amd64.json and arm64.json say were composed
(``release_computers.py compose``), signs in to GHCR, joins each app's two images into one index in
``COMPUTERS_REPOSITORY`` from the local registry they were composed into, keeping their digests, and writes their record
as FOLDER/computers.json. ``verify-record PATH`` fails unless PATH records each image agent-env ships once, with the
digest the registry serves under its tag now, and each computer app it records once, its index served as recorded.
``retry COMMAND...`` runs COMMAND, again while a registry rate-limits it or PyPI doesn't serve yet the
agentenv-framework-protocol an agent image's build installs.
"""

from __future__ import annotations

import argparse
import base64
import hashlib
import json
import os
import platform
Expand Down Expand Up @@ -65,6 +70,16 @@
# Each disk form's build args, by the images agent-env ships that they name; its SIZE_GB is its base's [forms.disk]
# min_disk_gb, read from the artifact.toml beside the base's Dockerfile.
DISK_ARGS = {"agentenv-ubuntu-24.04-disk": {"ROOTFS": "agentenv-ubuntu-24.04", "TOOLS": "agentenv-disk-tools"}}
# Each computer app a release publishes composed, by its installed env id (``release_computers.py``), into one
# repository, an index of each one's arches.
COMPUTERS = ("@local/agentenv-framework/browsers/chromium",)
COMPUTERS_REPOSITORY = f"{REGISTRY}/agentenv-computers"
COMPUTER_ARCHES = ("amd64", "arm64")
# A composed image in the local registry it was composed into, tagged by its key.
_COMPOSED = re.compile(r"^localhost:5000/agentenv-computers:([0-9a-f]{64})@(sha256:[0-9a-f]{64})$")
_COMPUTER_IMAGE = re.compile(rf"^{re.escape(COMPUTERS_REPOSITORY)}@sha256:[0-9a-f]{{64}}$")
_COMPUTER_INDEX = re.compile(rf"^{re.escape(COMPUTERS_REPOSITORY)}:index-[0-9a-f]{{64}}@sha256:[0-9a-f]{{64}}$")
_INDEX_TYPES = ("application/vnd.oci.image.index.v1+json", "application/vnd.docker.distribution.manifest.list.v2+json")
_TAG = re.compile(r"^src-[0-9a-f]{64}$")
_DIGEST = re.compile(r"^sha256:[0-9a-f]{64}$")
_HEALTHY_WITHIN_SECONDS = 120
Expand Down Expand Up @@ -156,6 +171,7 @@ def main() -> int:
smoke.add_argument("image")
smoke.add_argument("--launcher")
commands.add_parser("publish").add_argument("missing")
commands.add_parser("publish-computers").add_argument("folder", type=Path)
commands.add_parser("verify-record").add_argument("path", type=Path)
commands.add_parser("retry").add_argument("argv", nargs=argparse.REMAINDER, metavar="COMMAND")
args = parser.parse_args()
Expand All @@ -176,6 +192,8 @@ def main() -> int:
return _smoke(args.name, args.image, args.launcher)
if args.command == "publish":
return _publish(args.missing)
if args.command == "publish-computers":
return _publish_computers(args.folder)
problems = _check(args.entry) if args.command == "check" else _verify_record(args.path)
for problem in problems:
print(f"::error::{problem}")
Expand Down Expand Up @@ -238,6 +256,110 @@ def _publish(missing: str) -> int:
return 0


def _publish_computers(folder: Path) -> int:
"""Push each computer app the ARCH.json files in ``folder`` name to ``COMPUTERS_REPOSITORY`` as one index of its
arches' images, from the local registry they were composed into, each by its platform manifest's digest, which the
copy keeps, so each is the image its compose probed; and write ``folder``/computers.json, the record of them a
release wheel carries, each app's index and each arch's key, image and lock."""
composed, problems = {}, []
for arch in COMPUTER_ARCHES:
try:
entries = json.loads((folder / f"{arch}.json").read_text())
except (OSError, ValueError) as e:
problems.append(f"{folder / f'{arch}.json'} isn't a list of composed apps: {type(e).__name__}: {e}")
continue
problems += _check_composed(arch, entries)
composed[arch] = {entry.get("env"): entry for entry in entries if isinstance(entry, dict)}
if not problems and sorted(composed[arch]) != sorted(COMPUTERS):
problems.append(f"{folder / f'{arch}.json'} composes {sorted(composed[arch])}, not {list(COMPUTERS)}")
for problem in problems:
print(f"::error::{problem}")
if problems:
return 1
_sign_in()
computers = []
for env in COMPUTERS:
built = {arch: composed[arch][env] for arch in COMPUTER_ARCHES}
digests = {arch: _platform_manifest(_COMPOSED.match(entry["image"]).group(2), arch)
for arch, entry in built.items()}
if missing := [arch for arch, digest in digests.items() if digest is None]:
print(f"::error::{env}'s composed {', '.join(missing)} has no linux manifest of its arch")
return 1
images = {arch: f"{COMPUTERS_REPOSITORY}@{digest}" for arch, digest in digests.items()}
index = f"{COMPUTERS_REPOSITORY}:index-" + hashlib.sha256(
",".join(images[arch] for arch in COMPUTER_ARCHES).encode()).hexdigest()
if (code := _exists(index)) == 2:
return 1
if code == 1:
source = f"https://github.com/{os.environ['GITHUB_REPOSITORY']}"
created = _tried_again(lambda: _streamed([
"docker", "buildx", "imagetools", "create", "-t", index,
"--annotation", f"index:org.opencontainers.image.source={source}",
"--annotation", f"index:org.opencontainers.image.revision={os.environ['GITHUB_SHA']}",
*(f"localhost:5000/agentenv-computers@{digests[arch]}" for arch in COMPUTER_ARCHES)]))
if created.returncode:
return created.returncode
served, error = _served(index)
if served is None or _children(index) != set(digests.values()):
print(f"::error::{index} serves {served or f'nothing ({error})'}, and not an index of exactly "
f"{sorted(digests.values())}")
return 1
computers.append({"env": env, "form": "container", "index": f"{index}@{served}", "arches": {
arch: {"key": entry["key"], "image": images[arch], "lock": entry["lock"]}
for arch, entry in built.items()}})
print(f"published {env} as {index}@{served}, {', '.join(f'{arch} {images[arch]}' for arch in COMPUTER_ARCHES)}")
(folder / "computers.json").write_text(json.dumps(computers, indent=2) + "\n")
return 0


def _check_composed(arch: str, entries: object) -> list[str]:
"""Each problem with ``entries``, as ``release_computers.py compose`` writes a list of the apps it composed for
``arch``: an app a release publishes, its form, arch, key, image in the local registry under that key, and lock."""
if not isinstance(entries, list):
return [f"{arch}: {entries!r} isn't a list of composed apps"]
problems = []
for entry in entries:
if not isinstance(entry, dict) or set(entry) != {"env", "form", "arch", "key", "image", "lock"}:
problems.append(f"{arch}: {entry!r} isn't an app's env, form, arch, key, image and lock")
continue
image, lock = _COMPOSED.match(str(entry["image"])), entry["lock"]
if (entry["env"] not in COMPUTERS or entry["form"] != "container" or entry["arch"] != arch or image is None
or entry["key"] != f"sha256:{image.group(1)}" or not isinstance(lock, dict) or lock.get("arch") != arch
or not _DIGEST.match(str(lock.get("inputs")))):
problems.append(f"{arch}: {entry['env']!r} isn't an app a release publishes, composed for {arch} into the "
f"local registry under its key, with its lock: {entry['form']!r} {entry['arch']!r} "
f"{entry['key']!r} {entry['image']!r}")
return problems


def _raw(ref: str) -> dict | None:
"""The manifest the registry serves under ``ref``, as it serves it, or None."""
result = _tried_again(lambda: subprocess.run(["docker", "buildx", "imagetools", "inspect", ref, "--raw"],
capture_output=True, text=True))
try:
return json.loads(result.stdout) if result.returncode == 0 else None
except ValueError:
return None


def _platform_manifest(digest: str, arch: str) -> str | None:
"""The digest of the linux/``arch`` manifest of the local registry's composed image ``digest``: itself when it's a
manifest, else the one its index names for that platform, an attestation's aside."""
manifest = _raw(f"localhost:5000/agentenv-computers@{digest}")
if manifest is None:
return None
if manifest.get("mediaType") not in _INDEX_TYPES and "manifests" not in manifest:
return digest
return next((child["digest"] for child in manifest.get("manifests", [])
if child.get("platform", {}).get("os") == "linux" and child["platform"].get("architecture") == arch),
None)


def _children(index: str) -> set[str]:
"""The digests of the manifests the index under ``index`` names."""
return {child["digest"] for child in (_raw(index) or {}).get("manifests", [])}


def _checked(entry: str) -> bool:
problems = _check(entry)
for problem in problems:
Expand Down Expand Up @@ -367,9 +489,9 @@ def _verify_record(path: Path) -> list[str]:
images = record["images"]
except (OSError, ValueError, KeyError, TypeError) as e:
return [f"{path} isn't a record of published images: {type(e).__name__}: {e}"]
if set(record) != {"images"} or not isinstance(images, list):
if not {"images"} <= set(record) <= {"images", "computers"} or not isinstance(images, list):
return [f"{path} isn't a record of published images: it holds {sorted(record)}"]
problems, seen = [], []
problems, seen = _verify_computers(path, record.get("computers", [])), []
for image in images:
if not isinstance(image, dict) or set(image) != {"repository", "tag", "digest"}:
problems.append(f"{path}: {image!r} isn't an image's repository, tag and digest")
Expand All @@ -394,6 +516,43 @@ def _verify_record(path: Path) -> list[str]:
return problems


def _verify_computers(path: Path, computers: object) -> list[str]:
"""Each problem with ``computers``, as a release wheel records the computer apps it published composed: an app of
``COMPUTERS`` at most once, its index served under its tag with the digest recorded and naming exactly its arches'
images, each by digest, with the key it was composed under and its lock. A release that composed none records
none."""
if not isinstance(computers, list):
return [f"{path}: {computers!r} isn't a list of composed computers"]
problems, seen = [], []
for computer in computers:
if not isinstance(computer, dict) or set(computer) != {"env", "form", "index", "arches"}:
problems.append(f"{path}: {computer!r} isn't a composed computer's env, form, index and arches")
continue
env, arches, index = computer["env"], computer["arches"], str(computer["index"])
seen.append(env)
if env not in COMPUTERS or computer["form"] != "container" or not _COMPUTER_INDEX.match(index):
problems.append(f"{path}: {env!r} isn't a computer app a release publishes, as a container index in "
f"{COMPUTERS_REPOSITORY}: {computer['form']!r} {index!r}")
continue
if not isinstance(arches, dict) or sorted(arches) != sorted(COMPUTER_ARCHES) or not all(
isinstance(built, dict) and set(built) == {"key", "image", "lock"} and _DIGEST.match(str(built["key"]))
and _COMPUTER_IMAGE.match(str(built["image"])) and isinstance(built["lock"], dict)
and built["lock"].get("arch") == arch for arch, built in arches.items()):
problems.append(f"{path}: {env}'s arches aren't {list(COMPUTER_ARCHES)}, each a key, an image by digest in "
f"{COMPUTERS_REPOSITORY} and its lock")
continue
tagged, _, digest = index.partition("@")
served, error = _served(tagged)
if served != digest:
problems.append(f"{path}: {tagged} is recorded as {digest}, and the registry serves "
f"{served or f'nothing ({error})'}")
elif _children(tagged) != {built["image"].partition("@")[2] for built in arches.values()}:
problems.append(f"{path}: {tagged} doesn't name exactly the images recorded for {env}'s arches")
if repeated := sorted({env for env in seen if seen.count(env) > 1}):
problems.append(f"{path} records {', '.join(repeated)} more than once")
return problems


def _config(image: str) -> str:
"""The config object of ``image``'s OCI config, a local image's, as base64'd JSON."""
inspected = subprocess.run(["docker", "image", "inspect", "--format", "{{json .Config}}", image],
Expand Down
44 changes: 27 additions & 17 deletions .github/scripts/infra_images.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,10 @@
only the images that install the protocol, the agents' and the website browser's, which a release builds after it
publishes the protocol, as their protocol floor may name it, and ``--only disks`` the base computers' disk forms, built
over their published images. ``write PATH`` records the digest the registry serves under every
image's tag as a release wheel carries them, and fails if any is missing. ``public`` warns about each image a user, who
pulls anonymously, can't. The registry is read with ``GHCR_TOKEN`` (as ``GITHUB_ACTOR``) when it's set, anonymously
otherwise, and always anonymously by ``public``.
image's tag as a release wheel carries them, and fails if any is missing; with ``--computers FILE``, the composed
computers ``infra_build.py publish-computers`` wrote there too. ``public`` warns about each image a user, who pulls
anonymously, can't, and each composed computer's index the record ``--record`` names. The registry is read with
``GHCR_TOKEN`` (as ``GITHUB_ACTOR``) when it's set, anonymously otherwise, and always anonymously by ``public``.
"""

from __future__ import annotations
Expand All @@ -23,7 +24,9 @@
from pathlib import Path

from agent_env.env.bootstrap import PLAYWRIGHT_MCP_DOCKERFILE, PUBLISHED_IMAGES
from agent_env.env.published_images import BUNDLE_IMAGES, DISK_IMAGES, REGISTRY, PublishedImage, write_release_digests
from agent_env.env.published_images import (
BUNDLE_IMAGES, COMPUTERS_REPOSITORY, DISK_IMAGES, REGISTRY, PublishedImage, released_computers,
write_release_digests)
from agent_env.store.image_store.oci_registry_credentials import RegistryAuth
from agent_env.store.image_store.registry_api import NoSuchImageError, RegistryRefusedError, pin_digest

Expand All @@ -42,23 +45,27 @@ def main() -> int:
plan = commands.add_parser("plan")
plan.add_argument("--missing", action="store_true")
plan.add_argument("--only", choices=sorted(_GROUPS))
commands.add_parser("write").add_argument("path", type=Path)
commands.add_parser("public")
write = commands.add_parser("write")
write.add_argument("path", type=Path)
write.add_argument("--computers", type=Path)
commands.add_parser("public").add_argument("--record", type=Path)
args = parser.parse_args()
if args.command == "plan":
images = _GROUPS[args.only] if args.only else PUBLISHED_IMAGES
print(json.dumps([_described(image) for image in images
if not args.missing or _published_digest(image) is None]))
return 0
if args.command == "public":
_warn_unless_public()
_warn_unless_public(released_computers(args.record) if args.record else [])
return 0
digests = {image.ref(): _digest(image) for image in PUBLISHED_IMAGES}
if missing := sorted(ref for ref, digest in digests.items() if digest is None):
print(f"::error::the registry has no image under {', '.join(missing)}")
return 1
write_release_digests(digests, args.path)
print(f"recorded {len(digests)} published images in {args.path}")
computers = json.loads(args.computers.read_text()) if args.computers else []
write_release_digests(digests, args.path, computers)
released_computers(args.path)
print(f"recorded {len(digests)} published images and {len(computers)} composed computers in {args.path}")
return 0


Expand Down Expand Up @@ -88,17 +95,20 @@ def _published_digest(image: PublishedImage) -> str | None:
return None


def _warn_unless_public() -> None:
"""A warning for each image an anonymous pull doesn't get, which a new package's first push leaves it as: GHCR makes
a package private until its admin makes it public, and an install that can't pull it builds the image instead."""
for image in PUBLISHED_IMAGES:
def _warn_unless_public(computers: list[dict]) -> None:
"""A warning for each image, and each composed computer's index in ``computers``, an anonymous pull doesn't get,
which a new package's first push leaves it as: GHCR makes a package private until its admin makes it public, and an
install that can't pull it builds the image, or composes the computer, instead."""
refs = [(image.name, image.ref()) for image in PUBLISHED_IMAGES]
refs += [(COMPUTERS_REPOSITORY.rpartition("/")[2], computer["index"]) for computer in computers]
for name, ref in refs:
try:
pin_digest(image.ref(), None)
pin_digest(ref, None)
except ValueError as e:
why = str(e).split(": ", 1)[-1].split(";")[0] # what the registry answered, without the ref
settings = f"https://github.com/orgs/{REGISTRY.split('/')[1]}/packages/container/{image.name}/settings"
print(f"::warning title={image.name} isn't public::{image.ref()}: {why}. Make the package public at "
f"{settings}; until then an install can't pull it and builds the image instead.")
settings = f"https://github.com/orgs/{REGISTRY.split('/')[1]}/packages/container/{name}/settings"
print(f"::warning title={name} isn't public::{ref}: {why}. Make the package public at "
f"{settings}; until then an install can't pull it and builds it instead.")


def _auth() -> RegistryAuth | None:
Expand Down
Loading
Loading