diff --git a/.github/workflows/local-backends.yml b/.github/workflows/local-backends.yml index 5b58947c2..ed7fe161d 100644 --- a/.github/workflows/local-backends.yml +++ b/.github/workflows/local-backends.yml @@ -229,7 +229,8 @@ jobs: if: always() run: | .venv/bin/python .github/scripts/check_skip_policy.py junit-integration-slow.xml \ - --allow model_endpoint_configured remote_sandbox default_a2a_agent mcp_server_sources windows_media + --allow model_endpoint_configured remote_sandbox default_a2a_agent mcp_server_sources windows_media \ + omarchy_image # The tests' times, for .github/scripts/slow_test_durations.py to rebalance the shards with. - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 if: always() diff --git a/AGENTS.md b/AGENTS.md index 62d6457c7..1d233ec44 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -137,7 +137,7 @@ and deserialized through a registry. | `explorer/` | Optional local web UI: `agent-env up`, binds loopback `:8234`. | | `cli/` | Click CLI with the groups `a2a-agent`, `artifact`, `bundle`, `config`, `env`, `eval`, `plugin`, `run`, `task`, `up`. `task run`, `task run-batch` and `eval run` hand their runs to the configured `[runner]` and follow them. | | `agents/` | The `agents` bundle: the agents agent-env ships, one folder each under `agents/agents/`, with a smoke task each under `agents/tasks/`. Its Claude Code, `@local/agentenv-framework/agents/claude-code`, is the default agent. An agent's Python packages are its folder's `requirements.txt`, installed in its image, never in agent-env's own environment. A run that pulls one of its agents reads the whole bundle, so `tst/unit/bundle/shipped_bundles_test.py` parses and checks every shipped bundle. | -| `computers/` | The `computers` bundle: the base computers agent-env ships as `computer` artifacts, `ubuntu-24.04` (XFCE on Xvfb, supervisord as the session user, settings as `AGENTENV_SETTING_`), aliased through `[artifacts]`. It has no tasks either. `disk/linux/` is the Linux disk form's recipe: `boot/`, whose Dockerfile makes a disk form (a GPT disk of an EFI system partition holding one UKI, Ubuntu's kernel with a busybox initramfs whose init reads a NoCloud seed and `switch_root`s into the image's entrypoint, and an ext4 root filesystem that is a container image's, plus `/etc/agentenv/vm/image.env` from its OCI config), `tools/`, the disk tools that make it unprivileged on the builder's own platform (`mkfs.ext4 -d`), and `launcher/`, the QEMU launcher that boots one under KVM. A base's `[forms.disk]` with no image is made from its container form; the composer makes each arch's disk from its composed container, keeping the base disk's EFI system partition, keyed by the container's key, the base disk and the recipe; gate 3 probes it where a provider boots disks natively (`disk_image_platforms()`), and a deploy's first boot with KVM probes one left `unverified`; a deploy naming `form = "disk"` boots it through `create_vm(image=..., env=...)`, its settings in the seed. | +| `computers/` | The `computers` bundle: the base computers agent-env ships as `computer` artifacts, `ubuntu-24.04` (XFCE on Xvfb, supervisord as the session user, settings as `AGENTENV_SETTING_`), and `omarchy-4.5`, a disk alone built on the composing machine from Omarchy's own VM image (its `[forms.disk]` `source`) by `disk/omarchy/`'s recipe, its services systemd user units of the graphical session that start through `with-seed`, booted by the launcher as a `linux-desktop` guest (`env/envs/computer/utils/omarchy_disk.py`, on `thin_disk.py`, which Windows' disks use too), aliased through `[artifacts]`. It has no tasks either. `disk/linux/` is the Linux disk form's recipe: `boot/`, whose Dockerfile makes a disk form (a GPT disk of an EFI system partition holding one UKI, Ubuntu's kernel with a busybox initramfs whose init reads a NoCloud seed and `switch_root`s into the image's entrypoint, and an ext4 root filesystem that is a container image's, plus `/etc/agentenv/vm/image.env` from its OCI config), `tools/`, the disk tools that make it unprivileged on the builder's own platform (`mkfs.ext4 -d`), and `launcher/`, the QEMU launcher that boots one under KVM. A base's `[forms.disk]` with no image is made from its container form; the composer makes each arch's disk from its composed container, keeping the base disk's EFI system partition, keyed by the container's key, the base disk and the recipe; gate 3 probes it where a provider boots disks natively (`disk_image_platforms()`), and a deploy's first boot with KVM probes one left `unverified`; a deploy naming `form = "disk"` boots it through `create_vm(image=..., env=...)`, its settings in the seed. | | `browsers/` | The `browsers` bundle: the website browsers agent-env ships, today `chromium`, a `computer_app` env (Chrome for Testing at the build Playwright pins, for amd64 and arm64, set up for unattended use by managed policies in `/etc/opt/chrome_for_testing/policies/managed`, opened at boot on its `start_urls`, its profile its data). All of it, its pin, install step, policies and command, is in `envs/chromium/chromium.py`; `tst/integration/computer/chromium_app_test.py` runs it. Like `drivers`, it has no tasks. | | `drivers/` | The `drivers` bundle: the computer drivers agent-env ships as `computer_driver` envs, the reference driver (X11 on Linux, recording through `x11grab`) aliased `reference`, and trycua's cua-driver aliased `cua-driver` (its daemon from the pinned PyPI wheel, its own instructions, tools and skill resources served to agents through `@server_instructions`, `forwarded_tools` and `@resource`, its tools beside `computer_*`, `computer/v1` on its desktop-scope calls, its skill pack leading the computer skill; `tst/integration/computer/cua_driver_test.py` runs it). It has no tasks: a bundle with neither tasks nor evals is refused unless its `aliases.toml` names something for other bundles to use. The shipped `desktop` example composes it with the `ubuntu-24.04` base into a computer env, which `tst/integration/computer/` deploys, probes, records and has an agent use. | | `examples/` | The example bundles agent-env ships (`hello`, and `echo-mcp`, which needs Docker, and `desktop`, a computer env of `ubuntu-24.04` and `reference` with a probe task and an agent task). Tests run them, but they are examples for users, not fixtures. Every shipped bundle is registered under `agent_env.bundles` in `pyproject.toml`, ` = ""` shipping `src///`. | diff --git a/src/agent_env/artifact/artifacts/computer.py b/src/agent_env/artifact/artifacts/computer.py index b193393ff..e7e3d2d88 100644 --- a/src/agent_env/artifact/artifacts/computer.py +++ b/src/agent_env/artifact/artifacts/computer.py @@ -22,6 +22,8 @@ model_validator, ) +from agentenv_protocol.computer.platform import entry_for, platform_key + from agent_env.artifact.artifact import Artifact from agent_env.artifact.artifacts.docker_image import DockerImageArtifact from agent_env.artifact.ref import ArtifactRef @@ -89,11 +91,13 @@ class WindowsOS(BaseModel): #: The OS types a computer is, by ``os.type``. OS_TYPES: dict[str, type[BaseModel]] = {"linux": LinuxOS, "macos": MacOS, "windows": WindowsOS} -#: The forms each OS type's computers have: a Linux computer is a container and may be a disk too, a macOS one a VM -#: bundle alone, a Windows one a disk alone. -OS_FORMS = {"linux": ("container", "disk"), "macos": ("vm_bundle",), "windows": ("disk",)} -#: The init that starts a disk's programs on each OS type with a disk form. -DISK_INITS = {"linux": "supervisord", "windows": "agentenv-session"} +#: The forms each platform's computers have, by platform key, a distribution's replacing its OS type's: a Linux computer +#: is a container and may be a disk too, an Omarchy one a disk alone, as Hyprland needs a GPU a container doesn't get, a +#: macOS one a VM bundle alone, a Windows one a disk alone. The first is the one every computer of it has. +OS_FORMS = {"linux": ("container", "disk"), "linux/omarchy": ("disk",), "macos": ("vm_bundle",), "windows": ("disk",)} +#: The init that starts a disk's programs on each platform with a disk form, by platform key: on Linux the boot recipe's +#: supervisord, on Omarchy systemd's user units in its graphical session, on Windows agentenv-session. +DISK_INITS = {"linux": "supervisord", "linux/omarchy": "systemd", "windows": "agentenv-session"} class Session(BaseModel): @@ -136,17 +140,29 @@ class ContainerForm(BaseModel): input_channel: Literal["env"] +class DiskSource(BaseModel): + """The VM image a disk's base is built from: a distribution's own, fetched by ``url`` and checked by ``sha256``.""" + + model_config = ConfigDict(extra="forbid") + + url: str = Field(pattern=r"^https://\S+$") + sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + + class DiskForm(BaseModel): """The computer as a disk a VM boots from UEFI, whose init reads deploy settings from a NoCloud seed (``input_channel``): on Linux, an EFI system partition holding one UKI and its root filesystem, the image's own - ``init`` started; on Windows, agentenv-session at the session user's sign-in. With no ``image``, it's made when - composed: the container form's root filesystem on agent-env's Linux boot recipe, or Windows installed from - Microsoft's media on this machine (``computers/disk/windows``).""" + ``init`` started; on Omarchy, systemd's user units in its graphical session; on Windows, agentenv-session at the + session user's sign-in. With no ``image``, it's made when composed: from its ``source``, a distribution's own VM + image, on its recipe; else from the container form's root filesystem on agent-env's Linux boot recipe, or Windows + installed from Microsoft's media on this machine (``computers/disk/windows``). A disk that names no source is + written without one, so an agent-env from before sources still reads it.""" model_config = ConfigDict(extra="forbid") image: ArtifactRef | None = None - init: Literal["supervisord", "agentenv-session"] + source: DiskSource | None = Field(default=None, exclude_if=lambda source: source is None) + init: Literal["supervisord", "systemd", "agentenv-session"] input_channel: Literal["nocloud"] firmware: list[Literal["efi"]] = Field(default=["efi"], min_length=1) min_disk_gb: int = Field(ge=1, description="Its virtual size, which a deploy's disk is at least") @@ -238,21 +254,32 @@ def _os_type_supported(cls, value: Any) -> Any: @model_validator(mode="after") def _forms_fit_os(self) -> Self: - allowed = OS_FORMS[self.os.type] + platform = platform_key(self.os.type, getattr(self.os, "distribution", None)) + allowed, kind = entry_for(OS_FORMS, platform), platform if platform in OS_FORMS else self.os.type if others := [form for form in FORMS if getattr(self.forms, form) is not None and form not in allowed]: - raise ValueError(f"forms.{others[0]}: a {self.os.type} computer's forms are {' and '.join(allowed)}") + raise ValueError(f"forms.{others[0]}: a {kind} computer's forms are {' and '.join(allowed)}") if getattr(self.forms, allowed[0]) is None: - raise ValueError(f"forms.{allowed[0]} is required: every {self.os.type} computer has one") + raise ValueError(f"forms.{allowed[0]} is required: every {kind} computer has one") if self.os.type == "macos" and self.arch != ["arm64"]: raise ValueError(f"arch: a macOS VM bundle runs on Apple silicon, arm64 alone, not {self.arch}") if self.os.type == "windows" and self.arch != ["amd64"]: raise ValueError(f"arch: a Windows computer is amd64 alone, not {self.arch}") - if self.forms.disk is not None and self.forms.disk.init != DISK_INITS[self.os.type]: - raise ValueError(f"forms.disk.init: a {self.os.type} disk's init is {DISK_INITS[self.os.type]}, not " - f"{self.forms.disk.init}") - if self.os.type == "windows" and self.forms.disk.image is not None: - raise ValueError("forms.disk.image: a Windows disk is installed from Microsoft's media on the machine that " - "composes it, so it names no image") + disk = self.forms.disk + if disk is not None and disk.init != (init := entry_for(DISK_INITS, platform)): + raise ValueError(f"forms.disk.init: a {platform if platform in DISK_INITS else self.os.type} disk's init is " + f"{init}, not {disk.init}") + if disk is not None and disk.image is not None and disk.source is not None: + raise ValueError("forms.disk: a disk names an image or the source its base is built from, not both") + if platform == "linux/omarchy" and disk is not None and (disk.image is not None or disk.source is None): + raise ValueError("forms.disk: an Omarchy disk's base is built from Omarchy's own image on the machine that " + "composes it, so it names that image's source and no image") + if self.os.type == "windows" and (disk.image is not None or disk.source is not None): + raise ValueError("forms.disk: a Windows disk is installed from Microsoft's media on the machine that " + "composes it, so it names no image or source") + if (self.os.type == "linux" and disk is not None and self.forms.container is None and disk.image is None + and disk.source is None): + raise ValueError("forms.disk: with no container form to be made from, a Linux disk names its image or the " + "source its base is built from") return self @field_validator("forms", mode="before") @@ -295,7 +322,9 @@ def put(cls, *, inspect: bool = True, **fields: Any) -> Self: checks["vm_bundle"] = _unread(computer.arch, "a VM bundle is read by the host that boots it") if disk is not None and disk.image is None: checks["disk"] = _unread(computer.arch, "it's installed from Microsoft's media when composed" - if computer.os.type == "windows" else "it's made from the container form when composed") + if computer.os.type == "windows" else + "it's built from its source when composed, which checks its os-release" + if disk.source is not None else "it's made from the container form when composed") elif disk is not None: checks["disk"] = (_inspect(computer.disk_image(), computer.arch, functools.partial(_read_disk, disk.min_disk_gb)) diff --git a/src/agent_env/computers/README.md b/src/agent_env/computers/README.md index c5e8ca415..195c31ec9 100644 --- a/src/agent_env/computers/README.md +++ b/src/agent_env/computers/README.md @@ -191,5 +191,35 @@ Administrators reach; it launches the app in a Job object, which ends whatever t `agentadmin`, so an app's install step for the session user can't run on Windows, and gate 1 refuses an app that declares one there. +`artifacts/omarchy-4.5/` is Omarchy 4.5 (`amd64`), an Arch Linux desktop on Hyprland, as a disk alone: Hyprland needs a +GPU, which a container doesn't get. Its base is built from Omarchy's own stable VM image, which its `[forms.disk]` names +by `source`, a URL and a sha256, fetched once into this machine's cache and checked. It's booted once by the QEMU +launcher as a `linux-desktop` guest (a virtio GPU at the computer's resolution, 4 vCPUs and 8 GB), with root's ssh key +passed as a systemd credential, Omarchy signing its user `omarchy` in as it does at every boot, and made a base by +`disk/omarchy/recipe.sh`, run as root over ssh. The recipe refuses an image whose os-release isn't the artifact's +`omarchy` `4.5.0`; installs cua-driver's Hyprland plugin, pinned by URL and sha256 to the package Omarchy builds for its +Hyprland; adds `agentenv.lua` to Hyprland's config (the plugin loaded at start, a US keymap, the idle cursor hidden, the +display's mode); installs the seed unit; has SDDM log back in after a logout; sets Omarchy's toggles for an unattended +desktop (staying awake, no animations, no screensaver or suspend, notifications silenced); turns off crash toasts, +dictation and its keys, housekeeping timers that would run at boot, and the firewall, which lets in only ssh and +LocalSend while the launcher forwards only the computer's own ports; and locks `omarchy`'s password and leaves the +machine id empty. Trimmed, the image becomes the base's, with the key to root, and is pushed to the env's image store +under a tag of its recipe and source. Name it as `omarchy-4.5`, or by its id, +`@local/agentenv-framework/computers/omarchy-4.5`. + +An Omarchy computer env is composed on the same machine as a thin disk too, with what's Omarchy's own in +`omarchy_disk.py`: each layer is installed over ssh as root by a bash script composing writes, as its +`[install."linux/omarchy"]` says (its files under `/opt/agentenv/drivers/`, its downloads checked by sha256, its +venv on the system Python, its run scripts), and its services become systemd user units of the graphical session, which +start with the session, and again with it, through `with-seed`. The composed disk has no ssh: only a compose logs +in, and it boots the base. At every boot, before SDDM starts the session, +`agentenv-seed.service` reads the deploy's environment from the NoCloud seed with the Linux disk's own `seed-env` and +writes it for `with-seed`, root's and the session user's alone, the seed's trust files first; it sets the hostname and +the computer's `timezone` (UTC by default; a deploy is refused one that isn't an IANA zone, and one the disk doesn't +have is written as the refusal every service then names), names `host.docker.internal` at QEMU's address for the launcher, and writes the display's mode for Hyprland +from the computer's `resolution`. Gate 3 probes it at full. An Omarchy computer has no apps for now: gate 1 refuses one. When +Hyprland crashes, its watchdog brings it back in safe mode, without cua-driver's plugin, and cua-driver reports the +computer failed for good. + This bundle has no tasks of its own: other bundles pull its computers by alias, and `agent-env run computers` has nothing to run. diff --git a/src/agent_env/computers/aliases.toml b/src/agent_env/computers/aliases.toml index 720e5ae64..81e385b56 100644 --- a/src/agent_env/computers/aliases.toml +++ b/src/agent_env/computers/aliases.toml @@ -4,3 +4,4 @@ "ubuntu-24.04-headless" = ["ubuntu-24.04-headless"] "macos-26" = ["macos-26"] "windows-11" = ["windows-11"] +"omarchy-4.5" = ["omarchy-4.5"] diff --git a/src/agent_env/computers/artifacts/omarchy-4.5/artifact.toml b/src/agent_env/computers/artifacts/omarchy-4.5/artifact.toml new file mode 100644 index 000000000..5eb444870 --- /dev/null +++ b/src/agent_env/computers/artifacts/omarchy-4.5/artifact.toml @@ -0,0 +1,20 @@ +type = "computer" +description = "Omarchy 4.5, an Arch Linux desktop on Hyprland, built from Omarchy's own stable VM image on the Linux machine with KVM that composes it; its user is signed in at boot" +arch = ["amd64"] +os = { type = "linux", distribution = "omarchy", version = "4.5.0", display = "wayland" } +session = { user = "omarchy" } + +[forms.disk] +source = { url = "https://nightly.omarchy.org/stable/4.5.0/479/omarchy-4.5.0.qcow2", sha256 = "dcb401aabf1b01bf3c983a1b048bcd0ac7d66e4f5a755acd9452bf66e18cd5c4" } +init = "systemd" +input_channel = "nocloud" +firmware = ["efi"] +min_disk_gb = 40 + +[config.resolution] +default = "1920x1080" +phase = "deploy" + +[config.timezone] +default = "UTC" +phase = "deploy" diff --git a/src/agent_env/computers/disk/linux/launcher/launch b/src/agent_env/computers/disk/linux/launcher/launch index a9b66d1c3..ad6a630ee 100755 --- a/src/agent_env/computers/disk/linux/launcher/launch +++ b/src/agent_env/computers/disk/linux/launcher/launch @@ -5,7 +5,9 @@ # windows guest, which has no virtio drivers, boots from secure-boot UEFI with Microsoft's keys and a fresh TPM 2.0, # its disk on AHCI, an SSD to Windows, whose TRIM frees the overlay's clusters, its seed on a removable USB drive, since # Windows mounts a FAT filesystem with no partition table on removable media alone, an e1000e NIC, a 1920x1080 display -# and, under KVM, Hyper-V enlightenments. +# and, under KVM, Hyper-V enlightenments. A linux-desktop guest, a Wayland desktop, on amd64 alone, boots as a linux one +# does and gets a virtio GPU at the computer's resolution (its AGENTENV_SETTING_RESOLUTION, 1920x1080 when it has none), +# 4 CPUs and 8192 MiB unless sized, TRIM to its disk, and the systemd credentials VM_CREDENTIALS names over SMBIOS. # VM_DISK the disk, /disk/disk.img by default, which may be backed by another beside it; one ending .zst is # expanded onto this machine's disk first # VM_BASE the base of a thin VM_DISK ending .zst, compressed too, which a provider carries beside it: expanded @@ -18,7 +20,7 @@ # for here: socat serves each on 127.0.0.1 here, through to the host, as a local grant server's # VM_TRUST a folder of trust files, a local transfer CA's, that go into the seed as its trust/ folder # VM_ARCH the disk's arch, amd64 or arm64; this machine's by default -# VM_CPUS, VM_MEMORY_MB its size: 2 and 4096 by default +# VM_CPUS, VM_MEMORY_MB its size: 2 and 4096 by default, 4 and 8192 for a linux-desktop guest # VM_REQUIRE_KVM when set, fail rather than emulate where KVM can't run the disk # VM_STATE where its overlay, its seed and an expanded disk go: /var/lib/agentenv-vm by default # VM_KEEP a builder's overlay: boot on this file, kept across the guest's restarts and after it powers off, @@ -27,6 +29,8 @@ # VM_MEDIA for a windows guest, an installer ISO on a CD-ROM it boots first; Enter is pressed for its first # minute, past the ISO's "press any key" # VM_FILES for a windows guest, a folder whose files it finds on a removable FAT drive +# VM_CREDENTIALS for a linux-desktop guest, systemd credentials, space-separated NAME= pairs, as a builder's +# boot passes the key its ssh takes before the disk holds it # launch reset: boot the running computer afresh on a new overlay, its environment the VM_ENV_= lines on # stdin, returning once its new QEMU has started; its ports stay up throughout. # launch forward PORT: serve PORT of this machine's host to the running computer as VM_HOST_PORTS does, for another @@ -75,6 +79,28 @@ if [ "$guest" != windows ] && [ -n "${VM_MEDIA:-}${VM_FILES:-}" ]; then echo "launch: VM_MEDIA and VM_FILES are for a windows guest, not $guest" >&2 exit 2 fi +if [ "$guest" != linux-desktop ] && [ -n "${VM_CREDENTIALS:-}" ]; then + echo "launch: VM_CREDENTIALS are for a linux-desktop guest, not $guest" >&2 + exit 2 +fi +# resolution: the display's WIDTHxHEIGHT, the computer's resolution setting's, 1920x1080 when it has none. +resolution() { + local value=1920x1080 + if [ -n "${VM_ENV_AGENTENV_SETTING_RESOLUTION:-}" ]; then + value=$(printf '%s' "$VM_ENV_AGENTENV_SETTING_RESOLUTION" | base64 -d) + fi + if [[ ! "$value" =~ ^[1-9][0-9]*x[1-9][0-9]*$ ]]; then + echo "launch: the computer's resolution, $value, isn't WIDTHxHEIGHT" >&2 + return 2 + fi + echo "$value" +} + +cpus=${VM_CPUS:-2} memory=${VM_MEMORY_MB:-4096} +if [ "$guest" = linux-desktop ]; then + cpus=${VM_CPUS:-4} memory=${VM_MEMORY_MB:-8192} + resolution > /dev/null || exit 2 +fi work=$state/boot mkdir -p "$control" rm -f "$control/qemu.pid" "$control/reset" "$control/next.env" @@ -87,7 +113,10 @@ elif [ -n "${VM_REQUIRE_KVM:-}" ]; then exit 3 fi case "$guest/$arch" in - linux/*) ;; + linux/* | linux-desktop/amd64) ;; + linux-desktop/*) + echo "launch: a linux-desktop guest boots on amd64 alone, not $arch" >&2 + exit 2 ;; windows/amd64) [ "$accel" = kvm ] && cpu=host,hv_relaxed,hv_vapic,hv_spinlocks=0x1fff,hv_time ;; windows/*) echo "launch: a windows guest boots on amd64 alone, not $arch" >&2 @@ -192,11 +221,21 @@ boot() { cp /usr/share/AAVMF/AAVMF_CODE.fd "$work/code.fd" truncate -s 64M "$work/code.fd" "$work/vars.fd" fi - devices=(-drive file="$booted",format=qcow2,if=virtio + local trim="" + [ "$guest" = linux-desktop ] && trim=,discard=unmap,detect-zeroes=unmap + devices=(-drive file="$booted",format=qcow2,if=virtio$trim -drive file="$work/seed.img",format=raw,if=virtio,readonly=on -device virtio-net-pci,netdev=net -device virtio-rng-pci) + if [ "$guest" = linux-desktop ]; then + local size + size=$(resolution) + devices+=(-vga none -device "virtio-vga,xres=${size%x*},yres=${size#*x}") + for credential in ${VM_CREDENTIALS:-}; do + devices+=(-smbios "type=11,value=io.systemd.credential.binary:$credential") + done + fi fi - "${machine[@]}" -cpu "$cpu" -smp "${VM_CPUS:-2}" -m "${VM_MEMORY_MB:-4096}" \ + "${machine[@]}" -cpu "$cpu" -smp "$cpus" -m "$memory" \ -drive if=pflash,format=raw,file="$work/vars.fd" -netdev "user,id=net$forwards" "${devices[@]}" \ -display none -serial stdio ${restart[@]+"${restart[@]}"} & echo "$!" > "$control/qemu.pid" diff --git a/src/agent_env/computers/disk/omarchy/agentenv-seed b/src/agent_env/computers/disk/omarchy/agentenv-seed new file mode 100755 index 000000000..77404596e --- /dev/null +++ b/src/agent_env/computers/disk/omarchy/agentenv-seed @@ -0,0 +1,50 @@ +#!/bin/bash +# agentenv-seed: run each boot by agentenv-seed.service, before SDDM starts the session. It reads the launcher's NoCloud +# seed (CIDATA, as agentenv_protocol.computer.seed defines it) as the Linux disk's init does, with the same seed-env: +# the deploy's environment written for with-seed, which each of the computer's services starts through, the seed's trust +# files before it; the hostname; host.docker.internal at QEMU's address for the launcher; the time zone; and the +# display's mode, for the session's Hyprland. A seed this disk can't read, or a time zone it doesn't have, is written as +# its refusal, so every service then stops naming it. +set -euo pipefail +. /opt/agentenv/seed-env +user=$(cat /opt/agentenv/session-user) +home=$(getent passwd "$user" | cut -d: -f6) +seed="" +for _ in $(seq 50); do + seed=$(blkid -L CIDATA 2>/dev/null) && break + sleep 0.2 +done +exports="" hostname=computer +if [ -n "$seed" ]; then + mounted=$(mktemp -d) + mount -o ro "$seed" "$mounted" + if [ -f "$mounted/trust/ca-bundle.pem" ]; then + install -d /etc/agentenv + cp "$mounted/trust/ca-bundle.pem" "$mounted/trust/ca.pem" /etc/agentenv/ + exports="export SSL_CERT_FILE=/etc/agentenv/ca-bundle.pem REQUESTS_CA_BUNDLE=/etc/agentenv/ca-bundle.pem" + exports="$exports NODE_EXTRA_CA_CERTS=/etc/agentenv/ca.pem"$'\n' + fi + exports="$exports$(_seed_exports "$mounted")" + hostname=$(sed -n 's/^local-hostname: *//p' "$mounted/meta-data" 2>/dev/null | head -n 1) + umount "$mounted" + rmdir "$mounted" +fi +# setting NAME DEFAULT: the deploy's value of the computer's setting NAME, DEFAULT when it gives none. +setting() { env -i bash -c 'eval "$1" 2>/dev/null; v=AGENTENV_SETTING_$2; echo "${!v:-$3}"' _ "$exports" "$1" "$2"; } +timezone=$(setting TIMEZONE UTC) +if [[ "$timezone" != *..* && -f "/usr/share/zoneinfo/$timezone" ]]; then + ln -sf "/usr/share/zoneinfo/$timezone" /etc/localtime +else + exports="$exports"$'\n'"refused='its time zone, ${timezone//\'/}, is not in /usr/share/zoneinfo'; false" +fi +install -d -m 750 -g "$user" /run/agentenv +install -m 640 -g "$user" /dev/null /run/agentenv/environment +printf '%s\n' "$exports" > /run/agentenv/environment +hostname "${hostname:-computer}" +hostname > /etc/hostname +grep -q 'host\.docker\.internal' /etc/hosts || printf '10.0.2.2\thost.docker.internal\n' >> /etc/hosts +resolution=$(setting RESOLUTION 1920x1080) +[[ "$resolution" =~ ^[1-9][0-9]*x[1-9][0-9]*$ ]] || resolution=1920x1080 +install -m 644 -o "$user" -g "$user" /dev/null "$home/.config/hypr/agentenv-display.lua" +printf 'hl.monitor({ output = "", mode = "%s@60", position = "0x0", scale = 1 })\n' "$resolution" \ + > "$home/.config/hypr/agentenv-display.lua" diff --git a/src/agent_env/computers/disk/omarchy/agentenv-seed.service b/src/agent_env/computers/disk/omarchy/agentenv-seed.service new file mode 100644 index 000000000..7dc54fc8f --- /dev/null +++ b/src/agent_env/computers/disk/omarchy/agentenv-seed.service @@ -0,0 +1,12 @@ +[Unit] +Description=agent-env: the deploy's environment from the launcher's seed +After=local-fs.target systemd-udev-settle.service +Before=display-manager.service + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/opt/agentenv/agentenv-seed + +[Install] +WantedBy=multi-user.target diff --git a/src/agent_env/computers/disk/omarchy/agentenv.lua b/src/agent_env/computers/disk/omarchy/agentenv.lua new file mode 100644 index 000000000..d2b255096 --- /dev/null +++ b/src/agent_env/computers/disk/omarchy/agentenv.lua @@ -0,0 +1,12 @@ +-- agent-env's Hyprland drop-in, required last from hyprland.lua: cua-driver's plugin, loaded as Hyprland starts, as it +-- must be; a US keymap, the only one the plugin's keys take; the idle cursor hidden, so a capture leaves it out; and the +-- display's mode, which agentenv-seed writes each boot from the computer's resolution. +hl.env("GDK_SCALE", "1") +hl.plugin.load("/usr/lib/cua/hyprland/cua-hyprland-plugin.so") +hl.config({ + plugin = { cua = { enabled = true } }, + cursor = { inactive_timeout = 1 }, + input = { kb_rules = "evdev", kb_model = "pc105", kb_layout = "us", kb_variant = "", kb_options = "", + numlock_by_default = false }, +}) +pcall(require, "hypr.agentenv-display") diff --git a/src/agent_env/computers/disk/omarchy/compact.sh b/src/agent_env/computers/disk/omarchy/compact.sh new file mode 100644 index 000000000..808c7df64 --- /dev/null +++ b/src/agent_env/computers/disk/omarchy/compact.sh @@ -0,0 +1,6 @@ +# compact.sh: run as root over ssh as an Omarchy disk is about to shut down to be packed, a base or a composed layer. It +# removes what the computer doesn't need, package and pip caches and the journal, then trims the free space, which the +# launcher's disk, attached with discards, passes to its image, so packing it keeps only what Omarchy uses. +rm -rf /var/cache/pacman/pkg/* /root/.cache/pip /home/*/.cache/pip +journalctl --vacuum-size=1M --quiet || true +fstrim -av diff --git a/src/agent_env/computers/disk/omarchy/recipe.sh b/src/agent_env/computers/disk/omarchy/recipe.sh new file mode 100755 index 000000000..3fbea337a --- /dev/null +++ b/src/agent_env/computers/disk/omarchy/recipe.sh @@ -0,0 +1,73 @@ +#!/bin/bash +# recipe.sh USER DISTRIBUTION VERSION: run once as root over ssh in an Omarchy base's builder, Omarchy's own image booted +# by the QEMU launcher with its session user USER signed in. It refuses an image that isn't the DISTRIBUTION and VERSION +# its artifact declares, then makes it a computer's base: cua-driver's Hyprland plugin, pinned to the package built for +# this Omarchy's Hyprland; agent-env's Hyprland drop-in; the seed unit that hands each boot its deploy's environment; +# SDDM logging back in after a logout; Omarchy's toggles for an unattended desktop; and off, what a trial doesn't want: +# crash toasts, dictation and its keys, housekeeping timers that would run at boot, and the firewall, which would stop +# the computer's own ports. The session user's password is locked, and the machine id left empty, as Omarchy's own +# sealed images leave it. +set -euo pipefail +user=$1 distribution=$2 version=$3 +here=$(cd "$(dirname "$0")" && pwd) +home=$(getent passwd "$user" | cut -d: -f6) +#: cua-driver's Hyprland plugin, as Omarchy's own repository builds it for Omarchy 4.5's Hyprland 0.56.2-4. +plugin_url=https://pkgs.omarchy.org/stable/x86_64/cua-hyprland-plugin-0.33.2-3-x86_64.pkg.tar.zst +plugin_sha256=48a25c70dafdd423118c9937d3b469953e89a15cc0f278662f8fd728b523c47e + +. /etc/os-release +if [ "$ID" != "$distribution" ] || [ "$VERSION_ID" != "$version" ]; then + echo "recipe: this image is ${ID:-?} ${VERSION_ID:-?}, not $distribution $version as its artifact declares" >&2 + exit 3 +fi +echo "os-release: ID=$ID VERSION_ID=$VERSION_ID" + +plugin=$(mktemp --suffix=.pkg.tar.zst) +curl -fsSL --retry 5 -o "$plugin" "$plugin_url" +echo "$plugin_sha256 $plugin" | sha256sum --check --quiet +pacman -U --noconfirm --needed "$plugin" +rm -f "$plugin" + +install -d /opt/agentenv +install -m 644 "$here/seed-env" /opt/agentenv/seed-env +install -m 755 "$here/agentenv-seed" "$here/with-seed" /opt/agentenv/ +echo "$user" > /opt/agentenv/session-user +install -m 644 "$here/agentenv-seed.service" /etc/systemd/system/agentenv-seed.service +systemctl enable --quiet agentenv-seed.service +install -d /etc/sddm.conf.d +install -m 644 "$here/sddm-relogin.conf" /etc/sddm.conf.d/zz-agentenv.conf + +install -m 644 -o "$user" -g "$user" "$here/agentenv.lua" "$home/.config/hypr/agentenv.lua" +grep -q 'require("hypr.agentenv")' "$home/.config/hypr/hyprland.lua" \ + || echo 'require("hypr.agentenv")' >> "$home/.config/hypr/hyprland.lua" + +systemctl --global mask --quiet omarchy-crash-watch.service superwhisper.service +systemctl mask --quiet snapper-cleanup.timer plocate-updatedb.timer man-db.timer shadow.timer \ + archlinux-keyring-wkd-sync.timer +# With no dictation backend chosen, Omarchy binds no dictation keys (its bindings/dictation.lua), Alt+Space among them. +rm -f "$home/.config/omarchy/defaults/dictation" +# Omarchy's firewall lets in only ssh and LocalSend. A computer's network is the launcher's user-mode one, which only the +# ports agent-env forwards reach, its driver's and apps', so the firewall would stop just those. +ufw --force disable > /dev/null +systemctl disable --quiet ufw.service + +runtime=/run/user/$(id -u "$user") +for _ in $(seq 180); do + [ -S "$runtime/bus" ] && compgen -G "$runtime/hypr/*/.socket.sock" > /dev/null && break + sleep 1 +done +for _ in $(seq 180); do + [ -e "$home/.local/state/omarchy/done/first-run-user" ] && break + sleep 1 +done +# A command run in the session user's own systemd manager, with the environment its graphical session gave it. +session() { systemd-run --quiet --machine="$user@.host" --user --wait --pipe --collect -- "$@"; } +session omarchy toggle idle stay-awake +session omarchy toggle animations off +session omarchy toggle notification silencing +session omarchy toggle screensaver +session omarchy toggle suspend + +passwd --lock --quiet "$user" 2>/dev/null || passwd -l "$user" > /dev/null +truncate -s 0 /etc/machine-id +echo "recipe: done" diff --git a/src/agent_env/computers/disk/omarchy/sddm-relogin.conf b/src/agent_env/computers/disk/omarchy/sddm-relogin.conf new file mode 100644 index 000000000..a4af1bf9a --- /dev/null +++ b/src/agent_env/computers/disk/omarchy/sddm-relogin.conf @@ -0,0 +1,3 @@ +# agent-env: log the session user back in after a logout, as an agent may log out from Omarchy's menu. +[Autologin] +Relogin=true diff --git a/src/agent_env/computers/disk/omarchy/with-seed b/src/agent_env/computers/disk/omarchy/with-seed new file mode 100755 index 000000000..d1d742749 --- /dev/null +++ b/src/agent_env/computers/disk/omarchy/with-seed @@ -0,0 +1,10 @@ +#!/bin/bash +# with-seed COMMAND...: COMMAND run as the computer's services are, with its build's settings and the deploy's +# environment, which agentenv-seed wrote this boot from the launcher's seed; a seed this disk can't read stops it, naming +# why. +set -euo pipefail +if [ -f /opt/agentenv/build-environment ]; then + . /opt/agentenv/build-environment +fi +. /run/agentenv/environment || { echo "with-seed: ${refused:-the deploy's environment couldn't be read}" >&2; exit 1; } +exec "$@" diff --git a/src/agent_env/env/envs/computer/driver_env.py b/src/agent_env/env/envs/computer/driver_env.py index 5a8b1478e..6d34d7be0 100644 --- a/src/agent_env/env/envs/computer/driver_env.py +++ b/src/agent_env/env/envs/computer/driver_env.py @@ -10,12 +10,13 @@ from typing import TYPE_CHECKING, Any, ClassVar, NoReturn, Optional from agentenv_protocol.computer.driver import DriverDeclaration +from agentenv_protocol.computer.platform import entry_for from pydantic import BaseModel, ConfigDict, Field, ValidationError from agent_env.artifact import Artifact from agent_env.artifact.artifacts.file_artifact_universe import FileArtifactUniverse from agent_env.env.env import Env -from agent_env.env.envs.computer.utils.install import (DRIVER_INSTALLS, Install, folder_problems, installs, os_type_of, +from agent_env.env.envs.computer.utils.install import (DRIVER_INSTALLS, Install, folder_problems, installs, platform_problem, unsupported) from agent_env.store.ids import derive_id @@ -70,7 +71,7 @@ def to_dict(self) -> dict: @classmethod def from_dict(cls, data: dict) -> ComputerDriverEnv: files = Artifact.get(data["files"]["id"], version=data["files"]["version"]) - install = {key: DRIVER_INSTALLS[os_type_of(key)].model_validate(table) + install = {key: entry_for(DRIVER_INSTALLS, key).model_validate(table) for key, table in (data.get("install") or {}).items()} return cls(id=data["id"], version=data.get("version"), module=data["module"], files=files, declaration=DriverDeclaration.model_validate(data["declaration"]), install=install, diff --git a/src/agent_env/env/envs/computer/utils/boot.py b/src/agent_env/env/envs/computer/utils/boot.py index a982a6983..4192b5f94 100644 --- a/src/agent_env/env/envs/computer/utils/boot.py +++ b/src/agent_env/env/envs/computer/utils/boot.py @@ -18,6 +18,7 @@ from agentenv_protocol.computer import APP_SETTINGS_ENV, COMPUTER_READY, SETTINGS_ENV, TOOL_PREFIX_ENV from agentenv_protocol.computer.app import COMPUTER_APP from agentenv_protocol.computer.apps import ANSWER_TIMEOUT_S +from agentenv_protocol.computer.platform import entry_for from agent_env.config import get_config from agent_env.env.envs.computer.utils.composed import REPOSITORY @@ -51,10 +52,10 @@ class ComputerFailed(RuntimeError): """The computer's ``computer-ready`` status says it failed for good, a check that can't pass again.""" #: How many random bytes a computer's control token carries: 256 bits, past guessing. CONTROL_TOKEN_BYTES = 32 -#: The size of a disk's VM when a deploy sets none: what the desktop was measured booting and passing the probe in, and -#: for Windows, what Windows 11 asks of a PC. +#: The size of a disk's VM when a deploy sets none: what the desktop was measured booting and passing the probe in; for +#: Windows, what Windows 11 asks of a PC; and for Omarchy, what Hyprland's own CI warns it may crash with less than. DISK_CPU, DISK_MEMORY_MB = 2.0, 4096 -WINDOWS_CPU, WINDOWS_MEMORY_MB = 4.0, 8192 +DISK_SIZES = {"linux": (DISK_CPU, DISK_MEMORY_MB), "linux/omarchy": (4.0, 8192), "windows": (4.0, 8192)} #: The variable a Linux disk's init sizes its ``/dev/shm`` by, in MiB (``computers/disk/linux/boot/init``). SHM_MB_ENV = "AGENTENV_SHM_MB" @@ -178,12 +179,12 @@ async def boot(sandbox_provider: SandboxProvider, *, image: str, port: int, envi network_policy: Optional[NetworkPolicy], timeout: int, attribution: Optional[Attribution], disk_size_gb: float = 10, cpu: Optional[float] = None, memory_mb: Optional[int] = None, ready_s: float = READY_TIMEOUT_S, form: str = "container", held: bool = False, - os_type: str = "linux", shm_mb: Optional[int] = None) -> Booted: + platform: str = "linux", shm_mb: Optional[int] = None) -> Booted: """``image``, of ``form``, booted with ``environment`` and its control ``token`` on ``sandbox_provider``, which the caller chose to run its arch natively, its driver's ``port`` reached on the sandbox, its ``/dev/shm`` ``shm_mb`` MiB where the sandbox can size it, once the driver, asked with that token, says the computer is ready, or with ``held``, its boot apps' launch held, once it answers (``ready``); a boot that fails is torn down.""" - environment = {**environment, **control_environment(token), **shm_environment(form, os_type, shm_mb)} + environment = {**environment, **control_environment(token), **shm_environment(form, platform.partition("/")[0], shm_mb)} if form == "vm_bundle": sandbox = await sandbox_provider.create_vm( image=image, env=environment, exposed_ports=[port], network_policy=network_policy, @@ -192,8 +193,7 @@ async def boot(sandbox_provider: SandboxProvider, *, image: str, port: int, envi sandbox = await sandbox_provider.create_vm( image=image, env=environment, exposed_ports=[port], network_policy=network_policy, disk_size_gb=disk_size_gb, timeout=timeout, attribution=dict(attribution or {}), - cpu=cpu or (WINDOWS_CPU if os_type == "windows" else DISK_CPU), - memory=memory_mb or (WINDOWS_MEMORY_MB if os_type == "windows" else DISK_MEMORY_MB)) + cpu=cpu or entry_for(DISK_SIZES, platform)[0], memory=memory_mb or entry_for(DISK_SIZES, platform)[1]) else: sandbox = await sandbox_provider.create_container( image_name=image, port=port, env=environment, network_policy=network_policy, disk_size_gb=disk_size_gb, diff --git a/src/agent_env/env/envs/computer/utils/composer.py b/src/agent_env/env/envs/computer/utils/composer.py index 95fbc8e9a..1819d93d4 100644 --- a/src/agent_env/env/envs/computer/utils/composer.py +++ b/src/agent_env/env/envs/computer/utils/composer.py @@ -44,6 +44,7 @@ clashes, gate, macos_vm_bundle, + omarchy_disk, packages, probing, thin_disk, @@ -139,10 +140,11 @@ def compose(env: ComputerEnv, pairs: list[tuple[str, str]] | None = None, containers = {arch: pool.submit(_composed, env, base, driver, form, arch, builds, progress) for form, arch in pairs if form == "container"} futures = {future: ("container", arch) for arch, future in containers.items()} + built_from = {"windows": _composed_windows_disk, "linux/omarchy": _composed_omarchy_disk}.get(platform_of(base.os)) futures |= {pool.submit(_composed_disk, env, base, driver, arch, containers.get(arch), builds, progress): - ("disk", arch) for form, arch in pairs if form == "disk" and base.os.type != "windows"} - futures |= {pool.submit(_composed_windows_disk, env, base, driver, arch, provider, progress): ("disk", arch) - for form, arch in pairs if form == "disk" and base.os.type == "windows"} + ("disk", arch) for form, arch in pairs if form == "disk" and built_from is None} + futures |= {pool.submit(built_from, env, base, driver, arch, provider, progress): ("disk", arch) + for form, arch in pairs if form == "disk" and built_from is not None} futures |= {pool.submit(_composed_vm_bundle, env, base, driver, arch, provider, progress): ("vm_bundle", arch) for form, arch in pairs if form == "vm_bundle"} for future in as_completed(futures): @@ -402,7 +404,7 @@ def say(line: str) -> None: lambda: windows_disk.media(say), disk_gb, user, say, registry) script = windows_disk.write(build, parts, user=user, build_settings=settings, image={"base": made.image, "layers": [part.folder for part in parts]}) - expires = made.expires.isoformat(timespec="seconds") + expires = made.labels[windows_disk.EXPIRES_LABEL] key = attempt.key = _key(base=made.image, base_expires=expires, provision=script, context=_digest(context), build_settings=settings, form="disk", arch=arch, format=FORMAT) attempt.stage = "build" @@ -502,6 +504,62 @@ def _part(layer: _Layer, programs: dict[str, str], environment: dict[str, Any], environment, _venv(layer), spec if _venv(layer) else None, layer.served, layer.steps) +def _composed_omarchy_disk(env: ComputerEnv, base: ComputerArtifact, driver: ComputerDriverEnv, arch: str, + provider: SandboxProvider, progress: _Progress) -> dict: + """The Omarchy disk of ``arch``: one of its key the env's image store holds, reused; else composed with this + machine's Docker and KVM (``omarchy_disk``), on its base, this machine's, the store's or built from Omarchy's own + image, the driver installed over ssh as a layer on it, packed and pushed to the store under its key, as Linux disks + are. Then probed at full (gate 3). All of it holds a lock of its key, as a Windows disk's compose does.""" + pair = f"disk/{arch}" + with (_attempting(env, "disk", arch) as attempt, + tempfile.TemporaryDirectory(prefix="agentenv-compose-") as work): + if not boot.native_links(provider, arch, "disk", env.stored_id, base.os.type): + raise builder.Unbuildable(f"no link of {' or '.join(type(link).__name__ for link in provider.links)} boots " + f"linux/{arch} disks, so it can't be composed here: compose it on a Linux machine " + "with KVM, with the local sandbox provider") + build = Path(work) / "build" + context = build / "context" + context.mkdir(parents=True) + protocol = _stage_protocol(context) + layer, name = _stage_driver(context, driver, base, {}), gate.folder_name(driver.id, "driver") + prefix = omarchy_disk.DRIVERS / name + parts = [omarchy_disk.Part(layer.folder, prefix, layer.install, { + program: omarchy_disk.argv(command, prefix) + for program, command in _services(f"driver-{name}", layer.install).items()}, {"MCP_PORT": driver.port}, + (f"{omarchy_disk.BUILD}/context/protocol",) if protocol else ())] + values = env.computer_values() + settings = {name: values.get(name, setting.default) for name, setting in base.config.items() + if setting.phase == "build"} + + def say(line: str) -> None: + progress.say(f"{pair}: {line}") + + attempt.stage = "base" + registry = thin_disk.Registry(_image_store(env), REPOSITORY) + source, user = base.forms.disk.source, base.session.user + made = omarchy_disk.base( + _key(recipe=_digest(omarchy_disk.RECIPE), seed=hashlib.sha256(omarchy_disk.SEED_ENV.read_bytes()).hexdigest(), + source=source.sha256, user=user, os=base.os.model_dump(), form="omarchy-base", format=FORMAT), + lambda: omarchy_disk.source(source, say), user, base.os.distribution, base.os.version, say, registry) + script = omarchy_disk.write(build, parts, user=user, build_settings=settings, + image={"base": made.image, "layers": [part.folder for part in parts]}) + key = attempt.key = _key(base=made.image, provision=script, context=_digest(context), build_settings=settings, + form="disk", arch=arch, format=FORMAT) + attempt.stage = "build" + with holding_locks([f"omarchy-disk:{key}"], lambda: progress.at( + pair, "waiting", "waiting for the compose of the same disk under way on this machine")): + if (pinned := registry.served(thin_disk.disk_names(omarchy_disk.GUEST, key).tag)) is not None: + progress.at(pair, "reusing", "reusing the disk of the same key, composed earlier") + result = {"status": "unverified", "key": key, "image": pinned, "reused": True, "built_at": now()} + else: + progress.at(pair, "composing", "composing on this machine's Docker") + pinned = thin_disk.compose(omarchy_disk.GUEST, made, build, key, {KEY_LABEL: key}, say, registry) + result = {"status": "unverified", "key": key, "image": pinned, "reused": False, + "builder": "LocalSandboxProvider", "built_at": now()} + return _probed(env, driver, "disk", arch, result, provider, progress) + return attempt.result + + def _windows_part(layer: _Layer, prefix: PureWindowsPath, services: dict[str, list[str]], environment: dict[str, Any], protocol: bool) -> windows_disk.Part: """``layer`` as a Windows disk installs it under ``prefix``: its ``services``' argv by name, with ``environment``, @@ -977,12 +1035,12 @@ def _computer(env: ComputerEnv, base: ComputerArtifact, built: dict[str, str], d bundles: dict[str, str]) -> dict | None: """Write the env's computer over the images ``built``, by arch, joined into one index in the env's image store, signed in to it: its arches those, its OS and session the base's, its settings the base's with the env's values as their - defaults; with a disk form over ``disks`` when there's one for each of those arches; for a macOS base, over its VM - bundles alone. None when nothing built.""" + defaults; with a disk form over ``disks`` when there's one for each of those arches; for a base that's a disk alone, + Windows or Omarchy, over its disks alone; for a macOS base, over its VM bundles alone. None when nothing built.""" config = {name: setting.model_dump() for name, setting in base.config.items()} for name, value in env.computer_values().items(): config[name]["default"] = value - if base.os.type == "windows": + if base.forms.container is None and base.forms.disk is not None: if not disks: return None return _written(env, base, [arch for arch in base.arch if arch in disks], config, diff --git a/src/agent_env/env/envs/computer/utils/gate.py b/src/agent_env/env/envs/computer/utils/gate.py index b7bafede2..b56882daf 100644 --- a/src/agent_env/env/envs/computer/utils/gate.py +++ b/src/agent_env/env/envs/computer/utils/gate.py @@ -3,8 +3,10 @@ from __future__ import annotations +import functools import itertools import re +import zoneinfo from collections.abc import Sequence from pathlib import PurePosixPath from typing import TYPE_CHECKING, Any @@ -67,8 +69,9 @@ def arches(base: ComputerArtifact, driver: ComputerDriverEnv, apps: Sequence[Com def check_app(base: ComputerArtifact, driver: ComputerDriverEnv, app: ComputerAppEnv, at: str = "app") -> list[str]: """Every way ``app`` doesn't fit a composition of ``base`` and ``driver``, each named from ``at``: a platform it - doesn't support, no architecture in common, no install for the base's OS, a port the driver listens on too, or on - Windows an install step for the session user, as a Windows computer is composed over ssh as its admin.""" + doesn't support, no architecture in common, no install for the base's OS, a port the driver listens on too, any app + on Omarchy, whose disks compose a driver alone for now, or on Windows an install step for the session user, as a + Windows computer is composed over ssh as its admin.""" os = base.os if not _matching(base, app.declaration.supports): named = ", ".join(_shown(platform) for entry in app.declaration.supports for platform in supported(entry)) @@ -80,6 +83,9 @@ def check_app(base: ComputerArtifact, driver: ComputerDriverEnv, app: ComputerAp problems.append(f"{at}: {app.id} declares no install for {_os_shown(base)}") elif taken := sorted(set(_ports(app, platform_of(os))) & set(_driver_ports(driver, os))): problems.append(f"{at}: {app.id} listens on {', '.join(map(str, taken))}, as {driver.id} does") + if platform_of(os) == "linux/omarchy": + problems.append(f"{at}: an Omarchy computer composes its driver alone for now; {app.id} would need its install " + "composed onto Omarchy's disk, which agent-env doesn't do yet") if os.type == "windows" and (session := [step.name for step in app.declaration.install_steps if step.scope == "session" and (step.os is None or os.type in step.os)]): problems.append(f"{at}: {app.id}'s install steps {', '.join(session)} run as the session user, which a Windows " @@ -218,11 +224,19 @@ def check_computer_settings(base: ComputerArtifact, values: Any) -> tuple[dict[s elif base.os.type == "windows" and name == "resolution" and value != WINDOWS_DISPLAY: problems.append(f"computer_settings.resolution: a Windows computer's display is {WINDOWS_DISPLAY} for now, " f"not {value}") + elif name == "timezone" and value not in _time_zones(): + problems.append(f"computer_settings.timezone: {value!r} isn't an IANA time zone, such as UTC or " + "America/Chicago") else: checked[name] = value return checked, problems +@functools.cache +def _time_zones() -> frozenset[str]: + return frozenset(zoneinfo.available_timezones()) + + def _platform(base: ComputerArtifact, driver: ComputerDriverEnv) -> list[str]: os = base.os if not _matching(base, driver.declaration.supports): @@ -235,7 +249,8 @@ def _platform(base: ComputerArtifact, driver: ComputerDriverEnv) -> list[str]: if (install := for_os(driver.install, os)) is None: problems.append(f"driver: {driver.id} declares no install for {_os_shown(base)}") else: - init = base.forms.container.init if base.forms.container is not None else "launchd" + form = base.forms.container or base.forms.disk + init = form.init if form is not None else "launchd" problems += [f"driver: its service {service.name!r} runs as root, and {base.id}'s {init} runs every program as " f"{base.session.user}" for service in install.services if service.scope == "system"] return problems diff --git a/src/agent_env/env/envs/computer/utils/install.py b/src/agent_env/env/envs/computer/utils/install.py index faa0b1480..68882add1 100644 --- a/src/agent_env/env/envs/computer/utils/install.py +++ b/src/agent_env/env/envs/computer/utils/install.py @@ -9,7 +9,8 @@ class with ``@linux(...)``: ``files``, a folder of its own copied under the laye scripts are the protocol's, which an app's ``@linux(...)`` uses too. ``[install.macos]`` takes the same, but apt's packages and sources; its services are LaunchAgents of the login session, and its ``grants`` the TCC permissions an app bundle or an executable it installs needs. ``[install.windows]`` takes the same as macOS but grants: its run scripts are -PowerShell, run elevated, and its services run windowless in the session user's sign-in.""" +PowerShell, run elevated, and its services run windowless in the session user's sign-in. ``[install."linux/omarchy"]`` +takes Linux's but apt's packages and sources, Omarchy being Arch.""" from __future__ import annotations @@ -153,6 +154,27 @@ class MacAppInstall(MacInstall): services: list[MacService] = Field(default_factory=list) +def _without_apt(install: Install, os: str) -> None: + if install.packages is not None or install.sources: + raise ValueError(f"{os} has no apt: fetch what the layer needs with downloads, checked by sha256") + + +class OmarchyInstall(Install): + """How an Omarchy computer installs a layer: as on Linux, but with no apt.""" + + @model_validator(mode="after") + def _no_apt(self) -> OmarchyInstall: + _without_apt(self, "Omarchy") + return self + + +class OmarchyAppInstall(AppInstall): + @model_validator(mode="after") + def _no_apt(self) -> OmarchyAppInstall: + _without_apt(self, "Omarchy") + return self + + class WindowsService(Service): """A program agentenv-session starts, windowless, in the session user's sign-in, with the deploy's settings.""" @@ -178,9 +200,12 @@ class WindowsAppInstall(WindowsInstall): services: list[WindowsService] = Field(default_factory=list) -#: The install model of each OS type, for a driver and for an app. -DRIVER_INSTALLS: dict[str, type[Install]] = {"linux": Install, "macos": MacInstall, "windows": WindowsInstall} -APP_INSTALLS: dict[str, type[Install]] = {"linux": AppInstall, "macos": MacAppInstall, "windows": WindowsAppInstall} +#: The install model of each OS type, or of a distribution whose differs from its OS type's, for a driver and for an +#: app. +DRIVER_INSTALLS: dict[str, type[Install]] = {"linux": Install, "linux/omarchy": OmarchyInstall, "macos": MacInstall, + "windows": WindowsInstall} +APP_INSTALLS: dict[str, type[Install]] = {"linux": AppInstall, "linux/omarchy": OmarchyAppInstall, + "macos": MacAppInstall, "windows": WindowsAppInstall} _Entry = TypeVar("_Entry") @@ -222,7 +247,7 @@ def for_os(by_platform: dict[str, _Entry], os: Any) -> Optional[_Entry]: def installs(data: Any, models: dict[str, type[Install]] = DRIVER_INSTALLS) -> tuple[dict[str, Install], list[str]]: """The installs ``data``, an ``install`` table, declares by OS type or distribution (``platform_problem``), each - its OS type's model of ``models``, and the problems with it, each named by its key.""" + its platform's model of ``models`` (``entry_for``), and the problems with it, each named by its key.""" if not isinstance(data, dict): return {}, [f"install must be a table of OS types ({', '.join(OS_TYPES)}), not {data!r}"] found, problems = {}, [] @@ -231,7 +256,7 @@ def installs(data: Any, models: dict[str, type[Install]] = DRIVER_INSTALLS) -> t problems.append(f"install.{key}: an install is for {problem}") continue try: - found[key] = models[os_type_of(key)].model_validate(table) + found[key] = entry_for(models, key).model_validate(table) except ValidationError as e: problems.extend(f"install.{'.'.join(map(str, (key, *error['loc'])))}: {error['msg']}" for error in e.errors()) diff --git a/src/agent_env/env/envs/computer/utils/omarchy_disk.py b/src/agent_env/env/envs/computer/utils/omarchy_disk.py new file mode 100644 index 000000000..f43a67535 --- /dev/null +++ b/src/agent_env/env/envs/computer/utils/omarchy_disk.py @@ -0,0 +1,244 @@ +"""Omarchy computers' disks, thin disks (``thin_disk``) made with this machine's Docker and KVM. A base is Omarchy's own +VM image, fetched once into this machine's cache from its artifact's source and checked by sha256, booted once by the +QEMU launcher as a linux-desktop guest, with root's ssh key passed as a systemd credential, made a computer's base by +the recipe of ``computers/disk/omarchy`` run as root over ssh, then packed whole as a thin disk's base. A composed disk's +parts are installed by a bash script, ``provision.sh``, run as root: each layer's files, downloads, venv and run scripts, +and its services as systemd user units of the graphical session, which start through ``with-seed``, so each boot's +deploy environment is theirs.""" + +from __future__ import annotations + +import base64 +import hashlib +import json +import shlex +import shutil +import time +import urllib.request +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Any + +from agent_env.artifact.artifacts.computer import DiskSource +from agent_env.config.paths import cache_root +from agent_env.env.envs.computer.utils import thin_disk +from agent_env.env.envs.computer.utils.install import Install +from agent_env.env.envs.computer.utils.thin_disk import Base, Registry +from agent_env.providers.sandbox_providers.local_docker.disk import launcher +from agent_env.store.local_state import holding_locks + +_DISK = Path(__file__).resolve().parents[4] / "computers" / "disk" +#: What a base is made with: its recipe, run as root in its builder, and the seed's reader, the Linux disk's own. +RECIPE, SEED_ENV = _DISK / "omarchy", _DISK / "linux" / "boot" / "seed-env" +ADMIN = "root" +#: Where agent-env's own files are on an Omarchy computer, each layer's folder under them, and its services' units. +ROOT = PurePosixPath("/opt/agentenv") +DRIVERS = ROOT / "drivers" +BUILD = ROOT / "build" +UNITS = PurePosixPath("/etc/systemd/user") +#: What starts every service, with the deploy's environment (``computers/disk/omarchy/with-seed``). +WITH_SEED = ROOT / "with-seed" +OS_ID_LABEL, OS_VERSION_LABEL = "dev.agentenv.os-release.id", "dev.agentenv.os-release.version-id" +#: How long a builder may take to answer over ssh, and its recipe to run. +BOOT_TIMEOUT_S, RECIPE_TIMEOUT_S = 600, 1800 +#: The exit code of a recipe that found its image isn't what its artifact declares. +MISLABELLED = 3 +#: How many times a provision tries a layer's download, and how long it waits between. +FETCH_TRIES, FETCH_PAUSE_S = 30, 10 +#: What a base's build prints once, as it fetches Omarchy's image. +NOTICE = "omarchy: fetching Omarchy's image from {url} (about 4 GB) into {path}, once" + +#: Omarchy as a thin disk's guest: root over ssh runs bash, which a compose has unpack its build into ``BUILD`` and run +#: its ``provision.sh``, then trim with the recipe's ``compact.sh``, and power off. +GUEST = thin_disk.Guest( + name="omarchy", file="linux-desktop", user=ADMIN, + prepare=f"rm -rf {BUILD} && mkdir -p {BUILD}", + unpack=f"tar -xf - -C {BUILD}", + provision=f"bash {BUILD}/provision.sh", + cleanup=f"rm -rf {BUILD}", + compact="bash -c " + shlex.quote((RECIPE / "compact.sh").read_text()), + shutdown="systemctl poweroff") + + +@dataclass(frozen=True) +class Part: + """A layer as an Omarchy disk installs it: its folder under ``context/``, its prefix, its install, its services' + argv by name, the environment they're given, and what its venv installs beside its requirements.""" + + folder: str + prefix: PurePosixPath + install: Install + services: dict[str, list[str]] + environment: dict[str, Any] + wants: tuple[str, ...] = () + + +def source_path(sha256: str) -> Path: + """Where this machine's cache keeps the image of ``sha256``.""" + return cache_root() / "omarchy" / "images" / f"{sha256}.qcow2" + + +def source(disk_source: DiskSource, say: Callable[[str], None]) -> Path: + """The image ``disk_source`` names in this machine's cache, fetched first when it isn't there and refused when it + doesn't come with its sha256.""" + path = source_path(disk_source.sha256) + with holding_locks([f"omarchy-source:{path}"]): + if not path.exists(): + path.parent.mkdir(parents=True, exist_ok=True) + say(NOTICE.format(url=disk_source.url, path=path)) + partial, digest = path.with_suffix(".part"), hashlib.sha256() + with urllib.request.urlopen(disk_source.url, timeout=60) as response, partial.open("wb") as out: + while chunk := response.read(1 << 20): + digest.update(chunk) + out.write(chunk) + if digest.hexdigest() != disk_source.sha256: + partial.unlink() + raise RuntimeError(f"omarchy: {disk_source.url} came with sha256 {digest.hexdigest()}, not " + f"{disk_source.sha256}") + partial.rename(path) + return path + + +def base(key: str, image: Callable[[], Path], user: str, distribution: str, version: str, + say: Callable[[str], None], registry: Registry) -> Base: + """The base of ``key``, as ``thin_disk.base`` finds or builds one: built from the image ``image`` gives, its session + user ``user``, refused unless it's the ``distribution`` and ``version`` its artifact declares, which it records.""" + def built(work: Path, keys: Path) -> dict[str, str]: + _built(work, image(), keys, user, distribution, version, say) + return {OS_ID_LABEL: distribution, OS_VERSION_LABEL: version} + + return thin_disk.base(GUEST, key, built, say, registry) + + +def _built(work: Path, source_image: Path, keys: Path, user: str, distribution: str, version: str, + say: Callable[[str], None]) -> None: + """``source_image`` made a base, ``work/base.img``: booted on an overlay by the launcher as a linux-desktop guest, + root's key in a systemd credential, the recipe run over ssh, then trimmed, powered off and folded into one image.""" + out, recipe = work / "out", work / "recipe" + out.mkdir() + shutil.copytree(RECIPE, recipe) + shutil.copy(SEED_ENV, recipe / "seed-env") + (out / "guest").write_text(GUEST.file) + name = f"agentenv-omarchy-build-{keys.name}" + say(f"booting Omarchy's image in {name}") + mounts = ("-v", f"{source_image}:/in/source.qcow2:ro") + with thin_disk.running(name, {"VM_KEEP": "/out/disk.img", "VM_DISK": "/in/source.qcow2", + "VM_CREDENTIALS": f"tmpfiles.extra={_root_key(keys)}"}, + ["-v", f"{out}:/out", *mounts], launcher()) as (container, port): + thin_disk.answering(GUEST, container, port, keys, time.monotonic() + BOOT_TIMEOUT_S) + say("making it a base over ssh") + thin_disk.ssh(port, keys, ADMIN, "rm -rf /root/agentenv-recipe && mkdir /root/agentenv-recipe") + thin_disk.ssh(port, keys, ADMIN, "tar -xf - -C /root/agentenv-recipe", input=thin_disk.tarred(recipe)) + done = thin_disk.ssh(port, keys, ADMIN, " ".join(map(shlex.quote, ( + "bash", "/root/agentenv-recipe/recipe.sh", user, distribution, version))), check=False, + timeout=RECIPE_TIMEOUT_S) + if done.returncode == MISLABELLED: + raise RuntimeError(f"omarchy: {done.stderr.strip() or done.stdout.strip()}") + if done.returncode != 0: + raise RuntimeError(f"omarchy: its recipe exited {done.returncode}: " + f"{(done.stderr + done.stdout).strip()[-3000:]}") + thin_disk.ssh(port, keys, ADMIN, "rm -rf /root/agentenv-recipe") + thin_disk.compacted(GUEST, port, keys, say) + thin_disk.shut_down(GUEST, container, port, keys) + thin_disk.converted(out, launcher(), args=mounts) + shutil.move(out / "disk.img", work / "base.img") + + +def _root_key(keys: Path) -> str: + """The systemd credential, base64'd, whose tmpfiles lines give root the base's key, as Omarchy applies at boot.""" + lines = (f"d /root/.ssh 0700 root root -\n" + f"f+ /root/.ssh/authorized_keys 0600 root root - {(keys / 'id.pub').read_text().strip()}\n") + return base64.b64encode(lines.encode()).decode() + + +def write(build: Path, parts: list[Part], *, user: str, build_settings: dict[str, Any], image: dict) -> str: + """Write ``build``'s ``provision.sh``, which installs ``parts`` from ``build/context``, their session-scoped run + scripts as the session user ``user``, and enables their services for the session, ``build_settings`` as every + service's ``AGENTENV_SETTING_`` and ``image`` as what the disk says it holds, then leaves the computer no ssh: + only a compose logs in, and it boots the base. Its text, which a composed disk's key covers.""" + lines = ["#!/bin/bash", "set -euo pipefail", f"build={BUILD}", f"user={shlex.quote(user)}", + 'home=$(getent passwd "$user" | cut -d: -f6)', _FETCH] + for part in parts: + lines += [f"# {part.folder}", *_installed_part(part)] + settings = "".join(f"export AGENTENV_SETTING_{name.upper()}={shlex.quote(str(value))}\n" + for name, value in build_settings.items()) + lines += [_written(ROOT / "build-environment", settings), + _written(ROOT / "image.json", json.dumps(image, sort_keys=True)), + "systemctl mask --quiet sshd.service sshd.socket"] + script = "\n".join(lines) + "\n" + (build / "provision.sh").write_text(script) + return script + + +def _installed_part(part: Part) -> list[str]: + """The lines that install ``part``: its files, downloads, venv and run scripts, each as root or, for ``session``, + as the session user in their home, and its services as user units of the graphical session, each enabled for every + user.""" + prefix, install = part.prefix, part.install + lines = [f"install -d {shlex.quote(str(prefix))}", + f'cp -a "$build/context/{part.folder}/." {shlex.quote(str(prefix))}/'] + for download in install.downloads or []: + if download.arch in (None, "amd64"): + to = prefix / download.to + lines += [f"install -d {shlex.quote(str(to.parent))}", + f"fetch {shlex.quote(download.url)} {download.sha256} {shlex.quote(str(to))}"] + if install.python is not None or part.wants: + venv = prefix / "venv" + wants = [*(("-r", str(prefix / install.python)) if install.python else ()), *part.wants] + lines += [f"python3 -m venv {shlex.quote(str(venv))}", + f"{venv}/bin/pip install -q --disable-pip-version-check {' '.join(map(shlex.quote, wants))}", + f"{venv}/bin/pip list --format=json > {shlex.quote(str(prefix / 'installed.json'))}"] + for run in install.run or []: + as_user = 'runuser -u "$user" -- env HOME="$home" ' if run.scope == "session" else "" + lines.append(f"(cd {shlex.quote(str(prefix))} && {as_user}PREFIX={shlex.quote(str(prefix))} " + f"bash {shlex.quote(str(prefix / run.script))})") + for name, argv in part.services.items(): + lines += [_written(UNITS / f"agentenv-{name}.service", unit(name, prefix, argv, part.environment)), + f"systemctl --global enable --quiet agentenv-{name}.service"] + return lines + + +def unit(name: str, prefix: PurePosixPath, argv: list[str], environment: dict[str, Any]) -> str: + """The systemd user unit of the service ``name``: started with the graphical session, which gives it Hyprland's + instance and Wayland's display, and again with it, as Hyprland restarts under a new instance signature; through + ``with-seed``, in its ``prefix``, with ``environment``; again whenever it exits.""" + lines = ["[Unit]", f"Description=agent-env: {name}", "PartOf=graphical-session.target", + "After=graphical-session.target", "ConditionEnvironment=WAYLAND_DISPLAY", "", + "[Service]", f"WorkingDirectory={prefix}", + *(f"Environment={_unit_word(f'{key}={value}')}" for key, value in environment.items()), + "ExecStart=" + " ".join(_unit_word(word) for word in (str(WITH_SEED), *argv)), + "Restart=always", "RestartSec=1", "", + "[Install]", "WantedBy=graphical-session.target"] + return "\n".join(lines) + "\n" + + +def argv(command: str, prefix: PurePosixPath) -> list[str]: + """A service's ``command`` as the argv its unit runs, split as a shell would, a relative program named under + ``prefix``.""" + words = shlex.split(command) + if "/" in words[0] and not PurePosixPath(words[0]).is_absolute(): + words[0] = str(prefix / words[0]) + return words + + +def _unit_word(word: str) -> str: + """``word`` as one word of a unit's line: quoted when it holds more than plain characters, its specifiers and + variables escaped, as systemd would otherwise expand them.""" + escaped = word.replace("%", "%%").replace("$", "$$") + if escaped and all(ch.isalnum() or ch in "_./:=+-,@" for ch in escaped): + return escaped + return '"' + escaped.replace("\\", "\\\\").replace('"', '\\"') + '"' + + +def _written(path: PurePosixPath, text: str) -> str: + """The line that writes ``text`` to ``path``.""" + return f"install -d {shlex.quote(str(path.parent))} && printf %s {shlex.quote(text)} > {shlex.quote(str(path))}" + + +#: A layer's download, fetched again on a server's error, as the base's recipe does, and refused unless it comes with +#: its sha256. +_FETCH = (f"fetch() {{ for try in $(seq {FETCH_TRIES}); do curl -fsSL -o \"$3\" \"$1\" && break; " + f"[ \"$try\" -lt {FETCH_TRIES} ] || {{ echo \"provision: $1 couldn't be fetched\" >&2; exit 1; }}; " + f"sleep {FETCH_PAUSE_S}; done; echo \"$2 $3\" | sha256sum --check --quiet || " + "{ echo \"provision: $1 came with another sha256 than $2\" >&2; exit 1; }; }") diff --git a/src/agent_env/env/envs/computer/utils/probing.py b/src/agent_env/env/envs/computer/utils/probing.py index 56c931137..878cd37cf 100644 --- a/src/agent_env/env/envs/computer/utils/probing.py +++ b/src/agent_env/env/envs/computer/utils/probing.py @@ -168,7 +168,7 @@ async def _probe(links: list[SandboxProvider], env: ComputerEnv, driver: Compute booted = await boot.boot(link, image=image, port=driver.port, environment=environment, token=token, network_policy=None, timeout=_PROBE_TTL_S, attribution=None, ready_s=boot.ready_timeout(env.composed_apps(), platform_of(env.computer.os)), - form=form, os_type=env.computer.os.type, shm_mb=env.shm_mb()) + form=form, platform=platform_of(env.computer.os), shm_mb=env.shm_mb()) except Exception as e: problems.append(f"{type(link).__name__}: {e}") continue diff --git a/src/agent_env/env/envs/computer/utils/thin_disk.py b/src/agent_env/env/envs/computer/utils/thin_disk.py index af6411221..a5d4f951b 100644 --- a/src/agent_env/env/envs/computer/utils/thin_disk.py +++ b/src/agent_env/env/envs/computer/utils/thin_disk.py @@ -2,8 +2,9 @@ empty overlay of it as ``disk/disk.img`` and the guest file beside, and a composed disk a layer on its base. A compose boots the base on an overlay kept on this machine, gives it its parts over ssh as the base's admin, by the key agent-env made for that base, shuts it down and packs the overlay onto the base's image, so every disk composed on a base shares -one copy of it. What's its guest OS's own, how its admin is reached and what its shell runs, is a ``Guest``; Windows' -is in ``windows_disk``.""" +one copy of it. A base is found or built by ``base``, the OS's own build making its image. What's its guest OS's own, +how its admin is reached and what its shell runs, is a ``Guest``; Windows' is in ``windows_disk``, Omarchy's in +``omarchy_disk``.""" from __future__ import annotations @@ -16,8 +17,7 @@ import tarfile import time from collections.abc import Callable, Iterator -from dataclasses import dataclass -from datetime import datetime +from dataclasses import dataclass, field from pathlib import Path, PurePosixPath from typing import NamedTuple @@ -115,13 +115,12 @@ def short(key: str) -> str: @dataclass(frozen=True) class Base: - """A base: its image's tag on this machine, the folder of its admin's key, when its OS's licence runs out, for one - whose does, and, for one the image store holds that this machine doesn't yet, its ref there, pulled when a layer is - composed on it.""" + """A base: its image's tag on this machine, the folder of its admin's key, the labels its image carries, and, for one + the image store holds that this machine doesn't yet, its ref there, pulled when a layer is composed on it.""" image: str keys: Path - expires: datetime | None = None + labels: dict[str, str] = field(default_factory=dict) pinned: str | None = None @@ -129,6 +128,43 @@ def base_lock(guest: Guest, image: str) -> str: return f"{guest.name}-base:{image}" +def base(guest: Guest, key: str, build: Callable[[Path, Path], dict[str, str]], say: Callable[[str], None], + registry: Registry, stale: Callable[[dict[str, str]], str | None] = lambda labels: None) -> Base: + """``guest``'s base of ``key``: this machine's, pushed to the image store unless it holds one of ``key``; else the + one the store holds, pulled with the key to its admin it carries when a layer is composed on it; else built and + pushed there. One whose labels ``stale`` gives a reason against is built afresh. ``build`` makes ``base.img`` in the + work folder it's given, its admin reached by the new key in the folder it's given, and returns the labels the base + records beside its key. One build of a key runs at once on a machine, and another waits for it.""" + (image, tag), keys = base_names(guest, key), cache_root() / guest.name / "bases" / short(key) + with holding_locks([base_lock(guest, image)], lambda: say("waiting for the build of the same base under way")): + if present(image) and (keys / "id").exists(): + if (why := stale(labels := local_labels(image))) is None: + if registry.served(tag) is None: + say("pushing the base to the image store, which doesn't hold it") + registry.push(image, tag) + return Base(image, keys, labels) + say(f"{why}; building it afresh") + elif (pinned := registry.served(tag)) is not None: + if (why := stale(labels := registry.labels(pinned))) is None: + return Base(image, keys, labels, pinned) + say(f"{why}; building it afresh") + shutil.rmtree(keys, ignore_errors=True) + keys.mkdir(parents=True) + subprocess.run(["ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-C", f"agentenv {image}", "-f", + str(keys / "id")], check=True) + with working(guest, f"base-{short(key)}") as work: + labels = {KEY_LABEL: key, **build(work, keys)} + say("packing the base") + packed = work / "packed" + packed.mkdir() + (work / "base.img").replace(packed / "base.img") + shutil.copy(keys / "id", packed / ADMIN_KEY.name) + pack(guest, packed, image, labels, base_image=None) + say("pushing the base to the image store") + registry.push(image, tag) + return Base(image, keys, labels) + + def here(guest: Guest, base_: Base, registry: Registry, say: Callable[[str], None]) -> None: """``base_`` on this machine, with the key to its admin: pulled from the image store when it's the store's, under the base's lock, so a compose of another disk on it waits, then finds it.""" @@ -168,12 +204,9 @@ def compose(guest: Guest, base_: Base, build: Path, key: str, labels: dict[str, name = f"agentenv-{guest.name}-compose-{short(key)}" say(f"booting its base in {name}") machine = asyncio.run(machine_for(base_.image)) - container = launched(name, {"VM_KEEP": "/out/disk.img"}, ["-v", f"{out}:/out", *machine.args], machine.image) - try: - port, deadline = ssh_port(container), time.monotonic() + COMPOSE_TIMEOUT_S - while ssh(port, base_.keys, guest.user, "echo ready", check=False, timeout=60).returncode != 0: - alive(guest, container, "booting", deadline) - time.sleep(BOOT_POLL_S) + with running(name, {"VM_KEEP": "/out/disk.img"}, ["-v", f"{out}:/out", *machine.args], + machine.image) as (container, port): + answering(guest, container, port, base_.keys, time.monotonic() + COMPOSE_TIMEOUT_S) say("installing its layers over ssh") ssh(port, base_.keys, guest.user, guest.prepare) ssh(port, base_.keys, guest.user, guest.unpack, input=tarred(build), timeout=600) @@ -183,8 +216,6 @@ def compose(guest: Guest, base_: Base, build: Path, key: str, labels: dict[str, ssh(port, base_.keys, guest.user, guest.cleanup) compacted(guest, port, base_.keys, say) shut_down(guest, container, port, base_.keys) - finally: - subprocess.run(["docker", "rm", "-f", container], capture_output=True) converted(out, machine.image, backing=f"/{DISK_BASE_FILE}", args=machine.args) subprocess.run(["docker", "run", "--rm", "-v", f"{out}:/out", "--entrypoint", "qemu-img", launcher(), "rebase", "-u", "-b", Path(DISK_BASE_FILE).name, "-F", "qcow2", "/out/disk.img"], @@ -247,6 +278,25 @@ def ssh_port(container: str) -> int: return int(docker("port", container, "22/tcp").splitlines()[0].rpartition(":")[2]) +@contextlib.contextmanager +def running(name: str, variables: dict[str, str], mounts: list[str], image: str) -> Iterator[tuple[str, int]]: + """A launcher container ``launched`` starts, with the port here its guest's ssh is on; removed when its use ends, + however it ends.""" + container = launched(name, variables, mounts, image) + try: + yield container, ssh_port(container) + finally: + subprocess.run(["docker", "rm", "-f", container], capture_output=True) + + +def answering(guest: Guest, container: str, port: int, keys: Path, deadline: float) -> None: + """Return once ``guest``'s admin answers over ssh; raise as ``alive`` does when its QEMU exits or ``deadline`` passes + first.""" + while ssh(port, keys, guest.user, "echo ready", check=False, timeout=60).returncode != 0: + alive(guest, container, "booting", deadline) + time.sleep(BOOT_POLL_S) + + def alive(guest: Guest, container: str, doing: str, deadline: float) -> None: """Raise when ``container`` has stopped or ``deadline`` has passed, with the end of its output.""" running = docker("inspect", "-f", "{{.State.Running}}", container) == "true" @@ -283,8 +333,6 @@ def pack(guest: Guest, folder: Path, image: str, labels: dict[str, str], base_im f"COPY {owner} disk.img guest /{Path(DISK_FILE).parent}/\n") else: dockerfile = f"FROM {base_image}\nCOPY {owner} disk.img guest /{Path(DISK_FILE).parent}/\n" - for name in ("answers", "out"): - shutil.rmtree(folder / name, ignore_errors=True) (folder / "Dockerfile").write_text(dockerfile) build_image(folder / "Dockerfile", folder, image, platform="linux/amd64", labels=labels) @@ -311,5 +359,6 @@ def tarred(folder: Path) -> bytes: return buffer.getvalue() -def label(image: str, name: str) -> str: - return json.loads(docker("image", "inspect", "-f", "{{json .Config.Labels}}", image) or "{}").get(name, "") +def local_labels(image: str) -> dict[str, str]: + """The labels of ``image`` in this machine's Docker.""" + return json.loads(docker("image", "inspect", "-f", "{{json .Config.Labels}}", image) or "null") or {} diff --git a/src/agent_env/env/envs/computer/utils/windows_disk.py b/src/agent_env/env/envs/computer/utils/windows_disk.py index 25dc84b2c..df4e6378e 100644 --- a/src/agent_env/env/envs/computer/utils/windows_disk.py +++ b/src/agent_env/env/envs/computer/utils/windows_disk.py @@ -13,7 +13,6 @@ import secrets import shlex import shutil -import subprocess import time import urllib.request from collections.abc import Callable @@ -28,7 +27,6 @@ from agent_env.env.envs.computer.utils import thin_disk from agent_env.env.envs.computer.utils.install import WindowsInstall from agent_env.env.envs.computer.utils.thin_disk import Base, Registry -from agent_env.providers.sandbox_providers.host_images import present from agent_env.providers.sandbox_providers.local_docker.disk import launcher from agent_env.store.local_state import holding_locks @@ -121,39 +119,24 @@ def media(say: Callable[[str], None]) -> Path: def base(key: str, iso: Callable[[], Path], disk_gb: int, user: str, say: Callable[[str], None], registry: Registry) -> Base: - """The base of ``key``: this machine's, pushed to the image store unless it holds one of ``key``; else the one the - store holds, pulled, with the key to its admin it carries, when a layer is composed on it; else Windows installed - from the ISO ``iso`` gives, and pushed there. One with fewer than ``RENEW_DAYS`` of its evaluation left is installed - afresh. One install of a key runs at once on a machine, and another waits for it.""" - (image, tag), keys = thin_disk.base_names(GUEST, key), cache_root() / "windows" / "bases" / thin_disk.short(key) - with holding_locks([thin_disk.base_lock(GUEST, image)], - lambda: say("waiting for the install of the same base under way")): - if present(image) and (keys / "id").exists(): - expires = datetime.fromisoformat(thin_disk.label(image, EXPIRES_LABEL)) - if not expiring(expires.isoformat()): - if registry.served(tag) is None: - say("pushing the base to the image store, which doesn't hold it") - registry.push(image, tag) - return Base(image, keys, expires) - say(f"its base's evaluation expires {expires:%Y-%m-%d}; installing it afresh") - elif (pinned := registry.served(tag)) is not None and not expiring( - stored := registry.labels(pinned).get(EXPIRES_LABEL)): - return Base(image, keys, datetime.fromisoformat(stored), pinned) - shutil.rmtree(keys, ignore_errors=True) - keys.mkdir(parents=True) - subprocess.run(["ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-C", f"agentenv {image}", "-f", - str(keys / "id")], check=True) - media = iso() - with thin_disk.working(GUEST, f"base-{thin_disk.short(key)}") as work: - facts = _installed(work, media, keys, disk_gb, user, say) - say("packing the base") - expires = datetime.now(timezone.utc) + timedelta(minutes=facts["grace_minutes"]) - shutil.copy(keys / "id", work / thin_disk.ADMIN_KEY.name) - thin_disk.pack(GUEST, work, image, {thin_disk.KEY_LABEL: key, EXPIRES_LABEL: expires.isoformat( - timespec="seconds"), BUILD_LABEL: facts["build"]}, base_image=None) - say("pushing the base to the image store") - registry.push(image, tag) - return Base(image, keys, expires) + """The base of ``key``, as ``thin_disk.base`` finds or builds one: Windows installed from the ISO ``iso`` gives, its + session user ``user``, on a disk of ``disk_gb``, recording when its evaluation expires and its build. One with fewer + than ``RENEW_DAYS`` of its evaluation left is installed afresh.""" + def installed(work: Path, keys: Path) -> dict[str, str]: + facts = _installed(work, iso(), keys, disk_gb, user, say) + expires = datetime.now(timezone.utc) + timedelta(minutes=facts["grace_minutes"]) + return {EXPIRES_LABEL: expires.isoformat(timespec="seconds"), BUILD_LABEL: facts["build"]} + + return thin_disk.base(GUEST, key, installed, say, registry, stale=_expiring) + + +def _expiring(labels: dict[str, str]) -> str | None: + """Why a base of ``labels`` is installed afresh: its evaluation nears its end; None while it doesn't.""" + if not expiring(expires := labels.get(EXPIRES_LABEL)): + return None + if expires is None: + return "its base doesn't record when its evaluation expires" + return f"its base's evaluation expires {datetime.fromisoformat(expires):%Y-%m-%d}" def _installed(work: Path, iso: Path, keys: Path, disk_gb: int, user: str, say: Callable[[str], None]) -> dict: @@ -170,18 +153,17 @@ def _installed(work: Path, iso: Path, keys: Path, disk_gb: int, user: str, say: for name in ("setup.ps1", "agentenv-session.py"): shutil.copy(RECIPE / name, answers / name) shutil.copy(keys / "id.pub", answers / "authorized_keys") - (out / "guest").write_text("windows") + (out / "guest").write_text(GUEST.file) (keys / "passwords.json").write_text(json.dumps({ADMIN: admin_password, user: user_password})) (keys / "passwords.json").chmod(0o600) name = f"agentenv-windows-install-{keys.name}" say(f"installing Windows from {iso.name} under QEMU in {name}; this takes about 15 minutes") started = time.monotonic() - container = thin_disk.launched( - name, {"VM_KEEP": "/out/disk.img", "VM_DISK_GB": str(disk_gb), "VM_MEDIA": "/in/media.iso", - "VM_FILES": "/in/answers"}, - ["-v", f"{out}:/out", "-v", f"{iso}:/in/media.iso:ro", "-v", f"{answers}:/in/answers:ro"], launcher()) - try: - port = thin_disk.ssh_port(container) + with thin_disk.running( + name, {"VM_KEEP": "/out/disk.img", "VM_DISK_GB": str(disk_gb), "VM_MEDIA": "/in/media.iso", + "VM_FILES": "/in/answers"}, + ["-v", f"{out}:/out", "-v", f"{iso}:/in/media.iso:ro", "-v", f"{answers}:/in/answers:ro"], + launcher()) as (container, port): deadline = started + INSTALL_TIMEOUT_S while not _signed_in(port, keys, user): thin_disk.alive(GUEST, container, "installing", deadline) @@ -191,8 +173,6 @@ def _installed(work: Path, iso: Path, keys: Path, disk_gb: int, user: str, say: facts = json.loads(thin_disk.ssh(port, keys, ADMIN, powershell(_FACTS)).stdout.strip().splitlines()[-1]) thin_disk.compacted(GUEST, port, keys, say) thin_disk.shut_down(GUEST, container, port, keys) - finally: - subprocess.run(["docker", "rm", "-f", container], capture_output=True) thin_disk.converted(out, launcher()) shutil.move(out / "disk.img", work / "base.img") return facts diff --git a/src/agent_env/providers/env_providers/env_computer_provider.py b/src/agent_env/providers/env_providers/env_computer_provider.py index 95a564df8..1322c06ba 100644 --- a/src/agent_env/providers/env_providers/env_computer_provider.py +++ b/src/agent_env/providers/env_providers/env_computer_provider.py @@ -181,7 +181,7 @@ async def _boot(self, env: ComputerEnv, builds: list[tuple[str, str, dict]], par attribution=attribution, disk_size_gb=disk_size_gb, cpu=cpu, memory_mb=memory_mb, ready_s=ready_timeout(env.composed_apps(), platform_of(env.computer.os)), form=form, - held=bool(params.site_routing), os_type=env.computer.os.type, shm_mb=shm_mb) + held=bool(params.site_routing), platform=platform_of(env.computer.os), shm_mb=shm_mb) self._container_sandboxes.append(booted.sandbox) self._environment_sandboxes[name] = booted.sandbox try: diff --git a/tst/integration/computer/browser_gateway_test.py b/tst/integration/computer/browser_gateway_test.py index d163f66b3..f6b179f57 100644 --- a/tst/integration/computer/browser_gateway_test.py +++ b/tst/integration/computer/browser_gateway_test.py @@ -27,6 +27,7 @@ from agent_env.env import Env from agent_env.env.envs.computer.utils import inside from agent_env.env.gateway import AGENT_ENV_GATEWAY_MCP_PORT +from agent_env.providers.sandbox_providers.host_images import host_platform from agent_env.providers.sandbox_providers.local_docker.provider import host_url_for from agent_env.store.routing import namespace_routing from agent_env.task.teardown import teardown_run @@ -34,6 +35,7 @@ from tst.integration.computer.browser_computer_test import (DATA_PAGE, SERVE_SITE, SERVE_START_S, SITE, _clients, _kill, _ready) from tst.integration.computer.browser_computer_test import _cdp as _driver_cdp +from tst.util.published_image import pull_published_else_build pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow] @@ -48,7 +50,7 @@ @pytest.fixture(scope="module") def gateway_image(): tag = f"agentenv-test-gateway:{uuid.uuid4().hex[:8]}" - subprocess.run(["docker", "build", "-q", "-t", tag, str(GATEWAY)], check=True, capture_output=True) + pull_published_else_build("gateway", GATEWAY / "Dockerfile", GATEWAY, tag, platform=host_platform()) yield tag subprocess.run(["docker", "rmi", "-f", tag], capture_output=True) diff --git a/tst/integration/computer/conftest.py b/tst/integration/computer/conftest.py index a4360da4e..543d63d12 100644 --- a/tst/integration/computer/conftest.py +++ b/tst/integration/computer/conftest.py @@ -23,9 +23,11 @@ from agent_env.bundle.resolve import BuiltImage, resolve_bundle from agent_env.config import configure, reset_config from agent_env.env import Env +from agent_env.providers.sandbox_providers.host_images import host_platform from agent_env.store.object_store.local.grant_server import grant_server from agent_env.store.routing import namespace_routing from agent_env.task.store import get_task_instance_store +from tst.util.published_image import pull_published_else_build BASE = Path(agent_env.__file__).parent / "computers" / "artifacts" / "ubuntu-24.04" CUA_DRIVER = Path(__file__).parent / "cua-driver" @@ -40,8 +42,10 @@ @pytest.fixture(scope="session") def base(): + """The ubuntu-24.04 base for this machine's platform: the image agent-env published from these sources, pulled, or + built here when they've changed, so a run doesn't install its desktop from Ubuntu's archive.""" tag = f"localhost:5000/agentenv-test-ubuntu-24.04:{uuid.uuid4().hex[:8]}" - docker("build", "-q", "-t", tag, str(BASE)) + pull_published_else_build("ubuntu-24.04", BASE / "Dockerfile", BASE, tag, platform=host_platform()) yield tag subprocess.run(["docker", "rmi", "-f", tag], capture_output=True) diff --git a/tst/integration/computer/disk_test.py b/tst/integration/computer/disk_test.py index 4630d29a0..a1336ed9c 100644 --- a/tst/integration/computer/disk_test.py +++ b/tst/integration/computer/disk_test.py @@ -17,6 +17,7 @@ from agent_env.env.published_images import DISK from agent_env.providers.sandbox_providers.host_images import host_platform from agent_env.utils.docker.build import DockerBuildError, build_image +from tst.util.published_image import pull_published_else_build pytestmark = pytest.mark.integration @@ -46,7 +47,7 @@ @pytest.fixture(scope="module") def tools(): tag = f"agentenv-test-disk-tools:{uuid.uuid4().hex[:8]}" - build_image(DISK / "tools" / "Dockerfile", DISK / "tools", tag, platform=None) + pull_published_else_build("disk tools", DISK / "tools" / "Dockerfile", DISK / "tools", tag, platform=host_platform()) yield tag subprocess.run(["docker", "rmi", "-f", tag], capture_output=True) diff --git a/tst/integration/computer/omarchy/artifacts/check_probe/verify.py b/tst/integration/computer/omarchy/artifacts/check_probe/verify.py new file mode 100644 index 000000000..cbe1301d7 --- /dev/null +++ b/tst/integration/computer/omarchy/artifacts/check_probe/verify.py @@ -0,0 +1,10 @@ +"""The conformance probe at full against the deployed Omarchy computer: one result per check, passing when it passed or +didn't apply. cua-driver declares no known failures on Omarchy, so none is waived.""" +from agentenv_protocol.computer.conformance import run_probe + + +async def verify(mcp_url, env_url): + report = await run_probe(env_url, "full") + return [{"id": f"{check.check} {check.name}", "result": check.status in ("passed", "not_applicable"), + "description": f"{check.status}: {check.detail}" if check.detail else check.status} + for check in report.checks] diff --git a/tst/integration/computer/omarchy/envs/desktop/env.toml b/tst/integration/computer/omarchy/envs/desktop/env.toml new file mode 100644 index 000000000..649e50bf1 --- /dev/null +++ b/tst/integration/computer/omarchy/envs/desktop/env.toml @@ -0,0 +1,3 @@ +type = "computer" +computer = "omarchy-4.5" +driver = "cua-driver" diff --git a/tst/integration/computer/omarchy/tasks/probe.json b/tst/integration/computer/omarchy/tasks/probe.json new file mode 100644 index 000000000..2fd068aec --- /dev/null +++ b/tst/integration/computer/omarchy/tasks/probe.json @@ -0,0 +1,5 @@ +[ + {"id": "env", "type": "deploy_env", "env_id": "desktop"}, + {"id": "probe", "type": "env_outcome_verifier", "env_id": "desktop", "file_artifact_id": "check_probe", + "verifier_id": "probe"} +] diff --git a/tst/integration/computer/omarchy_test.py b/tst/integration/computer/omarchy_test.py new file mode 100644 index 000000000..10c8aba68 --- /dev/null +++ b/tst/integration/computer/omarchy_test.py @@ -0,0 +1,22 @@ +"""The Omarchy test bundle through ``agent-env run``, on local Docker with KVM: its desktop, cua-driver on omarchy-4.5, +composed on this machine, its base built from Omarchy's image in this machine's cache unless one is held here, passes +the conformance probe at full. Needs Docker, KVM and the image.""" + +from pathlib import Path + +import pytest +from click.testing import CliRunner + +from agent_env.cli import cli +from tst.util.capabilities import skip_without_omarchy_image + +pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow, skip_without_omarchy_image()] + +BUNDLE = Path(__file__).parent / "omarchy" + + +def test_the_omarchy_desktop_passes_the_probe_at_full(state): + ran = CliRunner().invoke(cli, ["run", str(BUNDLE), "--task", "probe"]) + + assert ran.exit_code == 0, ran.output + assert "passed (probe: 1)" in ran.output, ran.output diff --git a/tst/integration/env/envs/playwright_mcp_local_test.py b/tst/integration/env/envs/playwright_mcp_local_test.py index 7ee1710b6..1c691f14c 100644 --- a/tst/integration/env/envs/playwright_mcp_local_test.py +++ b/tst/integration/env/envs/playwright_mcp_local_test.py @@ -22,6 +22,8 @@ from agentenv_protocol.types import WELL_KNOWN_PATH from agent_env.env.envs import playwright_mcp +from agent_env.providers.sandbox_providers.host_images import host_platform +from tst.util.published_image import pull_published_else_build pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow] @@ -88,7 +90,7 @@ def _processes(container: str, pattern: str, signal: int | None = None) -> list[ @pytest.fixture(scope="module") def image() -> str: - _docker("build", "-t", IMAGE, str(CONTEXT)) + pull_published_else_build("playwright-mcp", CONTEXT / "Dockerfile", CONTEXT, IMAGE, platform=host_platform()) return IMAGE diff --git a/tst/unit/env/envs/test_computer_omarchy.py b/tst/unit/env/envs/test_computer_omarchy.py new file mode 100644 index 000000000..86d915507 --- /dev/null +++ b/tst/unit/env/envs/test_computer_omarchy.py @@ -0,0 +1,390 @@ +"""An Omarchy computer env, composed on this machine's Docker with KVM: its base built once from Omarchy's own image in +this machine's cache, then the driver installed over ssh by a bash provision script as its ``[install."linux/omarchy"]`` +says, its services systemd user units of the graphical session started through with-seed, and packed as a thin layer on +the base under a tag of its key. An Omarchy base is a disk alone, built from its source, started by systemd; its apps +are refused for now. The base's build boots Omarchy's image as a linux-desktop guest, root's key a systemd credential, +and runs the recipe, which refuses an image its artifact mislabels.""" + +import base64 +import hashlib +import io +import shlex +import subprocess +from pathlib import Path, PurePosixPath +from types import SimpleNamespace + +import pytest +from pydantic import ValidationError + +from agentenv_protocol.computer import DriverDeclaration +from agentenv_protocol.computer.app import Download, RunScript + +from agent_env.artifact.artifacts.computer import ComputerArtifact, DiskForm, DiskSource +from agent_env.artifact.artifacts.file_artifact_universe import FileArtifactUniverse +from agent_env.env.env import Env +from agent_env.env.envs.computer.driver_env import ComputerDriverEnv +from agent_env.env.envs.computer.env import ComputerEnv +from agent_env.env.envs.computer.utils import boot, composer, gate, omarchy_disk, thin_disk +from agent_env.env.envs.computer.utils.composer import Composing +from agent_env.env.envs.computer.utils.install import Install, OmarchyInstall, Service, installs +from agent_env.providers.sandbox_providers.sandbox_provider import SandboxProvider + +URL = "https://example.com/desk.tar.gz" +SOURCE = {"url": "https://example.com/omarchy.qcow2", "sha256": "ab" * 32} +OMARCHY = {"type": "linux", "distribution": "omarchy", "version": "4.5.0", "display": "wayland"} +DISK = {"source": SOURCE, "init": "systemd", "input_channel": "nocloud", "min_disk_gb": 40} +SERVE = "venv/bin/python -m agentenv_protocol.computer.serve driver.py:Desk" +OMARCHY_SUPPORTS = [{"os": {"type": "linux", "distribution": ["omarchy"], "display": ["wayland"]}, "arch": ["amd64"]}] + + +class FakeLinux(SandboxProvider): + """This machine with KVM, which boots linux/amd64 disks or doesn't.""" + + def __init__(self, platforms=frozenset({"linux/amd64"})): + self.platforms = platforms + + @classmethod + def native_platforms(cls): + return frozenset({"linux/amd64"}) + + def disk_image_platforms(self): + return self.platforms + + +class FakeRegistry: + """The env's image store as thin disks are kept in it: what it serves, by tag, pinned.""" + + def __init__(self): + self.held = {} + + def served(self, tag): + return self.held.get(tag) + + def push(self, local, tag): + self.held[tag] = f"registry.test/agentenv-computers:{tag}@sha256:{len(self.held):064x}" + return self.held[tag] + + +class FakeDocker: + """omarchy_disk and thin_disk on one machine's Docker: bases built and disks composed and pushed to the image + store.""" + + def __init__(self, tmp_path: Path, registry: FakeRegistry): + self.tmp, self.registry, self.builds, self.composed = tmp_path, registry, [], [] + + def base(self, key, image, user, distribution, version, say, registry): + self.builds.append((key, image(), user, distribution, version)) + return thin_disk.Base(f"agentenv-omarchy-base:{key[7:23]}", self.tmp / "keys") + + def compose(self, guest, base, build, key, labels, say, registry): + self.composed.append((guest, base.image, (build / "provision.sh").read_text(), + sorted(p.relative_to(build).as_posix() for p in build.rglob("*") if p.is_file()), labels)) + return registry.push(*thin_disk.disk_names(guest, key)) + + +@pytest.fixture +def docker(local_stores, cli_routing, monkeypatch, tmp_path): + fake = FakeDocker(tmp_path, FakeRegistry()) + image = tmp_path / "omarchy.qcow2" + image.write_bytes(b"qcow2") + monkeypatch.setattr(omarchy_disk, "source", lambda disk_source, say: image) + monkeypatch.setattr(omarchy_disk, "base", fake.base) + monkeypatch.setattr(thin_disk, "compose", fake.compose) + monkeypatch.setattr(thin_disk, "Registry", lambda store, repository: fake.registry) + monkeypatch.setattr(composer, "get_config", lambda: SimpleNamespace( + get_image_store_for=lambda stored_id: SimpleNamespace(ensure_repository=lambda repository: None))) + monkeypatch.setattr(composer, "_probed", lambda env, driver, form, arch, result, provider, progress: { + **result, "status": "ready"}) + return fake + + +def _base(**changes) -> ComputerArtifact: + return ComputerArtifact.put(**{ + "id": "@local/~/kit/omarchy-4.5", "arch": ["amd64"], "os": OMARCHY, "session": {"user": "omarchy"}, + "config": {"resolution": {"default": "1920x1080", "phase": "deploy"}, + "timezone": {"default": "UTC", "phase": "deploy"}, "theme": {"default": "light", "phase": "build"}}, + "forms": {"disk": DISK}, **changes}) + + +def _driver(folder: Path, services: list[Service] | None = None) -> ComputerDriverEnv: + folder.mkdir(parents=True, exist_ok=True) + names = ("driver.py", "requirements.txt", "install.sh", "settings.sh") + for name, text in zip(names, ("class Desk: ...\n", "cua-driver==0.44.0\n", "tar xzf desk.tar.gz\n", + "mkdir -p ~/.config/desk\n")): + (folder / name).write_text(text) + files = FileArtifactUniverse.put_bundled("@local/~/kit/desk__files", files={name: folder / name for name in names}) + omarchy = OmarchyInstall( + python="requirements.txt", downloads=[Download(url=URL, sha256="cd" * 32, to="desk.tar.gz", arch="amd64"), + Download(url=URL, sha256="ef" * 32, to="arm.tar.gz", arch="arm64")], + run=[RunScript(script="install.sh"), RunScript(script="settings.sh", scope="session")], + services=services or [Service(name="daemon", command="venv/bin/python driver.py"), + Service(name="driver", command=SERVE)]) + put = ComputerDriverEnv.put( + id="@local/~/kit/desk", module="driver.py:Desk", files=files, port=18765, install={"linux/omarchy": omarchy}, + declaration=DriverDeclaration.model_validate({ + "name": "desk", "supports": OMARCHY_SUPPORTS, + "platforms": [{"platform": {"os": "linux", "distribution": "omarchy", "arch": "amd64", "display": "wayland"}, + "provides": [], "tools": ["computer_screenshot"], "features": {}}]})) + return Env.get(put.id, put.version) + + +def _put(driver: ComputerDriverEnv, link: SandboxProvider, base: ComputerArtifact | None = None, + **settings) -> ComputerEnv: + return ComputerEnv.put(id="@local/~/kit/desktop", computer=base or _base(), driver=driver, + environment_name="desktop", composing=Composing(sandbox_provider=link), + computer_settings=settings or None, apps=[]) + + +def test_an_omarchy_env_composes_a_thin_disk_on_its_base_built_from_omarchys_image_the_driver_as_user_units( + docker, tmp_path): + env = _put(_driver(tmp_path / "desk"), FakeLinux(), theme="dark") + + built = env.metadata["composed"]["disk"]["amd64"] + assert built["status"] == "ready" and not built["reused"] + assert built["image"] == docker.registry.held[built["key"].removeprefix("sha256:")] + ((base_key, image, user, distribution, version),) = docker.builds + assert (image, user, distribution, version) == (tmp_path / "omarchy.qcow2", "omarchy", "omarchy", "4.5.0") + ((guest, base, script, files, labels),) = docker.composed + assert guest is omarchy_disk.GUEST and base == f"agentenv-omarchy-base:{base_key[7:23]}" + assert labels[composer.KEY_LABEL] == built["key"] + prefix = "/opt/agentenv/drivers/desk" + lines = script.splitlines() + for line in [f'cp -a "$build/context/driver/." {prefix}/', f"fetch {URL} {'cd' * 32} {prefix}/desk.tar.gz", + f"python3 -m venv {prefix}/venv", f"(cd {prefix} && PREFIX={prefix} bash {prefix}/install.sh)", + "user=omarchy", f'(cd {prefix} && runuser -u "$user" -- env HOME="$home" PREFIX={prefix} ' + f"bash {prefix}/settings.sh)", + "systemctl --global enable --quiet agentenv-driver-desk-daemon.service", + "systemctl --global enable --quiet agentenv-driver-desk-driver.service", + "systemctl mask --quiet sshd.service sshd.socket"]: + assert line in lines, line + assert "printf %s 'export AGENTENV_SETTING_THEME=dark\n' > /opt/agentenv/build-environment" in script + assert not any("arm.tar.gz" in line for line in lines) + assert any(line.startswith(f"{prefix}/venv/bin/pip install") and f"-r {prefix}/requirements.txt" in line + for line in lines) + unit = omarchy_disk.unit("driver-desk-daemon", PurePosixPath(prefix), [f"{prefix}/venv/bin/python", "driver.py"], + {"MCP_PORT": 18765}) + assert f"printf %s {shlex.quote(unit)} > /etc/systemd/user/agentenv-driver-desk-daemon.service" in script + assert f"ExecStart=/opt/agentenv/with-seed {prefix}/venv/bin/python driver.py" in unit.splitlines() + assert {"provision.sh", "context/driver/driver.py", "context/driver/install.sh"} <= set(files) + computer = env.computer_artifact() + assert computer.os.distribution == "omarchy" and computer.forms.disk.init == "systemd" + assert env.metadata["computer"]["disks"] == {"amd64": built["image"]} + + +def test_a_compose_of_the_same_inputs_reuses_the_disk_the_image_store_holds(docker, tmp_path): + driver = _driver(tmp_path / "desk") + _put(driver, FakeLinux()) + again = _put(driver, FakeLinux()) + + assert again.metadata["composed"]["disk"]["amd64"]["reused"] and len(docker.composed) == 1 + + +def test_a_machine_that_boots_no_linux_disk_composes_no_omarchy_disk(docker, tmp_path): + env = _put(_driver(tmp_path / "desk"), FakeLinux(frozenset())) + + built = env.metadata["composed"]["disk"]["amd64"] + assert built["status"] == "failed" and "boots linux/amd64 disks" in built["reason"] + assert docker.builds == [] and docker.composed == [] + + +def test_an_omarchy_computer_refuses_apps_for_now(local_stores, cli_routing, tmp_path): + app = SimpleNamespace(id="@local/~/kit/notes", declaration=SimpleNamespace(supports=OMARCHY_SUPPORTS, + install_steps=[]), + install={"linux/omarchy": Install(services=[Service(name="sync", command="sync")])}) + + problems = gate.check_app(_base(), _driver(tmp_path / "desk"), app) + + assert problems == ["app: an Omarchy computer composes its driver alone for now; @local/~/kit/notes would need its " + "install composed onto Omarchy's disk, which agent-env doesn't do yet"] + + +@pytest.mark.parametrize("changes, error", [ + ({"forms": {"disk": DISK, "container": {"image": {"id": "x", "version": 1}, "init": "supervisord", + "input_channel": "env"}}}, + "forms.container: a linux/omarchy computer's forms are disk"), + ({"forms": {}}, "forms.disk is required: every linux/omarchy computer has one"), + ({"forms": {"disk": {**DISK, "init": "supervisord"}}}, + "forms.disk.init: a linux/omarchy disk's init is systemd, not supervisord"), + ({"forms": {"disk": {**DISK, "image": {"id": "x", "version": 1}}}}, + "forms.disk: a disk names an image or the source its base is built from, not both"), + ({"forms": {"disk": {key: value for key, value in DISK.items() if key != "source"}}}, + "forms.disk: an Omarchy disk's base is built from Omarchy's own image on the machine that composes it, so it " + "names that image's source and no image"), + ({"forms": {"disk": {**{key: value for key, value in DISK.items() if key != "source"}, + "image": {"id": "x", "version": 1}}}}, + "forms.disk: an Omarchy disk's base is built from Omarchy's own image"), + ({"forms": {"disk": {**DISK, "source": {"url": "http://example.com/omarchy.qcow2", "sha256": "ab" * 32}}}}, + "url"), +], ids=["no-container", "disk-required", "systemd", "image-or-source", "names-its-source", "no-image", "https"]) +def test_an_omarchy_base_is_a_disk_alone_built_from_its_source_and_started_by_systemd( + local_stores, cli_routing, changes, error): + with pytest.raises(ValidationError, match=error): + _base(**changes) + + +def test_an_omarchy_install_names_no_apt_packages_which_a_linux_one_may(): + table = {"packages": {"apt": ["htop"]}, "services": [{"name": "driver", "command": SERVE}]} + + assert installs({"linux/omarchy": table})[1] == [ + "install.linux/omarchy: Value error, Omarchy has no apt: fetch what the layer needs with downloads, checked by " + "sha256"] + assert installs({"linux": table})[1] == [] + + +def test_a_driver_service_that_runs_as_root_is_refused_naming_omarchys_systemd(docker, tmp_path): + with pytest.raises(ValueError, match="driver: its service 'driver' runs as root, and @local/~/kit/omarchy-4.5's " + "systemd runs every program as omarchy"): + _put(_driver(tmp_path / "desk", services=[Service(name="driver", command=SERVE, scope="system")]), + FakeLinux()) + + +def test_a_disk_form_naming_no_source_is_written_without_one_so_an_agent_env_from_before_sources_reads_it(): + linux = DiskForm(init="supervisord", input_channel="nocloud", min_disk_gb=16).model_dump(by_alias=True) + + assert "source" not in linux and DiskForm(**DISK).model_dump(by_alias=True)["source"] == SOURCE + + +def test_a_time_zone_is_checked_against_ianas_before_the_computer_boots(local_stores, cli_routing): + base = _base() + + assert gate.check_computer_settings(base, {"timezone": "America/Chicago"}) == ({"timezone": "America/Chicago"}, []) + assert gate.check_computer_settings(base, {"timezone": "Mars/Base"})[1] == [ + "computer_settings.timezone: 'Mars/Base' isn't an IANA time zone, such as UTC or America/Chicago"] + + +def test_a_windows_disk_names_no_source(local_stores, cli_routing): + with pytest.raises(ValidationError, match="a Windows disk is installed from Microsoft's media"): + ComputerArtifact.put(id="@local/~/kit/windows-11", arch=["amd64"], + os={"type": "windows", "version": "11", "edition": "Enterprise Evaluation"}, + session={"user": "user"}, forms={"disk": {**DISK, "init": "agentenv-session"}}) + + +def test_an_omarchy_bases_disk_is_checked_when_its_built_not_at_put(local_stores, cli_routing): + base = _base() + + (check,) = base.metadata["validated_images"]["disk"].values() + assert check["status"] == "uninspected" and "built from its source when composed" in check["reason"] + + +def test_the_shipped_omarchy_base_is_omarchys_stable_image(): + import tomllib + + shipped = tomllib.loads((Path(composer.__file__).parents[4] / "computers" / "artifacts" / "omarchy-4.5" / + "artifact.toml").read_text()) + assert shipped["os"] == OMARCHY and shipped["forms"]["disk"]["init"] == "systemd" + assert DiskSource(**shipped["forms"]["disk"]["source"]).url.endswith("/stable/4.5.0/479/omarchy-4.5.0.qcow2") + + +def test_an_omarchy_disk_boots_at_the_size_hyprland_wants_and_other_linux_disks_as_before(): + assert boot.entry_for(boot.DISK_SIZES, "linux/omarchy") == (4.0, 8192) + assert boot.entry_for(boot.DISK_SIZES, "linux/ubuntu") == (boot.DISK_CPU, boot.DISK_MEMORY_MB) + assert boot.entry_for(boot.DISK_SIZES, "windows") == (4.0, 8192) + + +def test_a_unit_quotes_what_systemd_would_otherwise_split_or_expand(): + text = omarchy_disk.unit("app-desk-sync", PurePosixPath("/opt/agentenv/apps/desk"), + ["/opt/agentenv/apps/desk/venv/bin/python", "sync.py", "--title", "100% $HOME's \"desk\""], + {"ENVIRONMENT_NAME": "desk", "GREETING": "hello world"}) + + assert ('ExecStart=/opt/agentenv/with-seed /opt/agentenv/apps/desk/venv/bin/python sync.py --title ' + '"100%% $$HOME\'s \\"desk\\""') in text.splitlines() + assert 'Environment="GREETING=hello world"' in text.splitlines() + assert "Environment=ENVIRONMENT_NAME=desk" in text.splitlines() + assert "WantedBy=graphical-session.target" in text.splitlines() + + +def test_a_service_runs_its_relative_program_from_the_layers_prefix(): + prefix = PurePosixPath("/opt/agentenv/drivers/desk") + + assert omarchy_disk.argv("venv/bin/python -m serve 'driver.py:Desk'", prefix) == [ + "/opt/agentenv/drivers/desk/venv/bin/python", "-m", "serve", "driver.py:Desk"] + assert omarchy_disk.argv("/usr/bin/foot --server", prefix) == ["/usr/bin/foot", "--server"] + + +def test_omarchys_image_is_fetched_into_the_cache_once_and_refused_unless_its_sha256_matches(monkeypatch, tmp_path): + monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path)) + body, fetched = b"omarchy's image", [] + + def urlopen(url, timeout): + fetched.append(url) + return io.BytesIO(body) + + monkeypatch.setattr(omarchy_disk.urllib.request, "urlopen", urlopen) + right = DiskSource(url="https://example.com/omarchy.qcow2", sha256=hashlib.sha256(body).hexdigest()) + wrong = DiskSource(url="https://example.com/other.qcow2", sha256="00" * 32) + + path = omarchy_disk.source(right, lambda line: None) + omarchy_disk.source(right, lambda line: None) + with pytest.raises(RuntimeError, match=f"came with sha256 {hashlib.sha256(body).hexdigest()}, not {'00' * 32}"): + omarchy_disk.source(wrong, lambda line: None) + + assert path.read_bytes() == body and fetched == [right.url, wrong.url] + assert not list(omarchy_disk.source_path("00" * 32).parent.glob("*.part")) + + +class FakeBuilder: + """A base's build on this machine: the launcher booting Omarchy's image, its guest over ssh, and the recipe's exit.""" + + def __init__(self, monkeypatch, recipe_exit=0, recipe_err=""): + self.launched, self.ran, self.converted, self.removed = None, [], None, [] + self.recipe_exit, self.recipe_err = recipe_exit, recipe_err + monkeypatch.setattr(omarchy_disk, "launcher", lambda: "launcher:1") + monkeypatch.setattr(thin_disk, "launched", self.launch) + monkeypatch.setattr(thin_disk, "ssh_port", lambda container: 2222) + monkeypatch.setattr(thin_disk, "ssh", self.ssh) + monkeypatch.setattr(thin_disk, "compacted", lambda guest, port, keys, say: self.ran.append(("compact",))) + monkeypatch.setattr(thin_disk, "shut_down", lambda guest, container, port, keys: self.ran.append(("off",))) + monkeypatch.setattr(thin_disk, "converted", self.convert) + monkeypatch.setattr(thin_disk.subprocess, "run", self.run) + + def launch(self, name, variables, mounts, image): + self.launched = (name, variables, mounts, image) + return "container" + + def ssh(self, port, keys, user, command, *, check=True, timeout=120, input=None): + self.ran.append((user, command)) + code = self.recipe_exit if "recipe.sh" in command else 0 + return subprocess.CompletedProcess([], code, "", self.recipe_err if code else "") + + def convert(self, folder, image, backing=None, args=()): + self.converted = (image, backing, args) + (folder / "disk.img").write_text("whole") + + def run(self, argv, **kwargs): + if argv[:3] == ["docker", "rm", "-f"]: + self.removed.append(argv[3]) + return subprocess.CompletedProcess(argv, 0) + + +def test_a_base_is_built_by_booting_omarchys_image_with_roots_key_and_running_the_recipe(monkeypatch, tmp_path): + builder = FakeBuilder(monkeypatch) + keys, work, image = tmp_path / "keys", tmp_path / "work", tmp_path / "omarchy.qcow2" + keys.mkdir() + work.mkdir() + (keys / "id.pub").write_text("ssh-ed25519 AAAA agentenv\n") + + omarchy_disk._built(work, image, keys, "omarchy", "omarchy", "4.5.0", lambda line: None) + + name, variables, mounts, launcher = builder.launched + assert variables["VM_KEEP"] == "/out/disk.img" and variables["VM_DISK"] == "/in/source.qcow2" + credential = base64.b64decode(variables["VM_CREDENTIALS"].removeprefix("tmpfiles.extra=")).decode() + assert "f+ /root/.ssh/authorized_keys 0600 root root - ssh-ed25519 AAAA agentenv" in credential + assert f"{image}:/in/source.qcow2:ro" in mounts and launcher == "launcher:1" + assert [step for step in builder.ran if step[0] == "root"][-2:] == [ + ("root", "bash /root/agentenv-recipe/recipe.sh omarchy omarchy 4.5.0"), ("root", "rm -rf /root/agentenv-recipe")] + assert builder.ran[-2:] == [("compact",), ("off",)] + assert builder.converted == ("launcher:1", None, ("-v", f"{image}:/in/source.qcow2:ro")) + assert (work / "base.img").read_text() == "whole" and (work / "out" / "guest").read_text() == "linux-desktop" + assert builder.removed == ["container"] + + +def test_a_base_built_from_an_image_its_artifact_mislabels_is_refused(monkeypatch, tmp_path): + FakeBuilder(monkeypatch, recipe_exit=omarchy_disk.MISLABELLED, + recipe_err="recipe: this image is arch rolling, not omarchy 4.5.0 as its artifact declares\n") + keys, work = tmp_path / "keys", tmp_path / "work" + keys.mkdir() + work.mkdir() + (keys / "id.pub").write_text("ssh-ed25519 AAAA agentenv\n") + + with pytest.raises(RuntimeError, match="omarchy: recipe: this image is arch rolling, not omarchy 4.5.0"): + omarchy_disk._built(work, tmp_path / "arch.qcow2", keys, "omarchy", "omarchy", "4.5.0", lambda line: None) diff --git a/tst/unit/env/envs/test_computer_windows.py b/tst/unit/env/envs/test_computer_windows.py index 5f8118f10..da52e44d1 100644 --- a/tst/unit/env/envs/test_computer_windows.py +++ b/tst/unit/env/envs/test_computer_windows.py @@ -77,8 +77,9 @@ def __init__(self, tmp_path: Path, registry: FakeRegistry): def base(self, key, iso, disk_gb, user, say, registry): if key not in self.bases: self.installs.append((key, iso(), disk_gb, user)) - self.bases[key] = datetime.now(timezone.utc) + timedelta(days=89) - return thin_disk.Base(f"agentenv-windows-base:{key[7:23]}", self.tmp / "keys", self.bases[key]) + self.bases[key] = (datetime.now(timezone.utc) + timedelta(days=89)).isoformat(timespec="seconds") + return thin_disk.Base(f"agentenv-windows-base:{key[7:23]}", self.tmp / "keys", + {windows_disk.EXPIRES_LABEL: self.bases[key]}) def compose(self, guest, base, build, key, labels, say, registry): self.composed.append((base.image, (build / "provision.ps1").read_bytes().decode(), @@ -329,19 +330,22 @@ def test_a_base_is_reused_until_its_evaluation_nears_its_end_then_installed_afre keys.mkdir(parents=True) (keys / "id").write_text("key") expires = datetime.now(timezone.utc) + timedelta(days=days_left) - installed = [] - monkeypatch.setattr(windows_disk, "present", lambda image: True) - monkeypatch.setattr(thin_disk, "label", lambda image, name: expires.isoformat(timespec="seconds")) - monkeypatch.setattr(windows_disk.subprocess, "run", lambda argv, **kwargs: (keys / "id").write_text("new")) - monkeypatch.setattr(windows_disk, "_installed", lambda *args: installed.append(args) or { - "grace_minutes": 90 * 24 * 60, "build": "26200.6584"}) + installed, said = [], [] + monkeypatch.setattr(thin_disk, "present", lambda image: True) + monkeypatch.setattr(thin_disk, "local_labels", lambda image: { + windows_disk.EXPIRES_LABEL: expires.isoformat(timespec="seconds")}) + monkeypatch.setattr(thin_disk.subprocess, "run", lambda argv, **kwargs: (keys / "id").write_text("new")) + monkeypatch.setattr(windows_disk, "_installed", lambda work, *args: (work / "base.img").write_text("") or + installed.append(args) or {"grace_minutes": 90 * 24 * 60, "build": "26200.6584"}) monkeypatch.setattr(thin_disk, "pack", lambda guest, folder, image, labels, base_image: None) registry = FakeRegistry() - base = windows_disk.base(key, lambda: tmp_path / "windows.iso", 64, "user", lambda line: None, registry) + base = windows_disk.base(key, lambda: tmp_path / "windows.iso", 64, "user", said.append, registry) assert len(installed) == installs and base.image == f"agentenv-windows-base:{key[7:23]}" - assert (base.expires - datetime.now(timezone.utc)).days == (days_left - 1 if not installs else 89) + left = datetime.fromisoformat(base.labels[windows_disk.EXPIRES_LABEL]) - datetime.now(timezone.utc) + assert left.days == (days_left - 1 if not installs else 89) + assert (f"its base's evaluation expires {expires:%Y-%m-%d}; building it afresh" in said) == bool(installs) assert list(registry.held) == [f"windows-base-{key[7:23]}"] @@ -352,8 +356,9 @@ def test_a_base_this_machine_holds_goes_to_an_image_store_lacking_it_once(monkey keys.mkdir(parents=True) (keys / "id").write_text("key") expires = datetime.now(timezone.utc) + timedelta(days=30) - monkeypatch.setattr(windows_disk, "present", lambda image: True) - monkeypatch.setattr(thin_disk, "label", lambda image, name: expires.isoformat(timespec="seconds")) + monkeypatch.setattr(thin_disk, "present", lambda image: True) + monkeypatch.setattr(thin_disk, "local_labels", lambda image: { + windows_disk.EXPIRES_LABEL: expires.isoformat(timespec="seconds")}) registry, pushed = FakeRegistry(), [] push = registry.push registry.push = lambda local, tag: pushed.append(local) or push(local, tag) @@ -379,7 +384,7 @@ def pull(pinned): monkeypatch.setattr(thin_disk, "take_admin_key", lambda image, at: (at.mkdir(), (at / "id").write_text("key"), made.set())) registry = SimpleNamespace(pull=pull) - base = thin_disk.Base("agentenv-windows-base:1", keys, datetime.now(timezone.utc), "registry.test/base@sha256:1") + base = thin_disk.Base("agentenv-windows-base:1", keys, pinned="registry.test/base@sha256:1") composes = [threading.Thread(target=thin_disk.here, args=(windows_disk.GUEST, base, registry, lambda line: None)) for _ in range(2)] @@ -399,13 +404,13 @@ def test_a_machine_without_the_base_takes_the_image_stores_unless_its_evaluation pinned = registry.push("agentenv-windows-base:x", f"windows-base-{key[7:23]}") expires = datetime.now(timezone.utc) + timedelta(days=60) registry.labels = lambda ref: {windows_disk.EXPIRES_LABEL: expires.isoformat(timespec="seconds")} - monkeypatch.setattr(windows_disk, "present", lambda image: False) + monkeypatch.setattr(thin_disk, "present", lambda image: False) fetched = [] base = windows_disk.base(key, lambda: fetched.append("iso"), 64, "user", lambda line: None, registry) assert base.pinned == pinned and base.image == f"agentenv-windows-base:{key[7:23]}" and fetched == [] - assert (base.expires - datetime.now(timezone.utc)).days == 59 + assert (datetime.fromisoformat(base.labels[windows_disk.EXPIRES_LABEL]) - datetime.now(timezone.utc)).days == 59 def test_a_windows_computers_display_is_1920x1080_for_now(local_stores, cli_routing): diff --git a/tst/unit/env/envs/test_disk_recipe.py b/tst/unit/env/envs/test_disk_recipe.py index c881695ad..6d500448c 100644 --- a/tst/unit/env/envs/test_disk_recipe.py +++ b/tst/unit/env/envs/test_disk_recipe.py @@ -376,13 +376,40 @@ def test_a_windows_disk_boots_from_secure_boot_on_ahci_and_e1000e_with_a_fresh_t assert running.wait(10) == 143 -@pytest.mark.parametrize("guest, arch, why", [("windows", "arm64", "a windows guest boots on amd64 alone, not arm64"), - ("plan9", "amd64", "no guest plan9")]) -def test_the_launcher_refuses_a_guest_it_cant_boot(launcher, tmp_path, guest, arch, why): +@pytest.mark.parametrize("resolution, display", [(None, "xres=1920,yres=1080"), ("MTI4MHg4MDA=", "xres=1280,yres=800")], + ids=["default", "the-computers-own"]) +def test_a_linux_desktop_guest_gets_a_virtio_gpu_at_its_resolution_trim_its_size_and_its_credentials( + launcher, tmp_path, resolution, display): + """A Wayland desktop's compositor needs a GPU, at the mode its monitor rule asks for.""" + launch, calls = launcher + disk = tmp_path / "disk" / "disk.img" + disk.parent.mkdir() + (disk.parent / "guest").write_text("linux-desktop\n") + running = launch(VM_DISK=str(disk), VM_PORTS="18765", VM_CREDENTIALS="tmpfiles.extra=ZCAvcnVu ssh.k=YQ==", + **({"VM_ENV_AGENTENV_SETTING_RESOLUTION": resolution} if resolution else {})) + (qemu,) = _logged(calls, "qemu-system-x86_64") + + assert "-machine q35,accel=" in qemu and "file=/usr/share/OVMF/OVMF_CODE_4M.fd " in qemu and "secboot" not in qemu + assert f"-vga none -device virtio-vga,{display} " in qemu and "-smp 4 -m 8192 " in qemu + assert ",format=qcow2,if=virtio,discard=unmap,detect-zeroes=unmap " in qemu + assert ("-smbios type=11,value=io.systemd.credential.binary:tmpfiles.extra=ZCAvcnVu " + "-smbios type=11,value=io.systemd.credential.binary:ssh.k=YQ== ") in qemu + running.terminate() + assert running.wait(10) == 143 + + +@pytest.mark.parametrize("guest, arch, variables, why", [ + ("windows", "arm64", {}, "a windows guest boots on amd64 alone, not arm64"), + ("plan9", "amd64", {}, "no guest plan9"), + ("linux-desktop", "arm64", {}, "a linux-desktop guest boots on amd64 alone, not arm64"), + ("linux", "amd64", {"VM_CREDENTIALS": "ssh.k=YQ=="}, "VM_CREDENTIALS are for a linux-desktop guest, not linux"), + ("linux-desktop", "amd64", {"VM_ENV_AGENTENV_SETTING_RESOLUTION": "d2lkZQ=="}, + "the computer's resolution, wide, isn't WIDTHxHEIGHT")]) +def test_the_launcher_refuses_a_guest_it_cant_boot(launcher, tmp_path, guest, arch, variables, why): launch, calls = launcher (tmp_path / "guest").write_text(guest) - refused = launch(VM_DISK=str(tmp_path / "disk.img"), VM_ARCH=arch) + refused = launch(VM_DISK=str(tmp_path / "disk.img"), VM_ARCH=arch, **variables) assert refused.wait(10) == 2 and why in refused.stderr.read() assert not calls.exists() or "qemu" not in calls.read_text() diff --git a/tst/unit/env/envs/test_thin_disk.py b/tst/unit/env/envs/test_thin_disk.py index f179ef2d2..c0d9345dd 100644 --- a/tst/unit/env/envs/test_thin_disk.py +++ b/tst/unit/env/envs/test_thin_disk.py @@ -1,6 +1,7 @@ import io import subprocess import tarfile +from pathlib import Path from types import SimpleNamespace import pytest @@ -86,3 +87,55 @@ def test_a_provision_that_fails_stops_the_compose_with_its_output_and_removes_th assert [command for _, command in machine.ran][-1] == "provision" and machine.removed == ["container"] assert machine.packed is None and machine.pushed == [] + + +def _keygen(argv, **kwargs): + Path(argv[-1]).write_text("key") + + +def test_a_base_is_built_once_then_reused_packed_from_its_image_and_key_alone_with_the_labels_its_build_gives( + monkeypatch, tmp_path): + monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path / "cache")) + built, packed, held = [], [], {} + monkeypatch.setattr(thin_disk, "present", lambda image: bool(packed)) + monkeypatch.setattr(thin_disk, "local_labels", lambda image: packed[-1][1]) + monkeypatch.setattr(thin_disk.subprocess, "run", _keygen) + monkeypatch.setattr(thin_disk, "pack", lambda guest, folder, image, labels, base_image: packed.append( + (sorted(path.name for path in folder.iterdir()), labels, base_image))) + registry = SimpleNamespace(served=held.get, push=lambda local, tag: held.setdefault(tag, f"registry.test/{tag}")) + + def build(work, keys): + built.append(keys) + (work / "base.img").write_text("disk") + (work / "scratch").mkdir() + return {"dev.agentenv.test": "1"} + + key = "sha256:" + "ef" * 32 + first = thin_disk.base(GUEST, key, build, lambda line: None, registry) + again = thin_disk.base(GUEST, key, build, lambda line: None, registry) + + assert len(built) == 1 and packed == [(["admin.key", "base.img"], {thin_disk.KEY_LABEL: key, + "dev.agentenv.test": "1"}, None)] + assert first == again == thin_disk.Base(f"agentenv-plan9-base:{'ef' * 8}", built[0], packed[0][1]) + assert list(held) == [f"plan9-base-{'ef' * 8}"] + + +def test_a_base_the_store_holds_that_its_guest_calls_stale_is_built_afresh_saying_why(monkeypatch, tmp_path): + monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path / "cache")) + said, built = [], [] + monkeypatch.setattr(thin_disk, "present", lambda image: False) + monkeypatch.setattr(thin_disk.subprocess, "run", _keygen) + monkeypatch.setattr(thin_disk, "pack", lambda guest, folder, image, labels, base_image: None) + registry = SimpleNamespace(served=lambda tag: "registry.test/old", labels=lambda pinned: {"age": "old"}, + push=lambda local, tag: "registry.test/new") + + def build(work, keys): + built.append(keys) + (work / "base.img").write_text("disk") + return {"age": "new"} + + base = thin_disk.base(GUEST, "sha256:" + "aa" * 32, build, said.append, registry, + stale=lambda labels: "it's old" if labels["age"] == "old" else None) + + assert len(built) == 1 and base.pinned is None and base.labels["age"] == "new" + assert "it's old; building it afresh" in said diff --git a/tst/unit/test_published_image.py b/tst/unit/test_published_image.py index 078e6a8f5..49fc7eaf1 100644 --- a/tst/unit/test_published_image.py +++ b/tst/unit/test_published_image.py @@ -44,6 +44,14 @@ def test_a_published_image_of_the_same_sources_is_pulled_by_its_amd64_digest_and assert docker["calls"] == [["pull", "--platform", "linux/amd64", pinned], ["tag", pinned, "env-gateway"]] +def test_an_image_for_another_platform_is_pulled_by_that_platforms_digest(docker): + published_image.pull_published_else_build("gateway", IMAGE.dockerfile, IMAGE.dockerfile.parent, "base", + platform="linux/arm64") + + pinned = f"{IMAGE.ref()}@sha256:{'b' * 64}" + assert docker["calls"] == [["pull", "--platform", "linux/arm64", pinned], ["tag", pinned, "base"]] + + @pytest.mark.parametrize("failing", ["pull", "tag"]) def test_a_failed_pull_or_tag_builds_the_image_instead(docker, failing, caplog): docker["fails"][failing] = "failed to copy: httpReadSeeker: failed open" diff --git a/tst/util/capabilities.py b/tst/util/capabilities.py index a7df686ff..1d4d46d17 100644 --- a/tst/util/capabilities.py +++ b/tst/util/capabilities.py @@ -12,6 +12,8 @@ import os import subprocess +import tomllib +from pathlib import Path import pytest @@ -73,6 +75,12 @@ #: cache once, about 7 GB; a test that composes one needs it there already, and KVM, which boots windows/amd64 disks. WINDOWS_MEDIA = "windows_media" +#: Composing an Omarchy computer builds its base from Omarchy's own VM image, which a run fetches into this machine's +#: cache once, about 4 GB; a test that composes one needs it there already, and KVM, which boots linux/amd64 disks. +OMARCHY_IMAGE = "omarchy_image" +#: The shipped Omarchy base, whose source the capability looks for. +OMARCHY_ARTIFACT = Path(__file__).parents[2] / "src" / "agent_env" / "computers" / "artifacts" / "omarchy-4.5" / "artifact.toml" + def missing_capability_reason(capability: str) -> str: return f"{MISSING_CAPABILITY_PREFIX}{capability}" @@ -223,6 +231,21 @@ def windows_media_is_available() -> bool: return "windows/amd64" in LocalSandboxProvider.disk_image_platforms() and media_path().exists() +def omarchy_image_is_available() -> bool: + """Whether this machine boots Omarchy's disks and holds the image the shipped Omarchy base is built from.""" + from agent_env.env.envs.computer.utils.omarchy_disk import source_path + from agent_env.providers.sandbox_providers.local_docker.provider import LocalSandboxProvider + + artifact = OMARCHY_ARTIFACT.read_text() + sha256 = tomllib.loads(artifact)["forms"]["disk"]["source"]["sha256"] + return "linux/amd64" in LocalSandboxProvider.disk_image_platforms() and source_path(sha256).exists() + + +def skip_without_omarchy_image() -> pytest.MarkDecorator: + """Collection-time ``skipif`` for tests that compose an Omarchy computer on local Docker.""" + return pytest.mark.skipif(not omarchy_image_is_available(), reason=missing_capability_reason(OMARCHY_IMAGE)) + + def skip_without_windows_media() -> pytest.MarkDecorator: """Collection-time ``skipif`` for tests that compose a Windows computer on local Docker.""" return pytest.mark.skipif(not windows_media_is_available(), reason=missing_capability_reason(WINDOWS_MEDIA)) diff --git a/tst/util/published_image.py b/tst/util/published_image.py index cd32f675a..bcc495ab7 100644 --- a/tst/util/published_image.py +++ b/tst/util/published_image.py @@ -1,5 +1,5 @@ -"""Tag an image on this machine's Docker the way the gateway tier's fixtures do: pulled when the registry serves the -image agent-env published from the same Dockerfile and build context, else built here.""" +"""Tag an image on this machine's Docker the way the gateway tier's and the base computer's fixtures do: pulled when the +registry serves the image agent-env published from the same Dockerfile and build context, else built here.""" from __future__ import annotations @@ -17,32 +17,34 @@ _PUBLISHED_IMAGES = {(image.dockerfile, image.dockerfile.parent): image for image in PUBLISHED_IMAGES} -def pull_published_else_build(name: str, dockerfile: Path, context: Path, tag: str) -> None: - """``tag``, pulled for linux/amd64 when the registry serves the published image of these sources, whose tag is a +def pull_published_else_build(name: str, dockerfile: Path, context: Path, tag: str, + platform: str = "linux/amd64") -> None: + """``tag``, pulled for ``platform`` when the registry serves the published image of these sources, whose tag is a digest of them, so a change to them is built; also built when the lookup or pull fails. A failed build raises.""" published = _PUBLISHED_IMAGES.get((dockerfile, context)) try: - if pinned := published and _amd64_image(published_pinned(published)): + if pinned := published and _platform_image(published_pinned(published), platform): logger.info(f"Pulling {name}'s published image {pinned}...") - _docker("pull", "--platform", "linux/amd64", pinned) + _docker("pull", "--platform", platform, pinned) _docker("tag", pinned, tag) return except (RuntimeError, ValueError) as e: logger.warning(f"Pulling {name}'s published image failed, so it's built: {e}") logger.info(f"Building {name} Docker image...") - _docker("build", "--platform", "linux/amd64", "-f", str(dockerfile), "-t", tag, str(context)) + _docker("build", "--platform", platform, "-f", str(dockerfile), "-t", tag, str(context)) -def _amd64_image(pinned: str | None) -> str | None: - """``pinned``'s linux/amd64 image, pinned by its own digest: the containerd image store keeps a pulled index whole, - and ``docker save`` of one it holds only some platforms of fails.""" +def _platform_image(pinned: str | None, platform: str) -> str | None: + """``pinned``'s image for ``platform``, pinned by its own digest: the containerd image store keeps a pulled index + whole, and ``docker save`` of one it holds only some platforms of fails.""" if pinned is None: return None manifest = read_manifest(pinned, None) if "manifests" not in manifest: return pinned + os, _, architecture = platform.partition("/") digest = next((m["digest"] for m in manifest["manifests"] - if m.get("platform", {}).get("os") == "linux" and m["platform"].get("architecture") == "amd64"), None) + if m.get("platform", {}).get("os") == os and m["platform"].get("architecture") == architecture), None) return digest and f"{pinned.partition('@')[0]}@{digest}"