From b6109fbbe3d24d2898b38015859c89a4643e281e Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Tue, 10 Nov 2026 16:50:01 -0800 Subject: [PATCH] feat(computers): cua-driver 0.47.4 on Omarchy, its Hyprland plugin built by the release's kit Omarchy's base builds trycua's cua-hyprland-plugin from the 0.47.4 release's source with the release's own build kit, unmodified, on what the kit pins: Arch's Hyprland 0.56.2-1 and GCC 16.1.1 20260728, from that day of the Arch Linux Archive, with that day's Hyprland libraries. It ends the builder's desktop session before Hyprland's config names the plugin. The driver takes cua-driver 0.47.4 from the release's Linux tarball on Hyprland, whose wheel isn't on PyPI, with 0.47.4's manifest. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/computers/README.md | 20 ++++-- .../computers/disk/omarchy/arch-day.conf | 15 +++++ .../computers/disk/omarchy/recipe.sh | 66 +++++++++++++------ src/agent_env/drivers/README.md | 21 +++--- .../drivers/envs/cua-driver/driver.py | 14 ++-- .../drivers/envs/cua-driver/env.toml | 4 ++ .../envs/cua-driver/install-hyprland.sh | 7 ++ .../drivers/envs/cua-driver/mcp-hyprland.json | 10 +-- .../envs/cua-driver/requirements-omarchy.txt | 9 ++- tst/unit/drivers/cua_driver_test.py | 22 ++++++- 10 files changed, 137 insertions(+), 51 deletions(-) create mode 100644 src/agent_env/computers/disk/omarchy/arch-day.conf create mode 100644 src/agent_env/drivers/envs/cua-driver/install-hyprland.sh diff --git a/src/agent_env/computers/README.md b/src/agent_env/computers/README.md index f1a450fcb..b9fd9ff67 100644 --- a/src/agent_env/computers/README.md +++ b/src/agent_env/computers/README.md @@ -202,13 +202,19 @@ by `source`, a URL and a sha256, fetched once into this machine's cache and chec launcher as a `linux-desktop` guest (a virtio GPU at the computer's resolution, 4 vCPUs and 8 GB), with root's ssh key passed as a systemd credential, Omarchy signing its user `omarchy` in as it does at every boot, and made a base by `disk/omarchy/recipe.sh`, run as root over ssh. The recipe refuses an image whose os-release isn't the artifact's -`omarchy` `4.5.0`; installs cua-driver's Hyprland plugin, pinned by URL and sha256 to the package Omarchy builds for its -Hyprland; adds `agentenv.lua` to Hyprland's config (the plugin loaded at start, a US keymap, the idle cursor hidden, the -display's mode); installs the seed unit; has SDDM log back in after a logout; sets Omarchy's toggles for an unattended -desktop (staying awake, no animations, no screensaver or suspend, notifications silenced); turns off crash toasts, -dictation and its keys, housekeeping timers that would run at boot, and the firewall, which lets in only ssh and -LocalSend while the launcher forwards only the computer's own ports; and locks `omarchy`'s password and leaves the -machine id empty. Trimmed, the image becomes the base's, with the key to root, and is pushed to the env's image store +`omarchy` `4.5.0`; adds `agentenv.lua` to Hyprland's config (cua-driver's Hyprland plugin loaded at start, a US keymap, +the idle cursor hidden, the display's mode); installs the seed unit; has SDDM log back in after a logout; sets Omarchy's +toggles for an unattended desktop (staying awake, no animations, no screensaver or suspend, notifications silenced); +turns off crash toasts, dictation and its keys, housekeeping timers that would run at boot, and the firewall, which lets +in only ssh and LocalSend while the launcher forwards only the computer's own ports; builds the plugin; and locks +`omarchy`'s password and leaves the machine id empty. The plugin is the one cua-driver's release at the driver's +Hyprland pin ships, built from its source by that release's kit, unmodified, as `omarchy` with `makepkg`, each archive +pinned by sha256. The kit builds only on what it pins, Arch's Hyprland 0.56.2-1 and the GCC 16.1.1 it was built with, +where Omarchy 4.5 ships newer builds of both, so first the recipe installs those from that day of the Arch Linux +Archive (`disk/omarchy/arch-day.conf`, packages signature-checked by pacman), with the Hyprland libraries of that day, +and cmake and ninja, which it removes after the build. It ends the builder's desktop session first, as the kit says to +install it, and only then adds the drop-in: Hyprland reloads its config whenever a file of it changes, and the one +running there can neither load the plugin nor show, without hanging, the error a config naming a missing one makes. Trimmed, the image becomes the base's, with the key to root, and is pushed to the env's image store under a tag of its recipe and source. Name it as `omarchy-4.5`, or by its id, `@local/agentenv-framework/computers/omarchy-4.5`. diff --git a/src/agent_env/computers/disk/omarchy/arch-day.conf b/src/agent_env/computers/disk/omarchy/arch-day.conf new file mode 100644 index 000000000..ce620bbf5 --- /dev/null +++ b/src/agent_env/computers/disk/omarchy/arch-day.conf @@ -0,0 +1,15 @@ +# pacman's configuration for the recipe's one transaction from the Arch Linux Archive: the day of Hyprland 0.56.2-1 and +# GCC 16.1.1 20260728, which trycua's Hyprland plugin kit pins, as a day snapshot, its packages signature-checked by +# pacman. Omarchy's own pacman.conf stays as it is. +[options] +Architecture = auto +SigLevel = Required DatabaseOptional +LocalFileSigLevel = Optional +DownloadUser = alpm +ParallelDownloads = 5 + +[core] +Server = https://archive.archlinux.org/repos/2026/08/08/$repo/os/$arch + +[extra] +Server = https://archive.archlinux.org/repos/2026/08/08/$repo/os/$arch diff --git a/src/agent_env/computers/disk/omarchy/recipe.sh b/src/agent_env/computers/disk/omarchy/recipe.sh index 3fbea337a..ea5dd06c4 100755 --- a/src/agent_env/computers/disk/omarchy/recipe.sh +++ b/src/agent_env/computers/disk/omarchy/recipe.sh @@ -1,19 +1,27 @@ #!/bin/bash # recipe.sh USER DISTRIBUTION VERSION: run once as root over ssh in an Omarchy base's builder, Omarchy's own image booted # by the QEMU launcher with its session user USER signed in. It refuses an image that isn't the DISTRIBUTION and VERSION -# its artifact declares, then makes it a computer's base: cua-driver's Hyprland plugin, pinned to the package built for -# this Omarchy's Hyprland; agent-env's Hyprland drop-in; the seed unit that hands each boot its deploy's environment; -# SDDM logging back in after a logout; Omarchy's toggles for an unattended desktop; and off, what a trial doesn't want: -# crash toasts, dictation and its keys, housekeeping timers that would run at boot, and the firewall, which would stop -# the computer's own ports. The session user's password is locked, and the machine id left empty, as Omarchy's own -# sealed images leave it. +# its artifact declares, then makes it a computer's base: agent-env's Hyprland drop-in; the seed unit that hands each +# boot its deploy's environment; SDDM logging back in after a logout; Omarchy's toggles for an unattended desktop; off, +# what a trial doesn't want: crash toasts, dictation and its keys, housekeeping timers that would run at boot, and the +# firewall, which would stop the computer's own ports; and cua-driver's Hyprland plugin, built by its release's kit on +# what the kit pins. The session user's password is locked, and the machine id left empty, as Omarchy's own sealed +# images leave it. set -euo pipefail user=$1 distribution=$2 version=$3 here=$(cd "$(dirname "$0")" && pwd) home=$(getent passwd "$user" | cut -d: -f6) -#: cua-driver's Hyprland plugin, as Omarchy's own repository builds it for Omarchy 4.5's Hyprland 0.56.2-4. -plugin_url=https://pkgs.omarchy.org/stable/x86_64/cua-hyprland-plugin-0.33.2-3-x86_64.pkg.tar.zst -plugin_sha256=48a25c70dafdd423118c9937d3b469953e89a15cc0f278662f8fd728b523c47e +#: cua-driver's Hyprland plugin, as its release's source and build kit, which the driver's Hyprland pin matches. +plugin_release=https://github.com/trycua/cua/releases/download/cua-driver-rs-v0.47.4 +plugin=cua-hyprland-plugin-0.47.4-09b38ab661aa8b7d46deb5ff98ca00063019f592 +plugin_sha256=fd2e7f6426388753c19b8898a920f1f0d19f3ce81d5f7d7ae9ef3a2dee2c3b98 +plugin_kit_sha256=4da83523664049e9fa5a5f3269294dfad9602e6c5a02b0534a8c6f5b29920354 +#: What the kit builds on, in place of Omarchy's newer builds: Arch's Hyprland 0.56.2-1 with the libraries it was built +#: against, and the GCC 16.1.1 it was built with, from that day of the Arch Linux Archive (arch-day.conf). +pinned=(hyprland aquamarine hyprtoolkit hyprland-guiutils gcc gcc-libs libasan libatomic libgcc libgfortran libgomp + libhwasan liblsan libobjc libquadmath libstdc++ libtsan libubsan) +#: What the kit builds with that Omarchy lacks, removed once it has. +build_tools=(cmake ninja) . /etc/os-release if [ "$ID" != "$distribution" ] || [ "$VERSION_ID" != "$version" ]; then @@ -22,12 +30,6 @@ if [ "$ID" != "$distribution" ] || [ "$VERSION_ID" != "$version" ]; then fi echo "os-release: ID=$ID VERSION_ID=$VERSION_ID" -plugin=$(mktemp --suffix=.pkg.tar.zst) -curl -fsSL --retry 5 -o "$plugin" "$plugin_url" -echo "$plugin_sha256 $plugin" | sha256sum --check --quiet -pacman -U --noconfirm --needed "$plugin" -rm -f "$plugin" - install -d /opt/agentenv install -m 644 "$here/seed-env" /opt/agentenv/seed-env install -m 755 "$here/agentenv-seed" "$here/with-seed" /opt/agentenv/ @@ -37,10 +39,6 @@ systemctl enable --quiet agentenv-seed.service install -d /etc/sddm.conf.d install -m 644 "$here/sddm-relogin.conf" /etc/sddm.conf.d/zz-agentenv.conf -install -m 644 -o "$user" -g "$user" "$here/agentenv.lua" "$home/.config/hypr/agentenv.lua" -grep -q 'require("hypr.agentenv")' "$home/.config/hypr/hyprland.lua" \ - || echo 'require("hypr.agentenv")' >> "$home/.config/hypr/hyprland.lua" - systemctl --global mask --quiet omarchy-crash-watch.service superwhisper.service systemctl mask --quiet snapper-cleanup.timer plocate-updatedb.timer man-db.timer shadow.timer \ archlinux-keyring-wkd-sync.timer @@ -68,6 +66,36 @@ session omarchy toggle notification silencing session omarchy toggle screensaver session omarchy toggle suspend +# The rest with the desktop session ended, as the plugin's kit says to install it: Hyprland reloads its config whenever +# a file of it changes, and the Hyprland running here can neither load the plugin nor show the error its config naming a +# missing one makes without hanging. The computer boots the pinned Hyprland with the drop-in loading the plugin. +systemctl stop sddm.service +loginctl terminate-user "$user" +for _ in $(seq 60); do pgrep -x Hyprland > /dev/null || break; sleep 1; done +if pgrep -x Hyprland > /dev/null; then + echo "recipe: Hyprland still runs a minute after the desktop session ended" >&2 + exit 1 +fi +install -m 644 -o "$user" -g "$user" "$here/agentenv.lua" "$home/.config/hypr/agentenv.lua" +grep -q 'require("hypr.agentenv")' "$home/.config/hypr/hyprland.lua" \ + || echo 'require("hypr.agentenv")' >> "$home/.config/hypr/hyprland.lua" +# The plugin's kit runs makepkg as an ordinary user, in a folder of its own, and its package goes in from there. +pacman --config "$here/arch-day.conf" -Sy +pacman --config "$here/arch-day.conf" -S --needed --noconfirm "${pinned[@]}" "${build_tools[@]}" +kit=$(mktemp -d) download=$(mktemp -d) +curl -fsSL --retry 5 -o "$kit/$plugin.tar.gz" "$plugin_release/$plugin.tar.gz" +curl -fsSL --retry 5 -o "$download/kit.tar.gz" "$plugin_release/$plugin-build-kit.tar.gz" +echo "$plugin_sha256 $kit/$plugin.tar.gz" | sha256sum --check --quiet +echo "$plugin_kit_sha256 $download/kit.tar.gz" | sha256sum --check --quiet +tar -xzf "$download/kit.tar.gz" -C "$kit" +(cd "$kit" && sha256sum --check --quiet SHA256SUMS) +chown -R "$user:$user" "$kit" +runuser -u "$user" -- env HOME="$home" bash -c 'cd "$1" && makepkg --noconfirm' makepkg "$kit" +pacman -U --noconfirm "$kit"/cua-hyprland-plugin-*.pkg.tar.zst +pacman -Rns --noconfirm "${build_tools[@]}" +rm -rf "$kit" "$download" /var/lib/pacman/sync/core.db* /var/lib/pacman/sync/extra.db* +pacman -Q hyprland gcc cua-hyprland-plugin + passwd --lock --quiet "$user" 2>/dev/null || passwd -l "$user" > /dev/null truncate -s 0 /etc/machine-id echo "recipe: done" diff --git a/src/agent_env/drivers/README.md b/src/agent_env/drivers/README.md index cf8b845c0..95c1afac6 100644 --- a/src/agent_env/drivers/README.md +++ b/src/agent_env/drivers/README.md @@ -81,11 +81,14 @@ cua-driver's UI Automation tree, whose walk of it fails. On a Hyprland desktop (Wayland), as Omarchy's is, cua-driver runs its Wayland backend (`CUA_DRIVER_RS_ENABLE_WAYLAND=1`) and sends input through the seats of Hyprland's `cua-hyprland-plugin`. Hyprland's session turns this on (`HYPRLAND_INSTANCE_SIGNATURE`), not any Wayland one, since the driver reaches Hyprland's IPC; `supports` names -Omarchy's Wayland desktop on amd64, which installs `[install."linux/omarchy"]`: `requirements-omarchy.txt` and the two -services, with no Hyprland plugin, which Omarchy's base carries. It runs cua-driver 0.44.0, its own pin, since 0.38.0 -refuses background input there, the delivery cua-driver's own tools default to: `computer-ready` checks its wheel's -executable (`hyprland/amd64` in `EXECUTABLES`) and the tools `mcp-hyprland.json` pins, which `requirements-omarchy.txt` -installs, while `requirements.txt` keeps 0.38.0 for the other desktops. +Omarchy's Wayland desktop on amd64, which installs `[install."linux/omarchy"]`: `requirements-omarchy.txt`, cua-driver's +release tarball for Linux, whose executable `install-hyprland.sh` unpacks into the driver's `cua-driver` folder, and the +two services, with no Hyprland plugin: Omarchy's base carries the plugin of the same cua-driver release, built by its +kit. It runs cua-driver 0.47.4, its own pin, which takes background input from any native Wayland app through the +plugin's agent seats, where 0.38.0 refuses background input there, the delivery cua-driver's own tools default to; it +comes from the release, since 0.47.4's wheel isn't on PyPI, and moving the pin moves the plugin's release in +`computers/disk/omarchy/recipe.sh` with it. `computer-ready` checks the release's executable (`hyprland/amd64` in +`EXECUTABLES`) and the tools `mcp-hyprland.json` pins, while `requirements.txt` keeps 0.38.0 for the other desktops. The daemon starts once the compositor's socket answers, on a socket in the session user's runtime folder. What the driver asks Hyprland (`hyprctl -j`) is in `hyprland.py`, beside `driver.py`. Text goes to the desktop in the foreground, @@ -116,15 +119,17 @@ failure. The driver's services otherwise restart with the graphical session, as To move it to another cua-driver version: -1. Pin the version in `requirements.txt`, and Hyprland's own in `requirements-omarchy.txt`. +1. Pin the version in `requirements.txt`, and Hyprland's own in `[install."linux/omarchy"]`'s download with the plugin + of the same release in `computers/disk/omarchy/recipe.sh`. 2. Regenerate `mcp.json` from what that version's MCP server, `cua-driver mcp`, says at connect and lists: its `instructions`, each tool's `name`, `description`, `inputSchema`, `annotations` and `outputSchema`, and each resource's `uri`, `name`, `description` and `mimeType`. That takes a running daemon, so do it on a Linux test machine, `mcp-hyprland.json` the same way on a Hyprland desktop at its own pin, `mcp-macos.json` in a macOS computer, and `mcp-windows.json` in a Windows computer's signed-in session, on the driver's venv, since `cua-driver mcp` refuses Session 0, where ssh lands. -3. Set `EXECUTABLES` in `driver.py` to the sha256 of `cua_driver/bin/cua-driver` in each arch's wheel, and in the wheel - of Hyprland's own pin for `hyprland/amd64`, and of CuaDriver.app's executable for `macos/arm64`, and the release +3. Set `EXECUTABLES` in `driver.py` to the sha256 of `cua_driver/bin/cua-driver` in each arch's wheel, of the Linux + release tarball's `cua-driver` at Hyprland's own pin for `hyprland/amd64`, with the tarball's URL and sha256 in + `[install."linux/omarchy"]`, and of CuaDriver.app's executable for `macos/arm64`, and the release tarball's URL and sha256 in `env.toml`'s `[install.macos]` and its folder in `install-app.sh`; for `windows/amd64`, the zip's `cua-driver.exe`, its URL and sha256 in `[install.windows]` and its folder in `install-windows.ps1`. 4. Run `tst/unit/drivers/cua_driver_test.py`, then `tst/integration/computer/cua_driver_test.py` on Linux, where diff --git a/src/agent_env/drivers/envs/cua-driver/driver.py b/src/agent_env/drivers/envs/cua-driver/driver.py index 9345fb0a2..137dad4e4 100644 --- a/src/agent_env/drivers/envs/cua-driver/driver.py +++ b/src/agent_env/drivers/envs/cua-driver/driver.py @@ -87,18 +87,20 @@ def _desktop() -> str | None: #: What cua-driver's own MCP server (``cua-driver mcp``) says at connect and lists at the pinned version on each OS: its #: instructions, its tools, which the running daemon must list the same, and its skill resources. MANIFEST = Path(__file__).with_name(f"mcp-{_desktop()}.json" if _desktop() else "mcp.json") -#: Where the release's zip is unpacked on Windows, beside this file. +#: Where the release's zip is unpacked on Windows, and its Linux tarball's executable on Hyprland, beside this file. WINDOWS_EXECUTABLE = Path(__file__).with_name("cua-driver") / "cua-driver.exe" +HYPRLAND_EXECUTABLE = Path(__file__).with_name("cua-driver") / "cua-driver" #: CuaDriver.app's executable, which macOS runs the daemon as. APP_EXECUTABLE = Path("/Applications/CuaDriver.app/Contents/MacOS/cua-driver") #: The sha256 of the cua-driver executable each arch's pinned wheel installs, on macOS CuaDriver.app's, on Windows the -#: release zip's, and on Hyprland its own pin's wheel's, 0.44, which delivers background input there. +#: release zip's, and on Hyprland the release tarball's of its own pin, 0.47.4, the release whose Hyprland plugin +#: Omarchy's base builds. EXECUTABLES = { "amd64": "bf57b2f3778a0a7daa9388e241f20ce948b32a7a2f4dd102e497f34e54137934", "arm64": "26e9eee8165e9ce7b614b5e3d68e5d963231385a82b65236406718b5ce1ff936", "macos/arm64": "b4d974a9c1c92a700cd7be7ede987aea4a1e6835e37039ce3d628b71bd5f0ca8", "windows/amd64": "ffa58d88779ee219304b0c307f4cfa970fe822f830c6ecf9254eda5c29e22bf2", - "hyprland/amd64": "f4259d5135d23d4d03968e074d144a10ece6305ca28ef48edfb3ef5c9e724228", + "hyprland/amd64": "8dd0f74928cff484343ed65e3ac4d1dacdccc3950b8665e1102ac463af0a967b", } #: How the daemon delivers the pointer's actions: in the foreground on macOS, where it doesn't by default. FOREGROUND = {"delivery_mode": "foreground"} if MACOS else {} @@ -474,12 +476,14 @@ def _text(result: dict) -> str: def _executable() -> Path: - """cua-driver's executable: CuaDriver.app's on macOS, else the one in the driver's venv, found without importing its - package, which loads its SDK.""" + """cua-driver's executable: CuaDriver.app's on macOS, the release's beside the driver on Windows and Hyprland, else + the one in the driver's venv, found without importing its package, which loads its SDK.""" if MACOS: return APP_EXECUTABLE if WINDOWS: return WINDOWS_EXECUTABLE + if HYPRLAND: + return HYPRLAND_EXECUTABLE return Path(importlib.util.find_spec("cua_driver").submodule_search_locations[0]) / "bin" / "cua-driver" diff --git a/src/agent_env/drivers/envs/cua-driver/env.toml b/src/agent_env/drivers/envs/cua-driver/env.toml index ce2a6d9f0..ff8e7d8aa 100644 --- a/src/agent_env/drivers/envs/cua-driver/env.toml +++ b/src/agent_env/drivers/envs/cua-driver/env.toml @@ -11,6 +11,10 @@ services = [ [install."linux/omarchy"] python = "requirements-omarchy.txt" +downloads = [ + { url = "https://github.com/trycua/cua/releases/download/cua-driver-rs-v0.47.4/cua-driver-rs-0.47.4-linux-x86_64-binary.tar.gz", sha256 = "472de19fe6e347b6db009b010d4447ff8ed6d9af8459844c157acfa6770779e8", to = "cua-driver.tar.gz", arch = "amd64" }, +] +run = [{ script = "install-hyprland.sh" }] services = [ { name = "daemon", command = "venv/bin/python driver.py" }, { name = "driver", command = "venv/bin/python -m agentenv_protocol.computer.serve driver.py:CuaDriver" }, diff --git a/src/agent_env/drivers/envs/cua-driver/install-hyprland.sh b/src/agent_env/drivers/envs/cua-driver/install-hyprland.sh new file mode 100644 index 000000000..faf204b4b --- /dev/null +++ b/src/agent_env/drivers/envs/cua-driver/install-hyprland.sh @@ -0,0 +1,7 @@ +#!/bin/sh +# cua-driver's executable on Hyprland, from the release tarball the install downloaded and checked, in the driver's +# cua-driver folder. +set -eu +mkdir -p cua-driver +tar -xzf cua-driver.tar.gz -C cua-driver cua-driver +rm -f cua-driver.tar.gz diff --git a/src/agent_env/drivers/envs/cua-driver/mcp-hyprland.json b/src/agent_env/drivers/envs/cua-driver/mcp-hyprland.json index e6a4d3232..3bb9d3356 100644 --- a/src/agent_env/drivers/envs/cua-driver/mcp-hyprland.json +++ b/src/agent_env/drivers/envs/cua-driver/mcp-hyprland.json @@ -1,5 +1,5 @@ { - "instructions": "cua-driver: background GUI automation. No shell: for non-GUI outcomes use an app API/SDK, CLI or filesystem and read the result back there.\n\n1. `list_windows({app:\"Name\"})` or `launch_app` for `pid` and `window_id`.\n2. Read narrowly: `get_window_state` with `query:\"Save\"`. Re-read with `since:\"latest\"`.\n3. Act by `element_token` (AT-SPI): row `[N]` is `:N`. Pixels only for surfaces missing from the tree.\n4. Act through `run_steps({steps:[{tool,args}],observe:{}})`, even for one step: stops at the first failure, returns what changed. Loops, branches or retries: one `run_script` call.\n5. Verify at checkpoints, not after every action: `verify_state(pid, window_id, expect)`. `unknown` and `effect:\"unverifiable\"` are not success.\n\nUse the narrowest semantic route first: `set_window_frame` plus `list_windows` for geometry, typed browser tools for pages. Stay in the background; `delivery_mode:\"foreground\"` only if refused or unverifiable.\n\nChrome/Edge/Brave page: `get_browser_state({pid, window_id})`, its `detail.next_call` (`browser_prepare`) if refused, then `browser_click`/`browser_type`/`browser_pointer` (hover, drag); never desktop pointer tools.\n\n`start_session` is optional; repeat a short `session` label on every call that accepts it. On continuation/recent-work, call `history_status`; if ready, make one bounded initial `history_query` before broad discovery.\n\nGuides: `get_guide({topic})` (drag, browser, permissions, batching, menus), `skill://cua-driver/SKILL.md`, LINUX.md (X11/Wayland status, AT-SPI bus, BETA-level support).", + "instructions": "cua-driver: background GUI automation. No shell: for non-GUI outcomes use an app API/SDK, CLI or filesystem and read the result back there.\n\n1. `list_windows({app:\"Name\"})` or `launch_app` for `pid` and `window_id`.\n2. Read narrowly: `get_window_state` with `query:\"Save\"`. Re-read with `since:\"latest\"`.\n3. Act by `element_token` (AT-SPI): row `[N]` is `:N`. Pixels only for surfaces missing from the tree.\n4. Act through `run_steps({steps:[{tool,args}],observe:{}})`, even for one step, batching actions: stops at the first failure, returns what changed. Loops, branches or retries: one `run_script` call.\n5. Verify at checkpoints, not after every action: `verify_state(pid, window_id, expect)`. `unknown` and `effect:\"unverifiable\"` are not success.\n\nUse the narrowest semantic route first: `set_window_frame` plus `list_windows` for geometry, typed browser tools for pages. Stay in the background; `delivery_mode:\"foreground\"` only if refused/unverifiable.\n\nChrome/Edge/Brave page: `get_browser_state({pid, window_id})`, its `detail.next_call` (`browser_prepare`) if refused, then `browser_click`/`browser_type`/`browser_pointer` (hover, drag); never desktop pointer tools.\n\n`start_session` is optional; repeat a short `session` label on every call that accepts it. On continuation/recent-work, call `history_status`; if ready, make one bounded initial `history_query` before broad discovery.\n\nGuides: `get_guide({topic})` (drag, browser, permissions, batching, menus), `skill://cua-driver/SKILL.md`, LINUX.md (X11/Wayland status, AT-SPI bus, BETA-level support).", "tools": [ { "name": "list_apps", @@ -1598,7 +1598,7 @@ }, { "name": "click", - "description": "Click against a target pid. **Prefer `element_token` over pixel coordinates** — element_token works on backgrounded / hidden windows, surfaces a stable handle, and tells you what you're clicking via the cached AT-SPI element's role + label. Reach for `x, y` only when the target is a canvas / custom-drawn surface that doesn't appear in the AT-SPI tree.\n\nProvide either (window_id + x/y) or (pid + element_token). x/y are WINDOW-LOCAL pixels of window_id (the get_window_state screenshot frame); a point outside the window is refused (point_outside_window) and every pixel result reports the resolved window_point / screen_point. Background delivery first resolves the accessible under the point and fires its action (the result says `hit: \"\" action=...`; a file / list item is selected through its container: `selected: ...`), else presses the real virtual pointer there. A click that opens a popup menu names it (`popup: {window_id, bounds}`): pass that window_id to get_window_state to index its items. The token is stale once the next get_window_state of the same window replaces its snapshot — re-snapshot every turn before clicking.\n\nAfter a zoom call, pass from_zoom=true to auto-translate zoom-image coords back to full-window space.\n\n`count`: 1 (default), 2 for a double-click, 3 for a triple-click (selects a line / paragraph in editors) — one press train with real double-click cadence, in background and foreground alike; there is no separate triple_click tool.\n\nbutton: \"left\" (default), \"right\", or \"middle\". Defaults to left so the field is fully back-compat. X11: routes through XSendEvent ButtonPress/Release with the matching button code. Native Wayland: only left-button is supported via the virtual-pointer protocol — right/middle return an error rather than silently degrading to left. `modifier` holds ctrl/shift/alt/super for the click on X11. Native Wayland refuses modified pointer clicks until its input protocol can carry keyboard modifier state.", + "description": "Click against a target pid. **Prefer `element_token` over pixel coordinates** — element_token works on backgrounded / hidden windows, surfaces a stable handle, and tells you what you're clicking via the cached AT-SPI element's role + label. Reach for `x, y` only when the target is a canvas / custom-drawn surface that doesn't appear in the AT-SPI tree.\n\nProvide either (window_id + x/y) or (pid + element_token). x/y are WINDOW-LOCAL pixels of window_id (the get_window_state screenshot frame); a point outside the window is refused (point_outside_window) and every pixel result reports the resolved window_point / screen_point. Background delivery first resolves the accessible under the point and fires its action (the result says `hit: \"\" action=...`; a file / list item is selected through its container: `selected: ...`), else presses the real virtual pointer there. A click that opens a popup menu names it (`popup: {window_id, bounds}`): pass that window_id to get_window_state to index its items. The token is stale once the next get_window_state of the same window replaces its snapshot — re-snapshot every turn before clicking.\n\nAfter a zoom call, pass from_zoom=true to auto-translate zoom-image coords back to full-window space.\n\n`count`: 1 (default), 2 for a double-click, 3 for a triple-click (selects a line / paragraph in editors) — one press train with real double-click cadence, in background and foreground alike; there is no separate triple_click tool.\n\nbutton: \"left\" (default), \"right\", or \"middle\". Defaults to left so the field is fully back-compat. X11: routes through XSendEvent ButtonPress/Release with the matching button code. Native Wayland: only left-button is supported via the virtual-pointer protocol — right/middle return an error rather than silently degrading to left. `modifier` holds ctrl/shift/alt/super for the click on X11. Native Wayland refuses modified pointer clicks until its input protocol can carry keyboard modifier state. Several known actions: one `run_steps` (`get_guide({topic:\"batching\"})`).", "inputSchema": { "additionalProperties": false, "properties": { @@ -3907,7 +3907,7 @@ }, { "name": "type_text", - "description": "Type text to a window via XSendEvent (KeyPress/KeyRelease). No focus steal.", + "description": "Type text to a window via XSendEvent (KeyPress/KeyRelease). No focus steal. Several known actions: one `run_steps` (`get_guide({topic:\"batching\"})`).", "inputSchema": { "additionalProperties": false, "properties": { @@ -4219,7 +4219,7 @@ }, { "name": "press_key", - "description": "Press a key via XSendEvent to a window. No focus steal.", + "description": "Press a key via XSendEvent to a window. No focus steal. Several known actions: one `run_steps` (`get_guide({topic:\"batching\"})`).", "inputSchema": { "additionalProperties": false, "properties": { @@ -9640,7 +9640,7 @@ }, { "name": "run_steps", - "description": "The default way to act: run one or more action tools in ONE call, in order, stop at the first failure, and (with `observe`) get what changed in the same response. Use it whenever you know the next action(s) and would otherwise re-read the window after them: fill three fields, click, and see the result in one call instead of five. Even a single step plus `observe:{}` replaces an action call followed by a get_window_state call. Each step is `{tool, args}` where `tool` is one of click, double_click, right_click, set_value, type_text, press_key, hotkey, scroll, drag, move_cursor and `args` are exactly that tool's arguments. `pid` and `window_id` (or `app`/`window`) on run_steps itself are the default window of every step that names none. Reads (get_window_state, zoom) do not run between steps: one there is skipped; a get_window_state last is the `observe`, a zoom last runs after it. Every step goes through the same session, permission and approval checks as a direct call; a batch grants nothing extra. All steps are validated before the first one runs, so a malformed step 4 changes nothing. Element tokens and element_index values come from a get_window_state read before the batch; a step that changes the UI can invalidate later element targets, so put element-targeted actions before the actions that reshuffle the window, or target later steps by role/name. A token that went stale is retried once on the element with the same role and label, if exactly one matches.\n\nReturns per-step status (`ok` or the error message) and, when `observe` is given, ONE bounded get_window_state read after the last executed step (after a failure caused by a dialog holding focus, that dialog is read, even without `observe`). A step refused because its app ignores background input says how to re-send it; `foreground_fallback:true` retries such steps in the foreground once (the window is activated). That read is a `since:\"latest\"` diff by default: only the rows that changed since your last read of the window, plus a new snapshot_id. Without `observe` nothing is read. The batch uses one session: steps may omit `session` or repeat the batch's. `delay_ms` pauses between steps (max 2000). At most 64 steps; `delivery_mode` on run_steps is the default for its steps.\n\nTARGET BY NAME (no earlier read needed, so one batch can cross screens): a step can be written `{\"click\": {...args}}` as well as `{tool, args}`, and its args may name the target instead of carrying a token: `role` (button, textfield, link, checkbox, menuitem, ...; AX/UIA/AT-SPI names also work), `name` (the element's label; exact match wins over substring, case-insensitive), `nth` (0-based, when several match), and `app` (app name or bundle id) and/or `window` (title substring) instead of pid/window_id. The batch reads the window fresh, waits up to 3 s (step `timeout_ms`, max 10000) for the element to appear, and acts on it; with only `app` it uses that app's frontmost window, so a dialog that opened on top is found. Later steps inherit the window, so name `app` once. The first lookup in a window also reads it once for `observe`, so the end diff shows what the batch changed even in a window you never read. Several matches fail with the candidates listed; add `nth` or a fuller name.\n\nWAITS AND CHECKS: a step may add `wait_for` (checked before its action) and `expect` (one check or up to 4, checked after it); a step may also be only a check. A check is {role?, name?, text?, gone?, value?, value_contains?, enabled?, selected?, app?, window?, timeout_ms?}: `text` matches any row of the tree, static text included; `gone:true` waits for it to disappear. Checks poll until they hold or time out (wait_for 5 s, expect 2 s, max 10 s), and a check that does not hold fails the step with what was found instead. The report says which step failed, in which phase (wait_for, find, action, expect) and why. Whole batch: at most 60 s.\n\nExample (fill a form, submit, confirm the dialog, check the result):\n{\"steps\":[{\"set_value\":{\"app\":\"Safari\",\"role\":\"textfield\",\"name\":\"Email\",\"value\":\"ada@example.com\"}},{\"click\":{\"role\":\"button\",\"name\":\"Submit\"},\"expect\":[{\"role\":\"button\",\"name\":\"Confirm\"}]},{\"click\":{\"role\":\"button\",\"name\":\"Confirm\"},\"expect\":[{\"name\":\"Confirm\",\"gone\":true},{\"text\":\"Thanks\"}]}],\"observe\":{}} Batching rules: `get_guide({topic:\"batching\"})`.", + "description": "The default way to act: run one or more action tools in ONE call, in order, stop at the first failure, and (with `observe`) get what changed in the same response. Use it whenever you know the next action(s) and would otherwise re-read the window after them: fill three fields, click, and see the result in one call instead of five. Even a single step plus `observe:{}` replaces an action call followed by a get_window_state call. Each step is `{tool, args}` where `tool` is one of click, double_click, right_click, set_value, type_text, press_key, hotkey, scroll, drag, move_cursor and `args` are exactly that tool's arguments. `pid` and `window_id` (or `app`/`window`) on run_steps itself are the default window of every step that names none. Reads (get_window_state, zoom) do not run between steps: one there is skipped; a get_window_state last is the `observe`, a zoom last runs after it. Steps pass the same checks as direct calls. Element tokens and element_index values come from a get_window_state read before the batch; a step that changes the UI can invalidate later element targets, so put element-targeted actions before the actions that reshuffle the window, or target later steps by role/name. A token that went stale is retried once on the element with the same role and label, if exactly one matches.\n\nReturns per-step status (`ok` or the error message) and, when `observe` is given, ONE bounded get_window_state read after the last executed step (after a failure caused by a dialog holding focus, that dialog is read, even without `observe`). A step refused because its app ignores background input says how to re-send it; `foreground_fallback:true` retries such steps in the foreground once (the window is activated). That read is a `since:\"latest\"` diff by default: only the rows that changed since your last read of the window, plus a new snapshot_id. Without `observe` nothing is read. `delay_ms` pauses between steps (max 2000). At most 64 steps; `delivery_mode` on run_steps is the default for its steps.\n\nTARGET BY NAME (no earlier read needed, so one batch can cross screens): a step can be written `{\"click\": {...args}}` as well as `{tool, args}`, and its args may name the target instead of carrying a token: `role` (button, textfield, link, checkbox, menuitem, ...; AX/UIA/AT-SPI names also work), `name` (the element's label; exact match wins over substring, case-insensitive), `nth` (0-based, when several match), and `app` (app name or bundle id) and/or `window` (title substring) instead of pid/window_id. The batch reads the window fresh, waits up to 3 s (step `timeout_ms`, max 10000) for the element to appear, and acts on it; with only `app` it uses that app's frontmost window, so a dialog that opened on top is found. Later steps inherit the window, so name `app` once. The first lookup in a window also reads it once for `observe`, so the end diff shows what the batch changed even in a window you never read. Several matches fail with the candidates listed; add `nth` or a fuller name.\n\nWAITS AND CHECKS: a step may add `wait_for` (checked before its action) and `expect` (one check or up to 4, checked after it); a step may also be only a check. A check is {role?, name?, text?, gone?, value?, value_contains?, enabled?, selected?, app?, window?, timeout_ms?}: `text` matches any row of the tree, static text included; `gone:true` waits for it to disappear. Checks poll until they hold or time out (wait_for 5 s, expect 2 s, max 10 s), and a check that does not hold fails the step with what was found instead. The report says which step failed, in which phase (wait_for, find, action, expect) and why. Whole batch: at most 60 s.\n\nExample (fill a form, submit, confirm the dialog, check the result):\n{\"steps\":[{\"set_value\":{\"app\":\"Safari\",\"role\":\"textfield\",\"name\":\"Email\",\"value\":\"ada@example.com\"}},{\"click\":{\"role\":\"button\",\"name\":\"Submit\"},\"expect\":[{\"role\":\"button\",\"name\":\"Confirm\"}]},{\"click\":{\"role\":\"button\",\"name\":\"Confirm\"},\"expect\":[{\"name\":\"Confirm\",\"gone\":true},{\"text\":\"Thanks\"}]}],\"observe\":{}} Batching rules: `get_guide({topic:\"batching\"})`.", "inputSchema": { "additionalProperties": false, "properties": { diff --git a/src/agent_env/drivers/envs/cua-driver/requirements-omarchy.txt b/src/agent_env/drivers/envs/cua-driver/requirements-omarchy.txt index 0a462625e..9ca23495d 100644 --- a/src/agent_env/drivers/envs/cua-driver/requirements-omarchy.txt +++ b/src/agent_env/drivers/envs/cua-driver/requirements-omarchy.txt @@ -1,7 +1,6 @@ -# What a composed Omarchy computer installs into the driver's venv: cua-driver at Hyprland's own pin, whose executable -# the driver checks (EXECUTABLES["hyprland/amd64"]), with Omarchy's base carrying the Hyprland plugin it drives input -# through; python-xlib, which driver.py imports on any Linux. A release pins agentenv-framework-protocol to the version -# it ships. +# What a composed Omarchy computer installs into the driver's venv: python-xlib, which driver.py imports on any Linux. +# cua-driver itself is its release's executable at Hyprland's own pin (env.toml's downloads), whose sha256 the driver +# checks (EXECUTABLES["hyprland/amd64"]), with Omarchy's base carrying the same release's Hyprland plugin it drives +# input through. A release pins agentenv-framework-protocol to the version it ships. agentenv-framework-protocol[computer]>=0.1.444 -cua-driver==0.44.0 python-xlib>=0.33,<0.34 diff --git a/tst/unit/drivers/cua_driver_test.py b/tst/unit/drivers/cua_driver_test.py index c1d3e59a7..cfce56314 100644 --- a/tst/unit/drivers/cua_driver_test.py +++ b/tst/unit/drivers/cua_driver_test.py @@ -32,6 +32,9 @@ from agentenv_protocol.computer.platform import supported DRIVER = Path(agent_env.__file__).parent / "drivers" / "envs" / "cua-driver" / "driver.py" +#: cua-driver's Linux release tarball, its version the group. +RELEASE_TARBALL = (r"https://github\.com/trycua/cua/releases/download/cua-driver-rs-v([\d.]+)/" + r"cua-driver-rs-\1-linux-x86_64-binary\.tar\.gz") def _png(width: int, height: int) -> str: @@ -758,6 +761,7 @@ def test_in_a_hyprland_session_the_driver_pins_hyprlands_manifest_and_tells_agen loaded = _loaded(monkeypatch) assert loaded.MANIFEST.name == "mcp-hyprland.json" and loaded.CuaDriver.notes == loaded.hyprland.NOTES + assert loaded._executable() == DRIVER.with_name("cua-driver") / "cua-driver" assert module.MANIFEST.name == "mcp.json" and module.CuaDriver.notes == "" @@ -769,7 +773,21 @@ def test_an_omarchy_computer_installs_cua_driver_at_hyprlands_pin_and_the_driver assert str(for_os(found, omarchy).python) == "requirements-omarchy.txt" assert str(for_os(found, ubuntu).python) == "requirements.txt" assert for_os(found, omarchy).services == for_os(found, ubuntu).services - pins = (DRIVER.parent / "requirements-omarchy.txt").read_text().splitlines() - assert any(re.fullmatch(r"cua-driver==\d+(\.\d+)+", line) for line in pins) + assert [(d.to, d.arch) for d in for_os(found, omarchy).downloads] == [("cua-driver.tar.gz", "amd64")] + assert re.fullmatch(RELEASE_TARBALL, for_os(found, omarchy).downloads[0].url) + assert [run.script for run in for_os(found, omarchy).run] == ["install-hyprland.sh"] + assert not any(line.startswith("cua-driver") for line in + (DRIVER.parent / "requirements-omarchy.txt").read_text().splitlines()) platforms = [platform for entry in module.CuaDriver.supports for platform in supported(entry)] assert Platform(os="linux", distribution="omarchy", arch="amd64", display="wayland") in platforms + + +def test_omarchys_base_builds_the_hyprland_plugin_of_the_cua_driver_release_its_computers_install(): + found, _ = installs(tomllib.loads((DRIVER.parent / "env.toml").read_text())["install"]) + pin = re.fullmatch(RELEASE_TARBALL, for_os(found, SimpleNamespace(type="linux", distribution="omarchy")) + .downloads[0].url)[1] + recipe = (Path(agent_env.__file__).parent / "computers" / "disk" / "omarchy" / "recipe.sh").read_text() + + release = re.search(r"^plugin_release=\S+/cua-driver-rs-v(\S+)$", recipe, re.MULTILINE)[1] + source = re.search(r"^plugin=cua-hyprland-plugin-(\S+)-[0-9a-f]{40}$", recipe, re.MULTILINE)[1] + assert release == source == pin