diff --git a/src/agent_env/computers/README.md b/src/agent_env/computers/README.md index f1a450fcb..b9fd9ff67 100644 --- a/src/agent_env/computers/README.md +++ b/src/agent_env/computers/README.md @@ -202,13 +202,19 @@ by `source`, a URL and a sha256, fetched once into this machine's cache and chec launcher as a `linux-desktop` guest (a virtio GPU at the computer's resolution, 4 vCPUs and 8 GB), with root's ssh key passed as a systemd credential, Omarchy signing its user `omarchy` in as it does at every boot, and made a base by `disk/omarchy/recipe.sh`, run as root over ssh. The recipe refuses an image whose os-release isn't the artifact's -`omarchy` `4.5.0`; installs cua-driver's Hyprland plugin, pinned by URL and sha256 to the package Omarchy builds for its -Hyprland; adds `agentenv.lua` to Hyprland's config (the plugin loaded at start, a US keymap, the idle cursor hidden, the -display's mode); installs the seed unit; has SDDM log back in after a logout; sets Omarchy's toggles for an unattended -desktop (staying awake, no animations, no screensaver or suspend, notifications silenced); turns off crash toasts, -dictation and its keys, housekeeping timers that would run at boot, and the firewall, which lets in only ssh and -LocalSend while the launcher forwards only the computer's own ports; and locks `omarchy`'s password and leaves the -machine id empty. Trimmed, the image becomes the base's, with the key to root, and is pushed to the env's image store +`omarchy` `4.5.0`; adds `agentenv.lua` to Hyprland's config (cua-driver's Hyprland plugin loaded at start, a US keymap, +the idle cursor hidden, the display's mode); installs the seed unit; has SDDM log back in after a logout; sets Omarchy's +toggles for an unattended desktop (staying awake, no animations, no screensaver or suspend, notifications silenced); +turns off crash toasts, dictation and its keys, housekeeping timers that would run at boot, and the firewall, which lets +in only ssh and LocalSend while the launcher forwards only the computer's own ports; builds the plugin; and locks +`omarchy`'s password and leaves the machine id empty. The plugin is the one cua-driver's release at the driver's +Hyprland pin ships, built from its source by that release's kit, unmodified, as `omarchy` with `makepkg`, each archive +pinned by sha256. The kit builds only on what it pins, Arch's Hyprland 0.56.2-1 and the GCC 16.1.1 it was built with, +where Omarchy 4.5 ships newer builds of both, so first the recipe installs those from that day of the Arch Linux +Archive (`disk/omarchy/arch-day.conf`, packages signature-checked by pacman), with the Hyprland libraries of that day, +and cmake and ninja, which it removes after the build. It ends the builder's desktop session first, as the kit says to +install it, and only then adds the drop-in: Hyprland reloads its config whenever a file of it changes, and the one +running there can neither load the plugin nor show, without hanging, the error a config naming a missing one makes. Trimmed, the image becomes the base's, with the key to root, and is pushed to the env's image store under a tag of its recipe and source. Name it as `omarchy-4.5`, or by its id, `@local/agentenv-framework/computers/omarchy-4.5`. diff --git a/src/agent_env/computers/disk/omarchy/arch-day.conf b/src/agent_env/computers/disk/omarchy/arch-day.conf new file mode 100644 index 000000000..ce620bbf5 --- /dev/null +++ b/src/agent_env/computers/disk/omarchy/arch-day.conf @@ -0,0 +1,15 @@ +# pacman's configuration for the recipe's one transaction from the Arch Linux Archive: the day of Hyprland 0.56.2-1 and +# GCC 16.1.1 20260728, which trycua's Hyprland plugin kit pins, as a day snapshot, its packages signature-checked by +# pacman. Omarchy's own pacman.conf stays as it is. +[options] +Architecture = auto +SigLevel = Required DatabaseOptional +LocalFileSigLevel = Optional +DownloadUser = alpm +ParallelDownloads = 5 + +[core] +Server = https://archive.archlinux.org/repos/2026/08/08/$repo/os/$arch + +[extra] +Server = https://archive.archlinux.org/repos/2026/08/08/$repo/os/$arch diff --git a/src/agent_env/computers/disk/omarchy/recipe.sh b/src/agent_env/computers/disk/omarchy/recipe.sh index 3fbea337a..ea5dd06c4 100755 --- a/src/agent_env/computers/disk/omarchy/recipe.sh +++ b/src/agent_env/computers/disk/omarchy/recipe.sh @@ -1,19 +1,27 @@ #!/bin/bash # recipe.sh USER DISTRIBUTION VERSION: run once as root over ssh in an Omarchy base's builder, Omarchy's own image booted # by the QEMU launcher with its session user USER signed in. It refuses an image that isn't the DISTRIBUTION and VERSION -# its artifact declares, then makes it a computer's base: cua-driver's Hyprland plugin, pinned to the package built for -# this Omarchy's Hyprland; agent-env's Hyprland drop-in; the seed unit that hands each boot its deploy's environment; -# SDDM logging back in after a logout; Omarchy's toggles for an unattended desktop; and off, what a trial doesn't want: -# crash toasts, dictation and its keys, housekeeping timers that would run at boot, and the firewall, which would stop -# the computer's own ports. The session user's password is locked, and the machine id left empty, as Omarchy's own -# sealed images leave it. +# its artifact declares, then makes it a computer's base: agent-env's Hyprland drop-in; the seed unit that hands each +# boot its deploy's environment; SDDM logging back in after a logout; Omarchy's toggles for an unattended desktop; off, +# what a trial doesn't want: crash toasts, dictation and its keys, housekeeping timers that would run at boot, and the +# firewall, which would stop the computer's own ports; and cua-driver's Hyprland plugin, built by its release's kit on +# what the kit pins. The session user's password is locked, and the machine id left empty, as Omarchy's own sealed +# images leave it. set -euo pipefail user=$1 distribution=$2 version=$3 here=$(cd "$(dirname "$0")" && pwd) home=$(getent passwd "$user" | cut -d: -f6) -#: cua-driver's Hyprland plugin, as Omarchy's own repository builds it for Omarchy 4.5's Hyprland 0.56.2-4. -plugin_url=https://pkgs.omarchy.org/stable/x86_64/cua-hyprland-plugin-0.33.2-3-x86_64.pkg.tar.zst -plugin_sha256=48a25c70dafdd423118c9937d3b469953e89a15cc0f278662f8fd728b523c47e +#: cua-driver's Hyprland plugin, as its release's source and build kit, which the driver's Hyprland pin matches. +plugin_release=https://github.com/trycua/cua/releases/download/cua-driver-rs-v0.47.4 +plugin=cua-hyprland-plugin-0.47.4-09b38ab661aa8b7d46deb5ff98ca00063019f592 +plugin_sha256=fd2e7f6426388753c19b8898a920f1f0d19f3ce81d5f7d7ae9ef3a2dee2c3b98 +plugin_kit_sha256=4da83523664049e9fa5a5f3269294dfad9602e6c5a02b0534a8c6f5b29920354 +#: What the kit builds on, in place of Omarchy's newer builds: Arch's Hyprland 0.56.2-1 with the libraries it was built +#: against, and the GCC 16.1.1 it was built with, from that day of the Arch Linux Archive (arch-day.conf). +pinned=(hyprland aquamarine hyprtoolkit hyprland-guiutils gcc gcc-libs libasan libatomic libgcc libgfortran libgomp + libhwasan liblsan libobjc libquadmath libstdc++ libtsan libubsan) +#: What the kit builds with that Omarchy lacks, removed once it has. +build_tools=(cmake ninja) . /etc/os-release if [ "$ID" != "$distribution" ] || [ "$VERSION_ID" != "$version" ]; then @@ -22,12 +30,6 @@ if [ "$ID" != "$distribution" ] || [ "$VERSION_ID" != "$version" ]; then fi echo "os-release: ID=$ID VERSION_ID=$VERSION_ID" -plugin=$(mktemp --suffix=.pkg.tar.zst) -curl -fsSL --retry 5 -o "$plugin" "$plugin_url" -echo "$plugin_sha256 $plugin" | sha256sum --check --quiet -pacman -U --noconfirm --needed "$plugin" -rm -f "$plugin" - install -d /opt/agentenv install -m 644 "$here/seed-env" /opt/agentenv/seed-env install -m 755 "$here/agentenv-seed" "$here/with-seed" /opt/agentenv/ @@ -37,10 +39,6 @@ systemctl enable --quiet agentenv-seed.service install -d /etc/sddm.conf.d install -m 644 "$here/sddm-relogin.conf" /etc/sddm.conf.d/zz-agentenv.conf -install -m 644 -o "$user" -g "$user" "$here/agentenv.lua" "$home/.config/hypr/agentenv.lua" -grep -q 'require("hypr.agentenv")' "$home/.config/hypr/hyprland.lua" \ - || echo 'require("hypr.agentenv")' >> "$home/.config/hypr/hyprland.lua" - systemctl --global mask --quiet omarchy-crash-watch.service superwhisper.service systemctl mask --quiet snapper-cleanup.timer plocate-updatedb.timer man-db.timer shadow.timer \ archlinux-keyring-wkd-sync.timer @@ -68,6 +66,36 @@ session omarchy toggle notification silencing session omarchy toggle screensaver session omarchy toggle suspend +# The rest with the desktop session ended, as the plugin's kit says to install it: Hyprland reloads its config whenever +# a file of it changes, and the Hyprland running here can neither load the plugin nor show the error its config naming a +# missing one makes without hanging. The computer boots the pinned Hyprland with the drop-in loading the plugin. +systemctl stop sddm.service +loginctl terminate-user "$user" +for _ in $(seq 60); do pgrep -x Hyprland > /dev/null || break; sleep 1; done +if pgrep -x Hyprland > /dev/null; then + echo "recipe: Hyprland still runs a minute after the desktop session ended" >&2 + exit 1 +fi +install -m 644 -o "$user" -g "$user" "$here/agentenv.lua" "$home/.config/hypr/agentenv.lua" +grep -q 'require("hypr.agentenv")' "$home/.config/hypr/hyprland.lua" \ + || echo 'require("hypr.agentenv")' >> "$home/.config/hypr/hyprland.lua" +# The plugin's kit runs makepkg as an ordinary user, in a folder of its own, and its package goes in from there. +pacman --config "$here/arch-day.conf" -Sy +pacman --config "$here/arch-day.conf" -S --needed --noconfirm "${pinned[@]}" "${build_tools[@]}" +kit=$(mktemp -d) download=$(mktemp -d) +curl -fsSL --retry 5 -o "$kit/$plugin.tar.gz" "$plugin_release/$plugin.tar.gz" +curl -fsSL --retry 5 -o "$download/kit.tar.gz" "$plugin_release/$plugin-build-kit.tar.gz" +echo "$plugin_sha256 $kit/$plugin.tar.gz" | sha256sum --check --quiet +echo "$plugin_kit_sha256 $download/kit.tar.gz" | sha256sum --check --quiet +tar -xzf "$download/kit.tar.gz" -C "$kit" +(cd "$kit" && sha256sum --check --quiet SHA256SUMS) +chown -R "$user:$user" "$kit" +runuser -u "$user" -- env HOME="$home" bash -c 'cd "$1" && makepkg --noconfirm' makepkg "$kit" +pacman -U --noconfirm "$kit"/cua-hyprland-plugin-*.pkg.tar.zst +pacman -Rns --noconfirm "${build_tools[@]}" +rm -rf "$kit" "$download" /var/lib/pacman/sync/core.db* /var/lib/pacman/sync/extra.db* +pacman -Q hyprland gcc cua-hyprland-plugin + passwd --lock --quiet "$user" 2>/dev/null || passwd -l "$user" > /dev/null truncate -s 0 /etc/machine-id echo "recipe: done" diff --git a/src/agent_env/drivers/README.md b/src/agent_env/drivers/README.md index cf8b845c0..95c1afac6 100644 --- a/src/agent_env/drivers/README.md +++ b/src/agent_env/drivers/README.md @@ -81,11 +81,14 @@ cua-driver's UI Automation tree, whose walk of it fails. On a Hyprland desktop (Wayland), as Omarchy's is, cua-driver runs its Wayland backend (`CUA_DRIVER_RS_ENABLE_WAYLAND=1`) and sends input through the seats of Hyprland's `cua-hyprland-plugin`. Hyprland's session turns this on (`HYPRLAND_INSTANCE_SIGNATURE`), not any Wayland one, since the driver reaches Hyprland's IPC; `supports` names -Omarchy's Wayland desktop on amd64, which installs `[install."linux/omarchy"]`: `requirements-omarchy.txt` and the two -services, with no Hyprland plugin, which Omarchy's base carries. It runs cua-driver 0.44.0, its own pin, since 0.38.0 -refuses background input there, the delivery cua-driver's own tools default to: `computer-ready` checks its wheel's -executable (`hyprland/amd64` in `EXECUTABLES`) and the tools `mcp-hyprland.json` pins, which `requirements-omarchy.txt` -installs, while `requirements.txt` keeps 0.38.0 for the other desktops. +Omarchy's Wayland desktop on amd64, which installs `[install."linux/omarchy"]`: `requirements-omarchy.txt`, cua-driver's +release tarball for Linux, whose executable `install-hyprland.sh` unpacks into the driver's `cua-driver` folder, and the +two services, with no Hyprland plugin: Omarchy's base carries the plugin of the same cua-driver release, built by its +kit. It runs cua-driver 0.47.4, its own pin, which takes background input from any native Wayland app through the +plugin's agent seats, where 0.38.0 refuses background input there, the delivery cua-driver's own tools default to; it +comes from the release, since 0.47.4's wheel isn't on PyPI, and moving the pin moves the plugin's release in +`computers/disk/omarchy/recipe.sh` with it. `computer-ready` checks the release's executable (`hyprland/amd64` in +`EXECUTABLES`) and the tools `mcp-hyprland.json` pins, while `requirements.txt` keeps 0.38.0 for the other desktops. The daemon starts once the compositor's socket answers, on a socket in the session user's runtime folder. What the driver asks Hyprland (`hyprctl -j`) is in `hyprland.py`, beside `driver.py`. Text goes to the desktop in the foreground, @@ -116,15 +119,17 @@ failure. The driver's services otherwise restart with the graphical session, as To move it to another cua-driver version: -1. Pin the version in `requirements.txt`, and Hyprland's own in `requirements-omarchy.txt`. +1. Pin the version in `requirements.txt`, and Hyprland's own in `[install."linux/omarchy"]`'s download with the plugin + of the same release in `computers/disk/omarchy/recipe.sh`. 2. Regenerate `mcp.json` from what that version's MCP server, `cua-driver mcp`, says at connect and lists: its `instructions`, each tool's `name`, `description`, `inputSchema`, `annotations` and `outputSchema`, and each resource's `uri`, `name`, `description` and `mimeType`. That takes a running daemon, so do it on a Linux test machine, `mcp-hyprland.json` the same way on a Hyprland desktop at its own pin, `mcp-macos.json` in a macOS computer, and `mcp-windows.json` in a Windows computer's signed-in session, on the driver's venv, since `cua-driver mcp` refuses Session 0, where ssh lands. -3. Set `EXECUTABLES` in `driver.py` to the sha256 of `cua_driver/bin/cua-driver` in each arch's wheel, and in the wheel - of Hyprland's own pin for `hyprland/amd64`, and of CuaDriver.app's executable for `macos/arm64`, and the release +3. Set `EXECUTABLES` in `driver.py` to the sha256 of `cua_driver/bin/cua-driver` in each arch's wheel, of the Linux + release tarball's `cua-driver` at Hyprland's own pin for `hyprland/amd64`, with the tarball's URL and sha256 in + `[install."linux/omarchy"]`, and of CuaDriver.app's executable for `macos/arm64`, and the release tarball's URL and sha256 in `env.toml`'s `[install.macos]` and its folder in `install-app.sh`; for `windows/amd64`, the zip's `cua-driver.exe`, its URL and sha256 in `[install.windows]` and its folder in `install-windows.ps1`. 4. Run `tst/unit/drivers/cua_driver_test.py`, then `tst/integration/computer/cua_driver_test.py` on Linux, where diff --git a/src/agent_env/drivers/envs/cua-driver/driver.py b/src/agent_env/drivers/envs/cua-driver/driver.py index 9345fb0a2..137dad4e4 100644 --- a/src/agent_env/drivers/envs/cua-driver/driver.py +++ b/src/agent_env/drivers/envs/cua-driver/driver.py @@ -87,18 +87,20 @@ def _desktop() -> str | None: #: What cua-driver's own MCP server (``cua-driver mcp``) says at connect and lists at the pinned version on each OS: its #: instructions, its tools, which the running daemon must list the same, and its skill resources. MANIFEST = Path(__file__).with_name(f"mcp-{_desktop()}.json" if _desktop() else "mcp.json") -#: Where the release's zip is unpacked on Windows, beside this file. +#: Where the release's zip is unpacked on Windows, and its Linux tarball's executable on Hyprland, beside this file. WINDOWS_EXECUTABLE = Path(__file__).with_name("cua-driver") / "cua-driver.exe" +HYPRLAND_EXECUTABLE = Path(__file__).with_name("cua-driver") / "cua-driver" #: CuaDriver.app's executable, which macOS runs the daemon as. APP_EXECUTABLE = Path("/Applications/CuaDriver.app/Contents/MacOS/cua-driver") #: The sha256 of the cua-driver executable each arch's pinned wheel installs, on macOS CuaDriver.app's, on Windows the -#: release zip's, and on Hyprland its own pin's wheel's, 0.44, which delivers background input there. +#: release zip's, and on Hyprland the release tarball's of its own pin, 0.47.4, the release whose Hyprland plugin +#: Omarchy's base builds. EXECUTABLES = { "amd64": "bf57b2f3778a0a7daa9388e241f20ce948b32a7a2f4dd102e497f34e54137934", "arm64": "26e9eee8165e9ce7b614b5e3d68e5d963231385a82b65236406718b5ce1ff936", "macos/arm64": "b4d974a9c1c92a700cd7be7ede987aea4a1e6835e37039ce3d628b71bd5f0ca8", "windows/amd64": "ffa58d88779ee219304b0c307f4cfa970fe822f830c6ecf9254eda5c29e22bf2", - "hyprland/amd64": "f4259d5135d23d4d03968e074d144a10ece6305ca28ef48edfb3ef5c9e724228", + "hyprland/amd64": "8dd0f74928cff484343ed65e3ac4d1dacdccc3950b8665e1102ac463af0a967b", } #: How the daemon delivers the pointer's actions: in the foreground on macOS, where it doesn't by default. FOREGROUND = {"delivery_mode": "foreground"} if MACOS else {} @@ -474,12 +476,14 @@ def _text(result: dict) -> str: def _executable() -> Path: - """cua-driver's executable: CuaDriver.app's on macOS, else the one in the driver's venv, found without importing its - package, which loads its SDK.""" + """cua-driver's executable: CuaDriver.app's on macOS, the release's beside the driver on Windows and Hyprland, else + the one in the driver's venv, found without importing its package, which loads its SDK.""" if MACOS: return APP_EXECUTABLE if WINDOWS: return WINDOWS_EXECUTABLE + if HYPRLAND: + return HYPRLAND_EXECUTABLE return Path(importlib.util.find_spec("cua_driver").submodule_search_locations[0]) / "bin" / "cua-driver" diff --git a/src/agent_env/drivers/envs/cua-driver/env.toml b/src/agent_env/drivers/envs/cua-driver/env.toml index ce2a6d9f0..ff8e7d8aa 100644 --- a/src/agent_env/drivers/envs/cua-driver/env.toml +++ b/src/agent_env/drivers/envs/cua-driver/env.toml @@ -11,6 +11,10 @@ services = [ [install."linux/omarchy"] python = "requirements-omarchy.txt" +downloads = [ + { url = "https://github.com/trycua/cua/releases/download/cua-driver-rs-v0.47.4/cua-driver-rs-0.47.4-linux-x86_64-binary.tar.gz", sha256 = "472de19fe6e347b6db009b010d4447ff8ed6d9af8459844c157acfa6770779e8", to = "cua-driver.tar.gz", arch = "amd64" }, +] +run = [{ script = "install-hyprland.sh" }] services = [ { name = "daemon", command = "venv/bin/python driver.py" }, { name = "driver", command = "venv/bin/python -m agentenv_protocol.computer.serve driver.py:CuaDriver" }, diff --git a/src/agent_env/drivers/envs/cua-driver/install-hyprland.sh b/src/agent_env/drivers/envs/cua-driver/install-hyprland.sh new file mode 100644 index 000000000..faf204b4b --- /dev/null +++ b/src/agent_env/drivers/envs/cua-driver/install-hyprland.sh @@ -0,0 +1,7 @@ +#!/bin/sh +# cua-driver's executable on Hyprland, from the release tarball the install downloaded and checked, in the driver's +# cua-driver folder. +set -eu +mkdir -p cua-driver +tar -xzf cua-driver.tar.gz -C cua-driver cua-driver +rm -f cua-driver.tar.gz diff --git a/src/agent_env/drivers/envs/cua-driver/mcp-hyprland.json b/src/agent_env/drivers/envs/cua-driver/mcp-hyprland.json index e6a4d3232..3bb9d3356 100644 --- a/src/agent_env/drivers/envs/cua-driver/mcp-hyprland.json +++ b/src/agent_env/drivers/envs/cua-driver/mcp-hyprland.json @@ -1,5 +1,5 @@ { - "instructions": "cua-driver: background GUI automation. No shell: for non-GUI outcomes use an app API/SDK, CLI or filesystem and read the result back there.\n\n1. `list_windows({app:\"Name\"})` or `launch_app` for `pid` and `window_id`.\n2. Read narrowly: `get_window_state` with `query:\"Save\"`. Re-read with `since:\"latest\"`.\n3. Act by `element_token` (AT-SPI): row `[N]` is `:N`. Pixels only for surfaces missing from the tree.\n4. Act through `run_steps({steps:[{tool,args}],observe:{}})`, even for one step: stops at the first failure, returns what changed. Loops, branches or retries: one `run_script` call.\n5. Verify at checkpoints, not after every action: `verify_state(pid, window_id, expect)`. `unknown` and `effect:\"unverifiable\"` are not success.\n\nUse the narrowest semantic route first: `set_window_frame` plus `list_windows` for geometry, typed browser tools for pages. Stay in the background; `delivery_mode:\"foreground\"` only if refused or unverifiable.\n\nChrome/Edge/Brave page: `get_browser_state({pid, window_id})`, its `detail.next_call` (`browser_prepare`) if refused, then `browser_click`/`browser_type`/`browser_pointer` (hover, drag); never desktop pointer tools.\n\n`start_session` is optional; repeat a short `session` label on every call that accepts it. On continuation/recent-work, call `history_status`; if ready, make one bounded initial `history_query` before broad discovery.\n\nGuides: `get_guide({topic})` (drag, browser, permissions, batching, menus), `skill://cua-driver/SKILL.md`, LINUX.md (X11/Wayland status, AT-SPI bus, BETA-level support).", + "instructions": "cua-driver: background GUI automation. No shell: for non-GUI outcomes use an app API/SDK, CLI or filesystem and read the result back there.\n\n1. `list_windows({app:\"Name\"})` or `launch_app` for `pid` and `window_id`.\n2. Read narrowly: `get_window_state` with `query:\"Save\"`. Re-read with `since:\"latest\"`.\n3. Act by `element_token` (AT-SPI): row `[N]` is `:N`. Pixels only for surfaces missing from the tree.\n4. Act through `run_steps({steps:[{tool,args}],observe:{}})`, even for one step, batching actions: stops at the first failure, returns what changed. Loops, branches or retries: one `run_script` call.\n5. Verify at checkpoints, not after every action: `verify_state(pid, window_id, expect)`. `unknown` and `effect:\"unverifiable\"` are not success.\n\nUse the narrowest semantic route first: `set_window_frame` plus `list_windows` for geometry, typed browser tools for pages. Stay in the background; `delivery_mode:\"foreground\"` only if refused/unverifiable.\n\nChrome/Edge/Brave page: `get_browser_state({pid, window_id})`, its `detail.next_call` (`browser_prepare`) if refused, then `browser_click`/`browser_type`/`browser_pointer` (hover, drag); never desktop pointer tools.\n\n`start_session` is optional; repeat a short `session` label on every call that accepts it. On continuation/recent-work, call `history_status`; if ready, make one bounded initial `history_query` before broad discovery.\n\nGuides: `get_guide({topic})` (drag, browser, permissions, batching, menus), `skill://cua-driver/SKILL.md`, LINUX.md (X11/Wayland status, AT-SPI bus, BETA-level support).", "tools": [ { "name": "list_apps", @@ -1598,7 +1598,7 @@ }, { "name": "click", - "description": "Click against a target pid. **Prefer `element_token` over pixel coordinates** — element_token works on backgrounded / hidden windows, surfaces a stable handle, and tells you what you're clicking via the cached AT-SPI element's role + label. Reach for `x, y` only when the target is a canvas / custom-drawn surface that doesn't appear in the AT-SPI tree.\n\nProvide either (window_id + x/y) or (pid + element_token). x/y are WINDOW-LOCAL pixels of window_id (the get_window_state screenshot frame); a point outside the window is refused (point_outside_window) and every pixel result reports the resolved window_point / screen_point. Background delivery first resolves the accessible under the point and fires its action (the result says `hit: \"\" action=...`; a file / list item is selected through its container: `selected: ...`), else presses the real virtual pointer there. A click that opens a popup menu names it (`popup: {window_id, bounds}`): pass that window_id to get_window_state to index its items. The token is stale once the next get_window_state of the same window replaces its snapshot — re-snapshot every turn before clicking.\n\nAfter a zoom call, pass from_zoom=true to auto-translate zoom-image coords back to full-window space.\n\n`count`: 1 (default), 2 for a double-click, 3 for a triple-click (selects a line / paragraph in editors) — one press train with real double-click cadence, in background and foreground alike; there is no separate triple_click tool.\n\nbutton: \"left\" (default), \"right\", or \"middle\". Defaults to left so the field is fully back-compat. X11: routes through XSendEvent ButtonPress/Release with the matching button code. Native Wayland: only left-button is supported via the virtual-pointer protocol — right/middle return an error rather than silently degrading to left. `modifier` holds ctrl/shift/alt/super for the click on X11. Native Wayland refuses modified pointer clicks until its input protocol can carry keyboard modifier state.", + "description": "Click against a target pid. **Prefer `element_token` over pixel coordinates** — element_token works on backgrounded / hidden windows, surfaces a stable handle, and tells you what you're clicking via the cached AT-SPI element's role + label. Reach for `x, y` only when the target is a canvas / custom-drawn surface that doesn't appear in the AT-SPI tree.\n\nProvide either (window_id + x/y) or (pid + element_token). x/y are WINDOW-LOCAL pixels of window_id (the get_window_state screenshot frame); a point outside the window is refused (point_outside_window) and every pixel result reports the resolved window_point / screen_point. Background delivery first resolves the accessible under the point and fires its action (the result says `hit: \"\" action=...`; a file / list item is selected through its container: `selected: ...`), else presses the real virtual pointer there. A click that opens a popup menu names it (`popup: {window_id, bounds}`): pass that window_id to get_window_state to index its items. The token is stale once the next get_window_state of the same window replaces its snapshot — re-snapshot every turn before clicking.\n\nAfter a zoom call, pass from_zoom=true to auto-translate zoom-image coords back to full-window space.\n\n`count`: 1 (default), 2 for a double-click, 3 for a triple-click (selects a line / paragraph in editors) — one press train with real double-click cadence, in background and foreground alike; there is no separate triple_click tool.\n\nbutton: \"left\" (default), \"right\", or \"middle\". Defaults to left so the field is fully back-compat. X11: routes through XSendEvent ButtonPress/Release with the matching button code. Native Wayland: only left-button is supported via the virtual-pointer protocol — right/middle return an error rather than silently degrading to left. `modifier` holds ctrl/shift/alt/super for the click on X11. Native Wayland refuses modified pointer clicks until its input protocol can carry keyboard modifier state. Several known actions: one `run_steps` (`get_guide({topic:\"batching\"})`).", "inputSchema": { "additionalProperties": false, "properties": { @@ -3907,7 +3907,7 @@ }, { "name": "type_text", - "description": "Type text to a window via XSendEvent (KeyPress/KeyRelease). No focus steal.", + "description": "Type text to a window via XSendEvent (KeyPress/KeyRelease). No focus steal. Several known actions: one `run_steps` (`get_guide({topic:\"batching\"})`).", "inputSchema": { "additionalProperties": false, "properties": { @@ -4219,7 +4219,7 @@ }, { "name": "press_key", - "description": "Press a key via XSendEvent to a window. No focus steal.", + "description": "Press a key via XSendEvent to a window. No focus steal. Several known actions: one `run_steps` (`get_guide({topic:\"batching\"})`).", "inputSchema": { "additionalProperties": false, "properties": { @@ -9640,7 +9640,7 @@ }, { "name": "run_steps", - "description": "The default way to act: run one or more action tools in ONE call, in order, stop at the first failure, and (with `observe`) get what changed in the same response. Use it whenever you know the next action(s) and would otherwise re-read the window after them: fill three fields, click, and see the result in one call instead of five. Even a single step plus `observe:{}` replaces an action call followed by a get_window_state call. Each step is `{tool, args}` where `tool` is one of click, double_click, right_click, set_value, type_text, press_key, hotkey, scroll, drag, move_cursor and `args` are exactly that tool's arguments. `pid` and `window_id` (or `app`/`window`) on run_steps itself are the default window of every step that names none. Reads (get_window_state, zoom) do not run between steps: one there is skipped; a get_window_state last is the `observe`, a zoom last runs after it. Every step goes through the same session, permission and approval checks as a direct call; a batch grants nothing extra. All steps are validated before the first one runs, so a malformed step 4 changes nothing. Element tokens and element_index values come from a get_window_state read before the batch; a step that changes the UI can invalidate later element targets, so put element-targeted actions before the actions that reshuffle the window, or target later steps by role/name. A token that went stale is retried once on the element with the same role and label, if exactly one matches.\n\nReturns per-step status (`ok` or the error message) and, when `observe` is given, ONE bounded get_window_state read after the last executed step (after a failure caused by a dialog holding focus, that dialog is read, even without `observe`). A step refused because its app ignores background input says how to re-send it; `foreground_fallback:true` retries such steps in the foreground once (the window is activated). That read is a `since:\"latest\"` diff by default: only the rows that changed since your last read of the window, plus a new snapshot_id. Without `observe` nothing is read. The batch uses one session: steps may omit `session` or repeat the batch's. `delay_ms` pauses between steps (max 2000). At most 64 steps; `delivery_mode` on run_steps is the default for its steps.\n\nTARGET BY NAME (no earlier read needed, so one batch can cross screens): a step can be written `{\"click\": {...args}}` as well as `{tool, args}`, and its args may name the target instead of carrying a token: `role` (button, textfield, link, checkbox, menuitem, ...; AX/UIA/AT-SPI names also work), `name` (the element's label; exact match wins over substring, case-insensitive), `nth` (0-based, when several match), and `app` (app name or bundle id) and/or `window` (title substring) instead of pid/window_id. The batch reads the window fresh, waits up to 3 s (step `timeout_ms`, max 10000) for the element to appear, and acts on it; with only `app` it uses that app's frontmost window, so a dialog that opened on top is found. Later steps inherit the window, so name `app` once. The first lookup in a window also reads it once for `observe`, so the end diff shows what the batch changed even in a window you never read. Several matches fail with the candidates listed; add `nth` or a fuller name.\n\nWAITS AND CHECKS: a step may add `wait_for` (checked before its action) and `expect` (one check or up to 4, checked after it); a step may also be only a check. A check is {role?, name?, text?, gone?, value?, value_contains?, enabled?, selected?, app?, window?, timeout_ms?}: `text` matches any row of the tree, static text included; `gone:true` waits for it to disappear. Checks poll until they hold or time out (wait_for 5 s, expect 2 s, max 10 s), and a check that does not hold fails the step with what was found instead. The report says which step failed, in which phase (wait_for, find, action, expect) and why. Whole batch: at most 60 s.\n\nExample (fill a form, submit, confirm the dialog, check the result):\n{\"steps\":[{\"set_value\":{\"app\":\"Safari\",\"role\":\"textfield\",\"name\":\"Email\",\"value\":\"ada@example.com\"}},{\"click\":{\"role\":\"button\",\"name\":\"Submit\"},\"expect\":[{\"role\":\"button\",\"name\":\"Confirm\"}]},{\"click\":{\"role\":\"button\",\"name\":\"Confirm\"},\"expect\":[{\"name\":\"Confirm\",\"gone\":true},{\"text\":\"Thanks\"}]}],\"observe\":{}} Batching rules: `get_guide({topic:\"batching\"})`.", + "description": "The default way to act: run one or more action tools in ONE call, in order, stop at the first failure, and (with `observe`) get what changed in the same response. Use it whenever you know the next action(s) and would otherwise re-read the window after them: fill three fields, click, and see the result in one call instead of five. Even a single step plus `observe:{}` replaces an action call followed by a get_window_state call. Each step is `{tool, args}` where `tool` is one of click, double_click, right_click, set_value, type_text, press_key, hotkey, scroll, drag, move_cursor and `args` are exactly that tool's arguments. `pid` and `window_id` (or `app`/`window`) on run_steps itself are the default window of every step that names none. Reads (get_window_state, zoom) do not run between steps: one there is skipped; a get_window_state last is the `observe`, a zoom last runs after it. Steps pass the same checks as direct calls. Element tokens and element_index values come from a get_window_state read before the batch; a step that changes the UI can invalidate later element targets, so put element-targeted actions before the actions that reshuffle the window, or target later steps by role/name. A token that went stale is retried once on the element with the same role and label, if exactly one matches.\n\nReturns per-step status (`ok` or the error message) and, when `observe` is given, ONE bounded get_window_state read after the last executed step (after a failure caused by a dialog holding focus, that dialog is read, even without `observe`). A step refused because its app ignores background input says how to re-send it; `foreground_fallback:true` retries such steps in the foreground once (the window is activated). That read is a `since:\"latest\"` diff by default: only the rows that changed since your last read of the window, plus a new snapshot_id. Without `observe` nothing is read. `delay_ms` pauses between steps (max 2000). At most 64 steps; `delivery_mode` on run_steps is the default for its steps.\n\nTARGET BY NAME (no earlier read needed, so one batch can cross screens): a step can be written `{\"click\": {...args}}` as well as `{tool, args}`, and its args may name the target instead of carrying a token: `role` (button, textfield, link, checkbox, menuitem, ...; AX/UIA/AT-SPI names also work), `name` (the element's label; exact match wins over substring, case-insensitive), `nth` (0-based, when several match), and `app` (app name or bundle id) and/or `window` (title substring) instead of pid/window_id. The batch reads the window fresh, waits up to 3 s (step `timeout_ms`, max 10000) for the element to appear, and acts on it; with only `app` it uses that app's frontmost window, so a dialog that opened on top is found. Later steps inherit the window, so name `app` once. The first lookup in a window also reads it once for `observe`, so the end diff shows what the batch changed even in a window you never read. Several matches fail with the candidates listed; add `nth` or a fuller name.\n\nWAITS AND CHECKS: a step may add `wait_for` (checked before its action) and `expect` (one check or up to 4, checked after it); a step may also be only a check. A check is {role?, name?, text?, gone?, value?, value_contains?, enabled?, selected?, app?, window?, timeout_ms?}: `text` matches any row of the tree, static text included; `gone:true` waits for it to disappear. Checks poll until they hold or time out (wait_for 5 s, expect 2 s, max 10 s), and a check that does not hold fails the step with what was found instead. The report says which step failed, in which phase (wait_for, find, action, expect) and why. Whole batch: at most 60 s.\n\nExample (fill a form, submit, confirm the dialog, check the result):\n{\"steps\":[{\"set_value\":{\"app\":\"Safari\",\"role\":\"textfield\",\"name\":\"Email\",\"value\":\"ada@example.com\"}},{\"click\":{\"role\":\"button\",\"name\":\"Submit\"},\"expect\":[{\"role\":\"button\",\"name\":\"Confirm\"}]},{\"click\":{\"role\":\"button\",\"name\":\"Confirm\"},\"expect\":[{\"name\":\"Confirm\",\"gone\":true},{\"text\":\"Thanks\"}]}],\"observe\":{}} Batching rules: `get_guide({topic:\"batching\"})`.", "inputSchema": { "additionalProperties": false, "properties": { diff --git a/src/agent_env/drivers/envs/cua-driver/requirements-omarchy.txt b/src/agent_env/drivers/envs/cua-driver/requirements-omarchy.txt index 0a462625e..9ca23495d 100644 --- a/src/agent_env/drivers/envs/cua-driver/requirements-omarchy.txt +++ b/src/agent_env/drivers/envs/cua-driver/requirements-omarchy.txt @@ -1,7 +1,6 @@ -# What a composed Omarchy computer installs into the driver's venv: cua-driver at Hyprland's own pin, whose executable -# the driver checks (EXECUTABLES["hyprland/amd64"]), with Omarchy's base carrying the Hyprland plugin it drives input -# through; python-xlib, which driver.py imports on any Linux. A release pins agentenv-framework-protocol to the version -# it ships. +# What a composed Omarchy computer installs into the driver's venv: python-xlib, which driver.py imports on any Linux. +# cua-driver itself is its release's executable at Hyprland's own pin (env.toml's downloads), whose sha256 the driver +# checks (EXECUTABLES["hyprland/amd64"]), with Omarchy's base carrying the same release's Hyprland plugin it drives +# input through. A release pins agentenv-framework-protocol to the version it ships. agentenv-framework-protocol[computer]>=0.1.444 -cua-driver==0.44.0 python-xlib>=0.33,<0.34 diff --git a/tst/unit/drivers/cua_driver_test.py b/tst/unit/drivers/cua_driver_test.py index c1d3e59a7..cfce56314 100644 --- a/tst/unit/drivers/cua_driver_test.py +++ b/tst/unit/drivers/cua_driver_test.py @@ -32,6 +32,9 @@ from agentenv_protocol.computer.platform import supported DRIVER = Path(agent_env.__file__).parent / "drivers" / "envs" / "cua-driver" / "driver.py" +#: cua-driver's Linux release tarball, its version the group. +RELEASE_TARBALL = (r"https://github\.com/trycua/cua/releases/download/cua-driver-rs-v([\d.]+)/" + r"cua-driver-rs-\1-linux-x86_64-binary\.tar\.gz") def _png(width: int, height: int) -> str: @@ -758,6 +761,7 @@ def test_in_a_hyprland_session_the_driver_pins_hyprlands_manifest_and_tells_agen loaded = _loaded(monkeypatch) assert loaded.MANIFEST.name == "mcp-hyprland.json" and loaded.CuaDriver.notes == loaded.hyprland.NOTES + assert loaded._executable() == DRIVER.with_name("cua-driver") / "cua-driver" assert module.MANIFEST.name == "mcp.json" and module.CuaDriver.notes == "" @@ -769,7 +773,21 @@ def test_an_omarchy_computer_installs_cua_driver_at_hyprlands_pin_and_the_driver assert str(for_os(found, omarchy).python) == "requirements-omarchy.txt" assert str(for_os(found, ubuntu).python) == "requirements.txt" assert for_os(found, omarchy).services == for_os(found, ubuntu).services - pins = (DRIVER.parent / "requirements-omarchy.txt").read_text().splitlines() - assert any(re.fullmatch(r"cua-driver==\d+(\.\d+)+", line) for line in pins) + assert [(d.to, d.arch) for d in for_os(found, omarchy).downloads] == [("cua-driver.tar.gz", "amd64")] + assert re.fullmatch(RELEASE_TARBALL, for_os(found, omarchy).downloads[0].url) + assert [run.script for run in for_os(found, omarchy).run] == ["install-hyprland.sh"] + assert not any(line.startswith("cua-driver") for line in + (DRIVER.parent / "requirements-omarchy.txt").read_text().splitlines()) platforms = [platform for entry in module.CuaDriver.supports for platform in supported(entry)] assert Platform(os="linux", distribution="omarchy", arch="amd64", display="wayland") in platforms + + +def test_omarchys_base_builds_the_hyprland_plugin_of_the_cua_driver_release_its_computers_install(): + found, _ = installs(tomllib.loads((DRIVER.parent / "env.toml").read_text())["install"]) + pin = re.fullmatch(RELEASE_TARBALL, for_os(found, SimpleNamespace(type="linux", distribution="omarchy")) + .downloads[0].url)[1] + recipe = (Path(agent_env.__file__).parent / "computers" / "disk" / "omarchy" / "recipe.sh").read_text() + + release = re.search(r"^plugin_release=\S+/cua-driver-rs-v(\S+)$", recipe, re.MULTILINE)[1] + source = re.search(r"^plugin=cua-hyprland-plugin-(\S+)-[0-9a-f]{40}$", recipe, re.MULTILINE)[1] + assert release == source == pin