diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index b1cec4c44..4d1c61f64 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -692,15 +692,21 @@ driver booted with a control token is probed with it: `--token`, by default | Level | Checks | Needs on the computer | |---|---|---| | `basic` | P0 ready and a screenshot; P1 the card and tool schemas; P2 native frames, regions and JPEG; P5 refusals and their codes; X the declared extensions (`computer-display`, `computer-spaces`, `computer-keyboard`, `computer-exec`, `data/get`, per-role tool access, the pointer) | nothing | -| `full` | `basic`, then P3 colours land where the mapping says; P4 clicks show their effect; P5 refusals change nothing; P6 typed and pressed keys arrive; P7 scrolls and drags arrive; P9 commands and `data/add` run as one unprivileged user; X pointer and keyboard input, `data/add`, and a recording's segments matching their files with no gap, a repeated `request_id` joining it and `max_concurrent` held; on a page, X the page tools: a prompt answered, a file uploaded, a reload, navigation, back and forward, and tabs | Linux, `python3` with Tk (on a page, `python3` and `computer-page/v1`), a 1280x800 screen or larger, `computer-exec` and `data/add` | +| `full` | `basic`, then P3 colours land where the mapping says; P4 clicks show their effect; P5 refusals change nothing; P6 typed and pressed keys arrive; P7 scrolls and drags arrive; P9 commands and `data/add` run as one unprivileged user; X pointer and keyboard input, `data/add`, and a recording's segments matching their files with no gap, a repeated `request_id` joining it and `max_concurrent` held; on a page, X the page tools: a prompt answered, a file uploaded, a reload, navigation, back and forward, and tabs | Linux, `python3` with Tk (on Wayland, with PyGObject and GTK 3 instead; on a page, `python3` and `computer-page/v1`), a 1280x800 screen or larger, `computer-exec` and `data/add` | -At `full` the probe writes the Fixture Desk, a Tk window (`conformance/fixture.py`), into -`~/.agentenv-probe` with `data/add` and starts it with `computer-exec`. It aims at the boxes the +At `full` the probe writes the Fixture Desk, a Tk window (`conformance/fixture.py`), and its layout +(`conformance/desk.py`) into `~/.agentenv-probe` with `data/add` and starts it with `computer-exec`. It aims at the boxes the window's widgets report, reads back what reached them, and then removes the fixture. On X11 the window is maximized and kept above other windows, raising itself again when one covers it, so a window an app keeps on top doesn't take the clicks aimed at it; elsewhere it is full screen. When the fixture can't be staged, the `full` checks are unverified rather than failed. +On a driver whose platform's display is Wayland, the desk is drawn with GTK 3 as a native Wayland client +(`conformance/wayland_fixture.py`, staged beside `desk.py` in the Tk window's place), since Tk runs +only under XWayland. It goes full screen, takes its origin from Hyprland's IPC, keeps the clipboard +through wl-clipboard. There the probe selects a text box's text with GTK's own Ctrl+A rather than +Tk's Ctrl+/. + On a driver whose card says `screen: "page"`, the desk is an HTML page with the same widgets at the same places (`conformance/page_fixture.py`). The probe starts its server on loopback, which writes what the page reports to the same files, and opens the desk with `computer_tabs` in a tab of its diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py index e1b2105c2..c55db6c60 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/checks.py @@ -116,8 +116,15 @@ def __init__(self, driver: Driver, session: ClientSession, ready_timeout_s: floa self.layout: dict = {} self.fixture_pid: str | None = None self.page = self.features.get("screen") == "page" - self.fixture = "page_fixture.py" if self.page else "fixture.py" - self.windows = (self.features.get("platform") or {}).get("os") == "windows" + platform = self.features.get("platform") or {} + #: Whether the desk is the GTK one: on Wayland, where Tk runs only under XWayland, which a Wayland driver may + #: not reach. + self.gtk = not self.page and platform.get("display") == "wayland" + self.fixture = "page_fixture.py" if self.page else "wayland_fixture.py" if self.gtk else "fixture.py" + self.windows = platform.get("os") == "windows" + #: The chord that selects a text box's text: Tk's Ctrl+/, which the Tk desk binds everywhere, or on the GTK + #: desk GTK's own Ctrl+A. + self.select_all = ["Control", "a"] if self.gtk else ["Control", "/"] self.desk = "" self.tab: str | None = None self.opened: list[str] = [] @@ -562,12 +569,13 @@ def _grants(self) -> None: # full: the fixture async def _stage(self) -> str | None: - """The Fixture Desk started, or what stopped it: the Tk desk, or on a page, the HTML desk's server, with the - desk opened in a tab of the probe's own.""" - source = resources.files(__package__).joinpath(self.fixture).read_bytes() - added = await self.driver.add_file(f"{STAGE}/{self.fixture}", source) - if "error" in added: - return f"data/add refused it: {added['error'].get('message')}" + """The Fixture Desk started, or what stopped it: the Tk desk or on Wayland the GTK desk, either beside the layout + they share (desk.py), or on a page, the HTML desk's server, with the desk opened in a tab of the probe's own.""" + for file in [self.fixture] if self.page else [self.fixture, "desk.py"]: + source = resources.files(__package__).joinpath(file).read_bytes() + added = await self.driver.add_file(f"{STAGE}/{file}", source) + if "error" in added: + return f"data/add refused it: {added['error'].get('message')}" if self.windows: await self.driver.run(*WIN_REMOVE, f"{STAGE}/state") started = await self.driver.run(*WIN_START, f"{STAGE}/{self.fixture}", f"{STAGE}/state", f"{STAGE}/fixture.log") @@ -781,7 +789,7 @@ async def focus(box: str) -> None: async def clear() -> None: await focus("name") - await self.act("computer_key", {"keys": ["Control", "/"]}) + await self.act("computer_key", {"keys": self.select_all}) await self.act("computer_key", {"keys": ["Backspace"]}) async def unicode() -> str | None: @@ -796,7 +804,7 @@ async def events() -> str | None: async def chord() -> str | None: await focus("name") - await self.act("computer_key", {"keys": ["Control", "/"]}) + await self.act("computer_key", {"keys": self.select_all}) await self.act("computer_key", {"keys": ["Backspace"]}) return await self.settled(lambda s: s["name"] == "", lambda s: s["name"]) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/desk.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/desk.py new file mode 100644 index 000000000..addcc82ce --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/desk.py @@ -0,0 +1,65 @@ +"""The Fixture Desk's layout and what it records, shared by the Tk desk (fixture.py) and the GTK desk +(wayland_fixture.py), so the two can't drift apart. The probe stages it beside whichever desk it starts. It imports no +toolkit, since a Wayland computer may have no Tk.""" +from __future__ import annotations + +import json +import os +from pathlib import Path + +#: The colours a grid cell cycles through, one more on each click. +COLOURS = ("#ffffff", "#ff0000", "#00aa00", "#0000ff") +CELL = 50 +GRID = 8 +#: Single-pixel markers on a black field, at odd and even pixels: (x, y) within the field, then the colour. +MARKERS = ((11, 11, "#ff0000"), (12, 11, "#00ff00"), (11, 12, "#0000ff"), (12, 12, "#ffff00"), (37, 20, "#ff00ff"), + (20, 37, "#00ffff")) +CLIPBOARD = "agentenv-probe clipboard" +MIN_SIZE = (1280, 800) +#: Where each widget's top left goes, the name box's label at "label" and the list's frame at "list". +AT = {"label": (40, 40), "name": (140, 40), "save": (560, 34), "subscribe": (40, 100), "notes": (760, 40), + "list": (40, 160), "grid": (400, 160), "pad": (840, 160), "wheel": (840, 500), "field": (1080, 500)} +#: The widgets sized in pixels: the list's frame, the drawing pad, the wheel and the marker field. +SIZE = {"list": {"width": 300, "height": 400}, "pad": {"width": 400, "height": 300}, + "wheel": {"width": 200, "height": 120}, "field": {"width": 48, "height": 48}} +#: The list's rows, Item 1 to Item ITEMS. +ITEMS = 100 +#: How many of the latest keys and drags state.json keeps, and about how many points of a drag's path. +KEYS_KEPT, DRAGS_KEPT, PATH_POINTS = 100, 20, 50 +#: How often the desk reads inbox.txt. +INBOX_EVERY_MS = 200 + + +def new_state(width: int, height: int, windowing: str) -> dict: + """What state.json holds before anything reaches the desk.""" + return {"name": "", "notes": "", "keys": [], "saved_count": 0, "subscribed": False, "selected": None, + "scroll_top": 0.0, "cells": {}, "right_clicks": 0, "double_clicks": 0, "drags": [], + "wheel": {"up": 0, "down": 0, "left": 0, "right": 0}, "clipboard": None, "loaded": None, + "screen": [width, height], "windowing": windowing} + + +def write_state(state_dir: Path, state: dict) -> None: + """state.json replaced whole, so the probe never reads half of one.""" + part = state_dir / ".state.json.part" + part.write_text(json.dumps(state)) + os.replace(part, state_dir / "state.json") + + +def write_layout(state_dir: Path, boxes: dict, row_height: int, field: tuple[int, int]) -> None: + """layout.json: each widget's box on the screen, a list row's height, the markers' pixels from the field's top left, + and the cell size and clipboard text the probe checks against.""" + layout = {**boxes, "list_row_height": row_height, + "markers": [[field[0] + x, field[1] + y, colour] for x, y, colour in MARKERS], "cell": CELL, + "clipboard": CLIPBOARD} + (state_dir / "layout.json").write_text(json.dumps(layout)) + + +def record_key(state: dict, key: str) -> None: + state["keys"] = (state["keys"] + [key])[-KEYS_KEPT:] + + +def record_drag(state: dict, stroke: list) -> None: + """A finished drag: its ends, its number of points and its path thinned to about PATH_POINTS points.""" + path = stroke[::max(1, len(stroke) // PATH_POINTS)] + stroke[-1:] + drag = {"start": stroke[0], "end": stroke[-1], "points": len(stroke), "path": path} + state["drags"] = (state["drags"] + [drag])[-DRAGS_KEPT:] diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/fixture.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/fixture.py index e49769404..cdbb71769 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/fixture.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/fixture.py @@ -3,26 +3,19 @@ whatever lands in /inbox.txt, to /state.json, so the probe can aim at it and read back what arrived. It records the same effects under X11, macOS's Aqua and Windows, whose Tk names keys, binds select-all and reports the wheel differently. On X11 it stays above an app's windows (``keep_on_top``); elsewhere it's full screen. -Run as ``python3 fixture.py ``; it needs a 1280x800 screen or larger.""" +Its layout and what it records are desk.py's, which the GTK desk shares. Run as ``python3 fixture.py `` +beside desk.py; it needs a 1280x800 screen or larger.""" from __future__ import annotations import ctypes -import json -import os import sys import time import tkinter as tk from pathlib import Path -#: The colours a grid cell cycles through, one more on each click. -COLOURS = ("#ffffff", "#ff0000", "#00aa00", "#0000ff") -CELL = 50 -GRID = 8 -#: Single-pixel markers on a black field, at odd and even pixels: (x, y) within the field, then the colour. -MARKERS = ((11, 11, "#ff0000"), (12, 11, "#00ff00"), (11, 12, "#0000ff"), (12, 12, "#ffff00"), (37, 20, "#ff00ff"), - (20, 37, "#00ffff")) -CLIPBOARD = "agentenv-probe clipboard" -MIN_SIZE = (1280, 800) +from desk import (AT, CELL, CLIPBOARD, COLOURS, GRID, INBOX_EVERY_MS, ITEMS, MARKERS, MIN_SIZE, SIZE, new_state, + record_drag, record_key, write_layout, write_state) + #: X11's keysyms for the keys Aqua's Tk names by their character. KEYSYMS = {",": "comma", " ": "space"} #: A wheel notch's delta where Tk scales the wheel: Windows', and Tk 9's everywhere. @@ -46,15 +39,10 @@ def main(state_dir: Path) -> None: else: root.attributes("-fullscreen", True) root.configure(bg="#f0f0f0") - state: dict = {"name": "", "notes": "", "keys": [], "saved_count": 0, "subscribed": False, "selected": None, - "scroll_top": 0.0, "cells": {}, "right_clicks": 0, "double_clicks": 0, "drags": [], - "wheel": {"up": 0, "down": 0, "left": 0, "right": 0}, "clipboard": None, "loaded": None, - "screen": [width, height], "windowing": root.tk.call("tk", "windowingsystem")} + state = new_state(width, height, root.tk.call("tk", "windowingsystem")) def save() -> None: - part = state_dir / ".state.json.part" - part.write_text(json.dumps(state)) - os.replace(part, state_dir / "state.json") + write_state(state_dir, state) if width < MIN_SIZE[0] or height < MIN_SIZE[1]: state["error"] = f"the screen is {width}x{height}; the fixture needs {MIN_SIZE[0]}x{MIN_SIZE[1]}" @@ -63,13 +51,13 @@ def save() -> None: return font = ("DejaVu Sans", 16) - tk.Label(root, text="Name:", font=font, bg="#f0f0f0").place(x=40, y=40) + _place(tk.Label(root, text="Name:", font=font, bg="#f0f0f0"), "label") name = tk.Entry(root, font=font, width=30) - name.place(x=140, y=40) + _place(name, "name") def key(event) -> None: state["name"] = name.get() - state["keys"] = (state["keys"] + [KEYSYMS.get(event.keysym, event.keysym)])[-100:] + record_key(state, KEYSYMS.get(event.keysym, event.keysym)) state["clipboard"] = _clipboard(root) save() @@ -88,7 +76,7 @@ def on_save() -> None: save() button = tk.Button(root, text="Save", font=font, command=on_save, width=8) - button.place(x=560, y=34) + _place(button, "save") subscribed = tk.BooleanVar(value=False) @@ -98,10 +86,10 @@ def on_subscribe() -> None: subscribe = tk.Checkbutton(root, text="Subscribe", font=font, variable=subscribed, command=on_subscribe, bg="#f0f0f0") - subscribe.place(x=40, y=100) + _place(subscribe, "subscribe") notes = tk.Text(root, font=("DejaVu Sans", 14), width=28, height=3) - notes.place(x=760, y=40) + _place(notes, "notes") def note(event) -> None: state["notes"] = notes.get("1.0", "end-1c") @@ -110,7 +98,7 @@ def note(event) -> None: notes.bind("", note) frame = tk.Frame(root) - frame.place(x=40, y=160, width=300, height=400) + _place(frame, "list", **SIZE["list"]) listbox = tk.Listbox(frame, font=("DejaVu Sans", 14), exportselection=False) listbox.pack(side="left", fill="both", expand=True) bar = tk.Scrollbar(frame, command=listbox.yview) @@ -122,7 +110,7 @@ def scrolled(first, last) -> None: save() listbox.configure(yscrollcommand=scrolled) - for i in range(1, 101): + for i in range(1, ITEMS + 1): listbox.insert("end", f"Item {i}") def select(event) -> None: @@ -133,7 +121,7 @@ def select(event) -> None: listbox.bind("<>", select) grid = tk.Canvas(root, width=GRID * CELL, height=GRID * CELL, bg="#ffffff", highlightthickness=0) - grid.place(x=400, y=160) + _place(grid, "grid") for row in range(GRID): for col in range(GRID): grid.create_rectangle(col * CELL, row * CELL, (col + 1) * CELL, (row + 1) * CELL, fill=COLOURS[0], @@ -155,8 +143,8 @@ def double(event) -> None: grid.bind("", lambda e: cell(e, True)) grid.bind("", double) - pad = tk.Canvas(root, width=400, height=300, bg="#fffbe6", highlightthickness=0) - pad.place(x=840, y=160) + pad = tk.Canvas(root, **SIZE["pad"], bg="#fffbe6", highlightthickness=0) + _place(pad, "pad") stroke: list = [] def press(event) -> None: @@ -170,8 +158,7 @@ def motion(event) -> None: def release(event) -> None: if stroke: stroke.append((event.x, event.y)) - state["drags"] = (state["drags"] + [{"start": stroke[0], "end": stroke[-1], "points": len(stroke), - "path": stroke[::max(1, len(stroke) // 50)] + stroke[-1:]}])[-20:] + record_drag(state, stroke) save() stroke.clear() @@ -179,8 +166,8 @@ def release(event) -> None: pad.bind("", motion) pad.bind("", release) - wheel = tk.Canvas(root, width=200, height=120, bg="#e6f0ff", highlightthickness=0) - wheel.place(x=840, y=500) + wheel = tk.Canvas(root, **SIZE["wheel"], bg="#e6f0ff", highlightthickness=0) + _place(wheel, "wheel") def wheeled(horizontal: bool, steps: int) -> None: way = ("left" if horizontal else "up") if steps > 0 else ("right" if horizontal else "down") @@ -202,8 +189,8 @@ def scrolled(delta: int) -> None: if tk.TkVersion >= 8.7: wheel.bind("", lambda e: scrolled(e.delta)) - field = tk.Canvas(root, width=48, height=48, bg="#000000", highlightthickness=0, bd=0) - field.place(x=1080, y=500) + field = tk.Canvas(root, **SIZE["field"], bg="#000000", highlightthickness=0, bd=0) + _place(field, "field") for x, y, colour in MARKERS: field.create_rectangle(x, y, x + 1, y + 1, fill=colour, outline="", width=0) @@ -212,28 +199,29 @@ def poll_inbox() -> None: if message.exists() and (text := message.read_text()) != state["loaded"]: state["loaded"] = text save() - root.after(200, poll_inbox) + root.after(INBOX_EVERY_MS, poll_inbox) - def write_layout() -> None: + def layout() -> None: root.update_idletasks() boxes = {key: [w.winfo_rootx(), w.winfo_rooty(), w.winfo_width(), w.winfo_height()] for key, w in (("name", name), ("save", button), ("subscribe", subscribe), ("notes", notes), ("list", listbox), ("grid", grid), ("pad", pad), ("wheel", wheel), ("field", field))} - boxes["list_row_height"] = listbox.bbox(0)[3] if listbox.bbox(0) else 0 - boxes["markers"] = [[field.winfo_rootx() + x, field.winfo_rooty() + y, colour] for x, y, colour in MARKERS] - boxes["cell"] = CELL - boxes["clipboard"] = CLIPBOARD save() - (state_dir / "layout.json").write_text(json.dumps(boxes)) + write_layout(state_dir, boxes, listbox.bbox(0)[3] if listbox.bbox(0) else 0, + (field.winfo_rootx(), field.winfo_rooty())) root.clipboard_clear() root.clipboard_append(CLIPBOARD) state["clipboard"] = CLIPBOARD - root.after(300, write_layout) + root.after(300, layout) root.after(500, poll_inbox) root.mainloop() +def _place(widget: tk.Widget, at: str, **size: int) -> None: + widget.place(x=AT[at][0], y=AT[at][1], **size) + + def keep_on_top(root: tk.Tk) -> None: """Keep the fixture over the windows an app opens, which would take the probe's clicks: maximized and above other windows rather than full screen, since xfwm4 keeps a window an app puts above over a full-screen one, and raised diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/wayland_fixture.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/wayland_fixture.py new file mode 100644 index 000000000..ccc7961f8 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/conformance/wayland_fixture.py @@ -0,0 +1,299 @@ +"""The probe's fixture on a Wayland desktop: fixture.py's Fixture Desk drawn with GTK 3 as a native Wayland client, +since Tk runs only under XWayland, whose windows a Wayland driver may not reach. Same widgets at the same places, and +the same layout.json and state.json, all from desk.py, which is staged beside it. Wayland tells a client +nothing of where its window is, so it goes full screen and takes its origin from Hyprland's IPC when it answers; the +clipboard goes through wl-clipboard, which needs no focus. Run as ``python3 wayland_fixture.py ``; it needs +a 1280x800 screen or larger and PyGObject.""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + +import gi + +gi.require_version("Gdk", "3.0") +gi.require_version("Gtk", "3.0") +from gi.repository import Gdk, GLib, Gtk # noqa: E402 + +from desk import (AT, CELL, CLIPBOARD, COLOURS, GRID, INBOX_EVERY_MS, ITEMS, MARKERS, MIN_SIZE, SIZE, # noqa: E402 + new_state, record_drag, record_key, write_layout, write_state) + +#: How often the fixture asks Hyprland where its window is until it's there, and for how long. +PLACE_EVERY_MS, PLACE_TRIES = 100, 50 +#: How long the fixture waits for hyprctl to answer, for wl-copy to take the clipboard and for wl-paste to read it. +HYPRCTL_TIMEOUT_S, COPY_TIMEOUT_S, PASTE_TIMEOUT_S = 2, 5, 2 +FONT = "DejaVu Sans" + + +def main(state_dir: Path) -> None: + state_dir.mkdir(parents=True, exist_ok=True) + GLib.set_prgname("FixtureDesk") + monitor = Gdk.Display.get_default().get_monitor(0).get_geometry() + width, height = monitor.width, monitor.height + state = new_state(width, height, "wayland") + + def save() -> None: + write_state(state_dir, state) + + if width < MIN_SIZE[0] or height < MIN_SIZE[1]: + state["error"] = f"the screen is {width}x{height}; the fixture needs {MIN_SIZE[0]}x{MIN_SIZE[1]}" + save() + return + + css = Gtk.CssProvider() + css.load_from_data(f"* {{ font-family: '{FONT}'; font-size: 16px; }} textview, treeview {{ font-size: 14px; }}" + f" window {{ background-color: #f0f0f0; }}".encode()) + Gtk.StyleContext.add_provider_for_screen(Gdk.Screen.get_default(), css, Gtk.STYLE_PROVIDER_PRIORITY_APPLICATION) + window = Gtk.Window(title="Fixture Desk") + window.set_decorated(False) + window.set_default_size(width, height) + desk = Gtk.Fixed() + window.add(desk) + + desk.put(Gtk.Label(label="Name:"), *AT["label"]) + name = Gtk.Entry(width_chars=30) + desk.put(name, *AT["name"]) + + def key(_widget, event) -> bool: + state["name"] = name.get_text() + record_key(state, Gdk.keyval_name(event.keyval)) + state["clipboard"] = _clipboard() + save() + return False + + name.connect("key-release-event", key) + + def on_save(_button) -> None: + state["saved_count"] += 1 + save() + + button = Gtk.Button(label="Save") + button.set_size_request(120, -1) + button.connect("clicked", on_save) + desk.put(button, *AT["save"]) + + def on_subscribe(check) -> None: + state["subscribed"] = check.get_active() + save() + + subscribe = Gtk.CheckButton(label="Subscribe") + subscribe.connect("toggled", on_subscribe) + desk.put(subscribe, *AT["subscribe"]) + + notes = Gtk.TextView() + notes.set_size_request(340, 80) + notes.set_accepts_tab(True) + + def note(_widget, _event) -> bool: + buffer = notes.get_buffer() + state["notes"] = buffer.get_text(buffer.get_start_iter(), buffer.get_end_iter(), True) + save() + return False + + notes.connect("key-release-event", note) + desk.put(notes, *AT["notes"]) + + items = Gtk.ListStore(str) + for i in range(1, ITEMS + 1): + items.append([f"Item {i}"]) + listing = Gtk.TreeView(model=items, headers_visible=False) + listing.append_column(Gtk.TreeViewColumn("Item", Gtk.CellRendererText(), text=0)) + scroller = Gtk.ScrolledWindow() + scroller.set_size_request(SIZE["list"]["width"], SIZE["list"]["height"]) + scroller.add(listing) + desk.put(scroller, *AT["list"]) + + def scrolled(adjustment) -> None: + upper = adjustment.get_upper() + state["scroll_top"] = adjustment.get_value() / upper if upper else 0.0 + save() + + scroller.get_vadjustment().connect("value-changed", scrolled) + + def select(selection) -> None: + model, row = selection.get_selected() + state["selected"] = model[row][0] if row else None + save() + + listing.get_selection().connect("changed", select) + + grid = _area(GRID * CELL, GRID * CELL) + desk.put(grid, *AT["grid"]) + + def draw_grid(_area_, cr) -> None: + for row in range(GRID): + for col in range(GRID): + cr.set_source_rgb(*_rgb(COLOURS[state["cells"].get(f"{row},{col}", 0)])) + cr.rectangle(col * CELL, row * CELL, CELL, CELL) + cr.fill() + cr.set_source_rgb(*_rgb("#999999")) + cr.set_line_width(1) + cr.rectangle(col * CELL + 0.5, row * CELL + 0.5, CELL - 1, CELL - 1) + cr.stroke() + + def click(_area_, event) -> bool: + if event.type == Gdk.EventType._2BUTTON_PRESS: + state["double_clicks"] += 1 + elif event.type == Gdk.EventType.BUTTON_PRESS and event.button in (1, 3): + row, col = int(event.y) // CELL, int(event.x) // CELL + state["cells"][f"{row},{col}"] = (state["cells"].get(f"{row},{col}", 0) + 1) % len(COLOURS) + state["right_clicks"] += event.button == 3 + grid.queue_draw() + save() + return True + + grid.connect("draw", draw_grid) + grid.connect("button-press-event", click) + + pad = _area(**SIZE["pad"]) + desk.put(pad, *AT["pad"]) + stroke: list = [] + drawn: list = [] + + def draw_pad(_area_, cr) -> None: + cr.set_source_rgb(*_rgb("#fffbe6")) + cr.paint() + cr.set_source_rgb(0, 0, 0) + cr.set_line_width(3) + for line in drawn + [stroke]: + for (x0, y0), (x1, y1) in zip(line, line[1:]): + cr.move_to(x0, y0) + cr.line_to(x1, y1) + cr.stroke() + + def press(_area_, event) -> bool: + if event.button == 1: + stroke[:] = [(round(event.x), round(event.y))] + return True + + def motion(_area_, event) -> bool: + if stroke: + stroke.append((round(event.x), round(event.y))) + pad.queue_draw() + return True + + def release(_area_, event) -> bool: + if stroke and event.button == 1: + stroke.append((round(event.x), round(event.y))) + record_drag(state, stroke) + drawn.append(list(stroke)) + stroke.clear() + save() + return True + + pad.connect("draw", draw_pad) + pad.connect("button-press-event", press) + pad.connect("motion-notify-event", motion) + pad.connect("button-release-event", release) + + wheel = _area(**SIZE["wheel"]) + wheel.add_events(Gdk.EventMask.SMOOTH_SCROLL_MASK) + desk.put(wheel, *AT["wheel"]) + + def wheeled(_area_, event) -> bool: + """A step a scroll event: by its smooth deltas, since GTK 3 on Wayland names a horizontal wheel's discrete + direction the wrong way round, a step left RIGHT, where its deltas and GTK 4 have it left.""" + _, dx, dy = event.get_scroll_deltas() + for way, delta in (("left" if dx < 0 else "right", dx), ("up" if dy < 0 else "down", dy)): + if delta: + state["wheel"][way] += 1 + save() + return True + + wheel.connect("draw", lambda _a, cr: (cr.set_source_rgb(*_rgb("#e6f0ff")), cr.paint())) + wheel.connect("scroll-event", wheeled) + + field = _area(**SIZE["field"]) + desk.put(field, *AT["field"]) + + def draw_field(_area_, cr) -> None: + cr.set_source_rgb(0, 0, 0) + cr.paint() + for x, y, colour in MARKERS: + cr.set_source_rgb(*_rgb(colour)) + cr.rectangle(x, y, 1, 1) + cr.fill() + + field.connect("draw", draw_field) + + def poll_inbox() -> bool: + message = state_dir / "inbox.txt" + if message.exists() and (text := message.read_text()) != state["loaded"]: + state["loaded"] = text + save() + return True + + tries = [0] + + def layout() -> bool: + tries[0] += 1 + origin = _origin() + if origin is None and tries[0] < PLACE_TRIES: + return True + ox, oy = origin or (0, 0) + boxes = {} + for key_, widget in (("name", name), ("save", button), ("subscribe", subscribe), ("notes", notes), + ("list", listing), ("grid", grid), ("pad", pad), ("wheel", wheel), ("field", field)): + x, y = widget.translate_coordinates(window, 0, 0) + allocation = widget.get_allocation() + boxes[key_] = [ox + x, oy + y, allocation.width, allocation.height] + first = listing.get_cell_area(Gtk.TreePath.new_first(), listing.get_column(0)) + fx, fy = field.translate_coordinates(window, 0, 0) + save() + write_layout(state_dir, boxes, first.height, (ox + fx, oy + fy)) + return False + + _set_clipboard(CLIPBOARD) + state["clipboard"] = CLIPBOARD + window.connect("destroy", Gtk.main_quit) + window.show_all() + window.fullscreen() + GLib.timeout_add(PLACE_EVERY_MS, layout) + GLib.timeout_add(INBOX_EVERY_MS, poll_inbox) + Gtk.main() + + +def _area(width: int, height: int) -> Gtk.DrawingArea: + area = Gtk.DrawingArea() + area.set_size_request(width, height) + area.add_events(Gdk.EventMask.BUTTON_PRESS_MASK | Gdk.EventMask.BUTTON_RELEASE_MASK + | Gdk.EventMask.BUTTON1_MOTION_MASK | Gdk.EventMask.SCROLL_MASK) + return area + + +def _origin() -> tuple[int, int] | None: + """Where Hyprland put this process's full-screen window, once it's full screen; None until then, or without + Hyprland.""" + try: + clients = json.loads(subprocess.run(["hyprctl", "-j", "clients"], capture_output=True, text=True, + timeout=HYPRCTL_TIMEOUT_S).stdout) + except (OSError, subprocess.TimeoutExpired, ValueError): + return None + mine = [c for c in clients if c.get("pid") == os.getpid() and c.get("title") == "Fixture Desk"] + return tuple(mine[0]["at"]) if mine and mine[0].get("fullscreen") else None + + +def _rgb(colour: str) -> tuple[float, float, float]: + return tuple(int(colour[i:i + 2], 16) / 255 for i in (1, 3, 5)) + + +def _set_clipboard(text: str) -> None: + try: + subprocess.run(["wl-copy", "--", text], check=True, timeout=COPY_TIMEOUT_S) + except (OSError, subprocess.SubprocessError): + Gtk.Clipboard.get(Gdk.SELECTION_CLIPBOARD).set_text(text, -1) + + +def _clipboard() -> str | None: + try: + return subprocess.run(["wl-paste", "--no-newline"], capture_output=True, text=True, + timeout=PASTE_TIMEOUT_S).stdout + except (OSError, subprocess.SubprocessError): + return Gtk.Clipboard.get(Gdk.SELECTION_CLIPBOARD).wait_for_text() + + +if __name__ == "__main__": + main(Path(sys.argv[1])) diff --git a/packages/agentenv-protocol/tests/test_computer_conformance.py b/packages/agentenv-protocol/tests/test_computer_conformance.py index 3a0742f97..0adacfd7e 100644 --- a/packages/agentenv-protocol/tests/test_computer_conformance.py +++ b/packages/agentenv-protocol/tests/test_computer_conformance.py @@ -8,6 +8,7 @@ import json import subprocess import sys +from pathlib import Path from types import ModuleType, SimpleNamespace import pytest @@ -18,7 +19,7 @@ from computer_fake import FakeScreen, MinimalDriver # noqa: E402 from mcp.types import CallToolResult, TextContent # noqa: E402 from PIL import ImageChops # noqa: E402 -from probe_fake import FixtureDriver, served # noqa: E402 +from probe_fake import FakeDesk, FixtureDriver, served # noqa: E402 from agentenv_protocol.computer import ( # noqa: E402 ComputerSettings, @@ -197,6 +198,48 @@ async def recorded(argv, *args, **kwargs): assert not [argv for argv in sent if argv[0] == "sh"] +class WaylandDesk(FakeDesk): + """The GTK desk as the probe uses it: GTK's own Ctrl+A selects a text box's text, and Tk's Ctrl+/ does nothing, + so a probe that sent it there would fail.""" + + def press(self, keys: list[str]) -> None: + if keys != ["Control", "/"]: + super().press(["Control", "/"] if keys == ["Control", "a"] else keys) + + +class WaylandDriver(FixtureDriver): + """A driver on a Wayland desktop, whose desk is the GTK one, started only from wayland_fixture.py.""" + + def __init__(self) -> None: + super().__init__(platform=Platform(os="linux", distribution="omarchy", version="4.5.0", arch="amd64", + display="wayland")) + self.desk = WaylandDesk(self) + self.staged: list[str] = [] + + def _fixture(self, argv): + text = " ".join(argv) + if f"python3 {checks.STAGE}/wayland_fixture.py" in text: + self.staged = sorted(path.name for path in (Path.home() / checks.STAGE).glob("*.py")) + self.desk.start() + return ExecResult(0, b"4242\n") + if f"python3 {checks.STAGE}/fixture.py" in text: + return ExecResult(1, b"", b"Tk runs under XWayland here") + if argv == [*checks.OWNER, f"{checks.STAGE}/wayland_fixture.py"]: + return ExecResult(0, b"1000\n") + return super()._fixture(argv) + + +@pytest.mark.asyncio +async def test_a_wayland_computer_is_probed_with_the_gtk_desk_beside_the_tk_desks_file_selecting_all_with_ctrl_a(home): + driver = WaylandDriver() + + report = await _probe(driver) + + assert report.passed, _failed(report) + assert driver.staged == ["desk.py", "wayland_fixture.py"] + assert ["kill", "4242"] in driver.commands and not (home / checks.STAGE).exists() + + class NeverReady(FixtureDriver): async def ready_checks(self): return [ReadyCheck("session", False, "no desktop session")] @@ -395,12 +438,30 @@ def test_the_fixture_gives_the_focus_back_to_the_text_box_that_had_it_when_it_ta def _fixture(monkeypatch): - """The fixture module, which runs in the computer, read with Tk stood in for, as this Python may have none.""" + """The fixture module, which runs in the computer, read with Tk stood in for, as this Python may have none, and + with desk.py importable as it is beside the fixture in the computer.""" monkeypatch.setitem(sys.modules, "tkinter", ModuleType("tkinter")) + monkeypatch.syspath_prepend(str(Path(checks.__file__).parent)) monkeypatch.delitem(sys.modules, "agentenv_protocol.computer.conformance.fixture", raising=False) return importlib.import_module("agentenv_protocol.computer.conformance.fixture") +def test_the_gtk_desk_starts_on_a_computer_without_tk(monkeypatch): + gi = ModuleType("gi") + gi.require_version = lambda namespace, version: None + gi.repository = SimpleNamespace(Gdk=None, GLib=None, Gtk=None) + monkeypatch.setitem(sys.modules, "gi", gi) + monkeypatch.setitem(sys.modules, "gi.repository", gi.repository) + monkeypatch.setitem(sys.modules, "tkinter", None) + monkeypatch.syspath_prepend(str(Path(checks.__file__).parent)) + for name in ("desk", "wayland_fixture"): + monkeypatch.delitem(sys.modules, name, raising=False) + + gtk_desk = importlib.import_module("wayland_fixture") + + assert gtk_desk.AT["grid"] == (400, 160) + + def test_a_refusal_must_carry_the_code_and_applied_no(): def result(error: bool, **data) -> CallToolResult: return CallToolResult(content=[TextContent(type="text", text="x")], structuredContent=data, isError=error) diff --git a/src/agent_env/drivers/README.md b/src/agent_env/drivers/README.md index b24c86cd6..564f089dc 100644 --- a/src/agent_env/drivers/README.md +++ b/src/agent_env/drivers/README.md @@ -75,6 +75,23 @@ the foreground between the window and its frame, so a few of the first burst's c at once all land); `[[known_failures.windows]]` names it. The text of Windows 11's Notepad can't be read through cua-driver's UI Automation tree, whose walk of it fails. +On a Hyprland desktop (Wayland), as Omarchy's is, cua-driver runs its Wayland backend (`CUA_DRIVER_RS_ENABLE_WAYLAND=1`) +and sends input through the seats of Hyprland's `cua-hyprland-plugin`. Hyprland's session turns this on +(`HYPRLAND_INSTANCE_SIGNATURE`), not any Wayland one, since the driver reaches Hyprland's IPC; `supports` names no +Wayland desktop yet, so gate 1 doesn't match it to one. The daemon starts once the compositor's socket answers, on a +socket in the session user's runtime folder. What the driver asks Hyprland (`hyprctl -j`) is in `hyprland.py`, beside +`driver.py`. Keys and text go to the window Hyprland has focused, in the foreground, and to the desktop with nothing +focused; clicks, drags and scrolls stay desktop-scope. `computer-ready` also checks that the plugin is loaded, enabled, +built for this Hyprland and taking input (`transport.ready`), and every input, a `computer_*` operation or one of +cua-driver's own tools that isn't read-only, checks it again first and fails as `device_unavailable`, sending nothing, +when it isn't. The runtime runs ready checks only until they first pass, and a Hyprland restarted after a crash runs in +safe mode with no plugins, where input is accepted but never arrives. A plugin loaded again into a running Hyprland has +no input seat until the desktop restarts, and a driver that outlives its Hyprland can't reach the next one, which has a +new instance signature, so the driver's services have to restart with the graphical session. Keys aimed at a window +skip Hyprland's own shortcuts, such as Super+2, and are refused while a layer such as Omarchy's menu holds the keyboard, +and non-ASCII text aimed at one is refused; sent to the desktop in the foreground, all three work, but Backspace, the +arrows and Home/End do nothing in a text box, so neither way alone covers both. + To move it to another cua-driver version: 1. Pin the version in `requirements.txt`. diff --git a/src/agent_env/drivers/envs/cua-driver/driver.py b/src/agent_env/drivers/envs/cua-driver/driver.py index 992aa66bc..9f0f5a826 100644 --- a/src/agent_env/drivers/envs/cua-driver/driver.py +++ b/src/agent_env/drivers/envs/cua-driver/driver.py @@ -8,7 +8,9 @@ Windows, and each call goes on a connection of its own, as cua-driver's own MCP server sends them. The framework and each caller, told apart by MCP session, have an unnamed cua-driver session of their own, which cua-driver starts again after ending it for idling, where a named one stays ended. A call the daemon can't take, as while it restarts, fails as unavailable; input whose reply is lost fails with its outcome unknown, since it may have -arrived. It needs ``agentenv-framework-protocol[computer]`` and ``cua-driver`` where it runs.""" +arrived. It needs ``agentenv-framework-protocol[computer]`` and ``cua-driver`` where it runs. On a Hyprland desktop +(Wayland) it runs cua-driver's Wayland backend, sends keys and text to the focused window in the foreground, and is +ready, and takes input, only while Hyprland has cua-driver's plugin loaded, asking Hyprland through hyprland.py.""" from __future__ import annotations import asyncio @@ -57,10 +59,22 @@ type_text, ) +import hyprland + MACOS, WINDOWS = sys.platform == "darwin", sys.platform == "win32" -#: Where cua-driver's daemon listens. -SOCKET = (r"\\.\pipe\agentenv-cua-driver" if WINDOWS else - str(Path("/tmp/agentenv-cua-driver.sock" if MACOS else "/run/agentenv/cua-driver.sock"))) +#: A Hyprland desktop, on Wayland, where cua-driver runs its Wayland backend and its input goes through the seats of +#: its Hyprland plugin; what follows reaches Hyprland's IPC, so it's Hyprland's session that turns it on, not any +#: Wayland one. +HYPRLAND = sys.platform == "linux" and bool(os.environ.get("HYPRLAND_INSTANCE_SIGNATURE")) +#: Where cua-driver's daemon listens: on Hyprland in the session user's runtime folder, which the session's units share. +if WINDOWS: + SOCKET = r"\\.\pipe\agentenv-cua-driver" +elif MACOS: + SOCKET = "/tmp/agentenv-cua-driver.sock" +elif HYPRLAND: + SOCKET = str(Path(os.environ["XDG_RUNTIME_DIR"]) / "agentenv-cua-driver.sock") +else: + SOCKET = "/run/agentenv/cua-driver.sock" #: What cua-driver's own MCP server (``cua-driver mcp``) says at connect and lists at the pinned version on each OS: its #: instructions, its tools, which the running daemon must list the same, and its skill resources. MANIFEST = Path(__file__).with_name("mcp-macos.json" if MACOS else "mcp-windows.json" if WINDOWS else "mcp.json") @@ -88,6 +102,8 @@ PERMISSIONS = ("accessibility", "screen_recording") #: What keeps cua-driver from reaching trycua: no telemetry and no update checks. QUIET = {"DO_NOT_TRACK": "1", "CUA_DRIVER_RS_TELEMETRY_ENABLED": "false", "CUA_DRIVER_RS_UPDATE_CHECK": "false"} +#: What turns on cua-driver's Wayland backend, which it takes only when asked. +WAYLAND_BACKEND = {"CUA_DRIVER_RS_ENABLE_WAYLAND": "1"} #: W3C modifier keys and the names cua-driver holds them by. MODIFIERS = {"Control": "ctrl", "Shift": "shift", "Alt": "alt", "Meta": "super", "Super": "super"} #: W3C key values and the names cua-driver presses them by; any printable ASCII character is pressed as itself. @@ -127,7 +143,8 @@ def __init__(self) -> None: def forwarded_tools(self) -> list[ForwardedTool]: return [ForwardedTool(EnvironmentTool(name=t["name"], description=t["description"], inputSchema=t["inputSchema"]), - functools.partial(self._forward, t["name"]), t["annotations"] or None, t.get("outputSchema")) + functools.partial(self._forward, t["name"], not (t["annotations"] or {}).get("readOnlyHint")), + t["annotations"] or None, t.get("outputSchema")) for t in _manifest()["tools"]] @resource("skill://cua-driver/{file}", listed=lambda env: _manifest()["resources"]) @@ -143,10 +160,10 @@ def cua_driver_instructions(self) -> str: async def ready_checks(self) -> list[ReadyCheck]: """That the daemon answers as the pinned cua-driver does: its executable's sha256 and its tools, checked - until they pass once.""" + until they pass once; and on Hyprland, every time, that it has cua-driver's plugin, which a restart drops.""" if self._pinned is None or not all(check.ok for check in self._pinned): self._pinned = await self._pinned_checks() - return self._pinned + return [*self._pinned, await hyprland.plugin()] if HYPRLAND else self._pinned async def bundled_skill(self) -> tuple[str, str, dict[str, str]]: if self._skill is None: @@ -190,19 +207,23 @@ async def scroll_at(self, x, y, direction, amount, modifiers) -> None: @type_text async def type_in(self, text) -> None: - if not MACOS and not WINDOWS and any(ord(ch) >= LATIN_1 for ch in text): + if not MACOS and not WINDOWS and not HYPRLAND and any(ord(ch) >= LATIN_1 for ch in text): await asyncio.to_thread(self._hold, text) - await self._send("type_text", text=text) + await self._send("type_text", text=text, **(await hyprland.focused() if HYPRLAND else {})) @press async def press_keys(self, keys) -> None: *held, key = keys - await self._send("press_key", modifiers=_modifiers(held), key=MODIFIERS.get(key) or NAMED.get(key, key)) + await self._send("press_key", modifiers=_modifiers(held), key=MODIFIERS.get(key) or NAMED.get(key, key), + **(await hyprland.focused() if HYPRLAND else {})) - async def _forward(self, tool: str, /, **arguments: Any) -> CallToolResult: - """A call to one of cua-driver's own tools in its caller's session, answered as cua-driver answers it.""" + async def _forward(self, tool: str, acts: bool, /, **arguments: Any) -> CallToolResult: + """A call to one of cua-driver's own tools in its caller's session, answered as cua-driver answers it; one that + isn't read-only is refused on Hyprland without cua-driver's plugin.""" session = self._callers.setdefault(ToolContext.current().caller.session, uuid.uuid4().hex) try: + if acts and HYPRLAND: + await hyprland.require_plugin() reply = await _request({"method": "call", "name": tool, "args": arguments, "session_id": session}) except DeviceUnavailable as e: return _refused(e.message) @@ -229,6 +250,8 @@ def _x_keymap(self) -> _Keymap: return self._keymap async def _send(self, name: str, **args: Any) -> None: + if HYPRLAND: + await hyprland.require_plugin() await self._call(name, DeliveryFailed, scope="desktop", **args) async def _call(self, name: str, failure: type[Exception], **args: Any) -> dict: @@ -426,21 +449,25 @@ def _skill_pack() -> tuple[str, str, dict[str, str]]: def _daemon() -> None: - """Run cua-driver's daemon once X answers: from the session user's home, where the apps it launches start too, on - the socket the driver reaches it at, with approvals off, since nobody is at the computer to give them, and nothing - sent to trycua. On Windows, in the session user's sign-in, as agentenv-session starts it, and waited on, since - Windows has no exec: one that exits is started again.""" + """Run cua-driver's daemon once the display answers, X's or on Hyprland the compositor's, there with its Wayland + backend: from the session user's home, where the apps it launches start too, on the socket the driver reaches it + at, with approvals off, since nobody is at the computer to give them, and nothing sent to trycua. On Windows, in + the session user's sign-in, as agentenv-session starts it, and waited on, since Windows has no exec: one that exits + is started again.""" if WINDOWS: os.chdir(Path.home()) sys.exit(subprocess.run([str(_executable()), "serve", "--socket", SOCKET, "--dangerously-bypass-approvals", "--no-overlay"], env={**os.environ, **QUIET}).returncode) - display = os.environ["DISPLAY"].lstrip(":").partition(".")[0] - while not Path(f"/tmp/.X11-unix/X{display}").is_socket(): + if HYPRLAND: + display = Path(os.environ["XDG_RUNTIME_DIR"]) / os.environ["WAYLAND_DISPLAY"] + else: + display = Path(f"/tmp/.X11-unix/X{os.environ['DISPLAY'].lstrip(':').partition('.')[0]}") + while not display.is_socket(): time.sleep(0.2) os.chdir(Path.home()) executable = str(_executable()) os.execve(executable, [executable, "serve", "--socket", SOCKET, "--dangerously-bypass-approvals", "--no-overlay"], - {**os.environ, **QUIET}) + {**os.environ, **QUIET, **(WAYLAND_BACKEND if HYPRLAND else {})}) if __name__ == "__main__": diff --git a/src/agent_env/drivers/envs/cua-driver/hyprland.py b/src/agent_env/drivers/envs/cua-driver/hyprland.py new file mode 100644 index 000000000..b940a3440 --- /dev/null +++ b/src/agent_env/drivers/envs/cua-driver/hyprland.py @@ -0,0 +1,64 @@ +"""What cua-driver's driver asks a Hyprland desktop, through Hyprland's IPC (``hyprctl -j``): where keys go, and whether +cua-driver's Hyprland plugin can take input. driver.py imports it from beside itself, its folder importable wherever +the driver runs, and calls it only on Hyprland.""" +from __future__ import annotations + +import asyncio +import json +import subprocess + +from agentenv_protocol.computer import DeviceUnavailable, ReadyCheck + +#: How long a request to Hyprland's IPC waits for its answer. +TIMEOUT_S = 5 + + +async def focused() -> dict: + """Where keys go: the focused window, whose Hyprland address is its cua-driver window id, in the foreground through + the plugin's seat, since a desktop-scope key with modifiers goes through libei, which Hyprland doesn't serve; with + nothing focused, the desktop.""" + window = await asyncio.to_thread(hyprctl, "activewindow") + if not window or not window.get("address"): + return {} + return {"pid": window["pid"], "window_id": int(window["address"], 16), "delivery_mode": "foreground"} + + +async def plugin() -> ReadyCheck: + """That Hyprland has cua-driver's plugin loaded, enabled, built for it and taking input; one restarted after a + crash runs in safe mode, with no plugins, where input is sent but never arrives, and a plugin loaded again into a + running Hyprland answers but has no input seat until the desktop restarts. When Hyprland doesn't answer at all, as + once it restarts under a new instance signature after the driver started, it says so instead.""" + status = await asyncio.to_thread(hyprctl, "cua:status") + transport = (status or {}).get("transport") or {} + if status is None and await asyncio.to_thread(hyprctl, "activeworkspace") is None: + problem = ("Hyprland doesn't answer on its IPC; if it restarted after this driver started, restart the driver " + "with it") + elif status is None: + problem = "Hyprland hasn't loaded cua-driver's plugin" + elif status.get("configured") is not True: + problem = "cua-driver's plugin is loaded but not enabled" + elif (status.get("abi") or {}).get("match") is not True: + problem = "cua-driver's plugin was built for another Hyprland" + elif transport.get("ready") is not True: + problem = f"cua-driver's plugin isn't taking input: {transport.get('last_error') or 'its transport is down'}" + else: + return ReadyCheck("cua-driver plugin", True, "loaded, enabled, built for this Hyprland and taking input") + return ReadyCheck("cua-driver plugin", False, problem) + + +async def require_plugin() -> None: + """Input refused unless cua-driver's plugin can take it: the runtime runs ready checks only until they first pass, + and input sent while Hyprland runs without the plugin is accepted but never arrives.""" + if not (check := await plugin()).ok: + raise DeviceUnavailable(check.detail) + + +def hyprctl(request: str) -> dict | None: + """Hyprland's JSON answer to ``request``, or None when it gives none, as for a plugin's request it doesn't know or + when Hyprland doesn't answer.""" + try: + out = subprocess.run(["hyprctl", "-j", request], capture_output=True, text=True, timeout=TIMEOUT_S) + answer = json.loads(out.stdout) + except (OSError, subprocess.TimeoutExpired, ValueError): + return None + return answer if isinstance(answer, dict) else None diff --git a/tst/integration/computer/recording/artifacts/drive_desk/verify.py b/tst/integration/computer/recording/artifacts/drive_desk/verify.py index 2e7b811e4..b79669f3c 100644 --- a/tst/integration/computer/recording/artifacts/drive_desk/verify.py +++ b/tst/integration/computer/recording/artifacts/drive_desk/verify.py @@ -13,7 +13,8 @@ from agentenv_protocol.computer.conformance.client import Driver THUMB = (64, 40) -FIXTURE = resources.files("agentenv_protocol.computer.conformance").joinpath("fixture.py").read_bytes() +DESK = {name: resources.files("agentenv_protocol.computer.conformance").joinpath(name).read_bytes() + for name in ("fixture.py", "desk.py")} def _thumb(result): @@ -23,7 +24,8 @@ def _thumb(result): async def verify(mcp_url, env_url): driver = await Driver.connect(env_url, 60) - await driver.add_file("desk/fixture.py", FIXTURE) + for name, source in DESK.items(): + await driver.add_file(f"desk/{name}", source) await driver.run("sh", "-c", "python3 desk/fixture.py fixture > desk/fixture.log 2>&1 &") for _ in range(60): # the Fixture Desk writes its layout once it is up out = await driver.run("cat", "fixture/layout.json") diff --git a/tst/integration/computer/recording_chaos_test.py b/tst/integration/computer/recording_chaos_test.py index 6fe83a82e..5f0c30641 100644 --- a/tst/integration/computer/recording_chaos_test.py +++ b/tst/integration/computer/recording_chaos_test.py @@ -33,7 +33,8 @@ ARCH = {"x86_64": "amd64", "amd64": "amd64", "aarch64": "arm64", "arm64": "arm64"}[platform.machine().lower()] -FIXTURE = resources.files("agentenv_protocol.computer.conformance").joinpath("fixture.py").read_bytes() +DESK = {name: resources.files("agentenv_protocol.computer.conformance").joinpath(name).read_bytes() + for name in ("fixture.py", "desk.py")} @pytest.fixture(scope="module") @@ -147,7 +148,8 @@ async def _ready(name: str) -> Driver: async def _layout(driver: Driver) -> dict: """The Fixture Desk, started through data/add and computer-exec as the probe starts it, once it wrote its layout.""" - await driver.add_file("desk/fixture.py", FIXTURE) + for name, source in DESK.items(): + await driver.add_file(f"desk/{name}", source) await driver.run("sh", "-c", "python3 desk/fixture.py fixture > desk/fixture.log 2>&1 &") for _ in range(120): out = await driver.run("cat", "fixture/layout.json") diff --git a/tst/unit/config/test_explain_and_sources.py b/tst/unit/config/test_explain_and_sources.py index 07d55ad1f..85d064ccd 100644 --- a/tst/unit/config/test_explain_and_sources.py +++ b/tst/unit/config/test_explain_and_sources.py @@ -163,7 +163,9 @@ def test_sources_says_how_the_file_was_found(config_file, tmp_path, monkeypatch) # Read for identity (USER), or set from [sandbox.providers.sail_vm.config] for the Sail SDK to read # (SAIL_*), never configuration agent-env takes, so none is a layer. # AGENT_ENV_INSTALLED_REQUIRES marks a command run again after installing what a bundle requires, so it never asks twice. -_NOT_CONFIGURATION = {"USER", "DISPLAY", "SAIL_API_KEY", "SAIL_RUNTIME_THREADS", "AGENT_ENV_INSTALLED_REQUIRES"} +# DISPLAY, WAYLAND_DISPLAY and XDG_RUNTIME_DIR are a computer's desktop session, which a driver in it reads. +_NOT_CONFIGURATION = {"USER", "DISPLAY", "WAYLAND_DISPLAY", "XDG_RUNTIME_DIR", "SAIL_API_KEY", "SAIL_RUNTIME_THREADS", + "AGENT_ENV_INSTALLED_REQUIRES"} def _env_vars_read_by(root): diff --git a/tst/unit/drivers/cua_driver_test.py b/tst/unit/drivers/cua_driver_test.py index bf613550e..f174573be 100644 --- a/tst/unit/drivers/cua_driver_test.py +++ b/tst/unit/drivers/cua_driver_test.py @@ -9,6 +9,7 @@ import importlib.util import io import json +import socket import sys import types import stat @@ -121,8 +122,11 @@ def sync(self): @pytest.fixture def module(monkeypatch): + """driver.py loaded as it is on an X11 desktop on Linux, whatever runs the tests, with X stood in for; a test of + another platform sets that platform's flags itself.""" monkeypatch.delenv("ENVIRONMENT_NAME", raising=False) monkeypatch.delenv("AGENTENV_COMPUTER_SETTINGS", raising=False) + monkeypatch.delenv("HYPRLAND_INSTANCE_SIGNATURE", raising=False) xlib = types.ModuleType("Xlib") xlib.display = types.ModuleType("Xlib.display") xlib.display.Display = FakeKeymap @@ -132,9 +136,13 @@ def module(monkeypatch): monkeypatch.setitem(sys.modules, "Xlib", xlib) monkeypatch.setitem(sys.modules, "Xlib.display", xlib.display) monkeypatch.setitem(sys.modules, "Xlib.error", xlib.error) + monkeypatch.syspath_prepend(str(DRIVER.parent)) + monkeypatch.delitem(sys.modules, "hyprland", raising=False) spec = importlib.util.spec_from_file_location("cua_driver_driver", DRIVER) loaded = importlib.util.module_from_spec(spec) - spec.loader.exec_module(loaded) + with monkeypatch.context() as linux: + linux.setattr(sys, "platform", "linux") + spec.loader.exec_module(loaded) return loaded @@ -443,3 +451,128 @@ def daemon_down(path, mode, buffering): monkeypatch.setattr(module, "open", daemon_down, raising=False) with pytest.raises(DeviceUnavailable, match="isn't answering"): module._piped({"method": "list"}) + + +#: What ``hyprctl -j cua:status`` answers while cua-driver's plugin is loaded, enabled, built for that Hyprland and +#: taking input. +PLUGIN_LOADED = {"configured": True, "abi": {"match": True}, "transport": {"ready": True, "last_error": ""}} +#: Its answer once it's loaded again into a running Hyprland, which gives it no input seat until the desktop restarts. +PLUGIN_RELOADED = {**PLUGIN_LOADED, "state": "discovery_only", "transport": { + "ready": False, + "last_error": "input seat lifetime unavailable; restart the desktop before loading a replacement plugin"}} + + +class FakeHyprland: + """Hyprland's IPC as ``hyprctl -j`` answers: an answer by request, a list of them answered one per ask, None for a + request it has no answer to, and every request asked, in order.""" + + def __init__(self, answers: dict): + self.answers, self.asked = answers, [] + + def __call__(self, request: str) -> dict | None: + self.asked.append(request) + answer = self.answers.get(request) + return answer.pop(0) if isinstance(answer, list) else answer + + +@pytest.fixture +def hyprland(module, monkeypatch) -> FakeHyprland: + """The driver as it runs in a Hyprland session, whatever OS the tests run on, with cua-driver's plugin taking + input and nothing focused; the fake IPC, whose answers a test changes.""" + for name, value in (("MACOS", False), ("WINDOWS", False), ("FOREGROUND", {}), ("HYPRLAND", True)): + monkeypatch.setattr(module, name, value) + fake = FakeHyprland({"cua:status": PLUGIN_LOADED, "activeworkspace": {"id": 1}}) + monkeypatch.setattr(module.hyprland, "hyprctl", fake) + return fake + + +@pytest.mark.asyncio +async def test_on_hyprland_keys_go_to_the_focused_window_in_the_foreground_and_text_skips_xs_keymap( + module, daemon, hyprland): + hyprland.answers["activewindow"] = {"address": "0x5557e0f44ad0", "pid": 42} + driver = module.CuaDriver() + + await driver.type_in("Hello, Wörld") + await driver.press_keys(["Control", "a"]) + await driver.click_at(10, 20, "left", 1, []) + + focused = {"pid": 42, "window_id": 0x5557E0F44AD0, "delivery_mode": "foreground"} + assert daemon.calls() == [ + ("type_text", {"scope": "desktop", "text": "Hello, Wörld", **focused}), + ("press_key", {"scope": "desktop", "key": "a", "modifiers": ["ctrl"], **focused}), + ("click", {"scope": "desktop", "x": 10, "y": 20, "button": "left", "count": 1, "modifier": []}), + ] + assert driver._keymap is None and hyprland.asked == ["activewindow", "cua:status"] * 2 + ["cua:status"] + + +@pytest.mark.asyncio +async def test_on_hyprland_with_nothing_focused_keys_go_to_the_desktop(module, daemon, hyprland): + await module.CuaDriver().press_keys(["Escape"]) + + assert daemon.calls() == [("press_key", {"scope": "desktop", "key": "escape", "modifiers": []})] + + +@pytest.mark.asyncio +async def test_on_hyprland_input_is_refused_and_not_sent_once_hyprland_runs_without_cua_drivers_plugin( + module, daemon, hyprland): + driver = module.CuaDriver() + await driver.click_at(10, 20, "left", 1, []) + + del hyprland.answers["cua:status"] + for act in (driver.click_at(10, 20, "left", 1, []), driver.press_keys(["Escape"]), driver.type_in("hi")): + with pytest.raises(module.DeviceUnavailable, match="Hyprland hasn't loaded cua-driver's plugin"): + await act + async with mcp_session(driver) as session: + clicked = await session.call_tool("click", {"pid": 42, "window_id": 7, "x": 1, "y": 2}) + listed = await session.call_tool("list_windows", {}) + + assert clicked.isError and clicked.content[0].text == "Hyprland hasn't loaded cua-driver's plugin" + assert not listed.isError + assert [name for name, _ in daemon.calls()] == ["click", "list_windows"] + + +@pytest.mark.asyncio +async def test_on_hyprland_readiness_checks_every_time_that_hyprland_has_cua_drivers_plugin( + module, daemon, hyprland, monkeypatch): + driver = module.CuaDriver() + monkeypatch.setattr(module, "_executable_sha256", lambda: module.EXECUTABLES[driver.platform.arch]) + hyprland.answers.update({ + "cua:status": [PLUGIN_LOADED, None, {**PLUGIN_LOADED, "configured": False}, + {**PLUGIN_LOADED, "abi": {"match": False}}, PLUGIN_RELOADED, None], + "activeworkspace": [{"id": 1}, None]}) + + checks = [await driver.ready_checks() for _ in range(6)] + + assert [(c.name, c.ok) for c in checks[0]] == [("cua-driver", True), ("cua-driver tools", True), + ("cua-driver plugin", True)] + assert [(c[-1].ok, c[-1].detail) for c in checks[1:]] == [ + (False, "Hyprland hasn't loaded cua-driver's plugin"), + (False, "cua-driver's plugin is loaded but not enabled"), + (False, "cua-driver's plugin was built for another Hyprland"), + (False, "cua-driver's plugin isn't taking input: input seat lifetime unavailable; restart the desktop before " + "loading a replacement plugin"), + (False, "Hyprland doesn't answer on its IPC; if it restarted after this driver started, restart the driver " + "with it")] + assert sum(1 for _, r in daemon.requests if r["method"] == "list") == 1 + + +@pytest.mark.usefixtures("hyprland") +def test_on_hyprland_the_daemon_waits_for_the_compositor_and_runs_cua_drivers_wayland_backend(module, monkeypatch): + started = [] + + def execve(path, argv, env): + started.append((argv, env)) + raise SystemExit + with tempfile.TemporaryDirectory(dir="/tmp", prefix="cd-") as runtime: + monkeypatch.setenv("XDG_RUNTIME_DIR", runtime) + monkeypatch.setenv("WAYLAND_DISPLAY", "wayland-1") + monkeypatch.setattr(module, "_executable", lambda: Path("/opt/cua-driver")) + monkeypatch.setattr(module.os, "execve", execve) + with socket.socket(socket.AF_UNIX) as compositor: + compositor.bind(str(Path(runtime) / "wayland-1")) + with pytest.raises(SystemExit): + module._daemon() + + (argv, env), = started + assert argv[:2] == ["/opt/cua-driver", "serve"] and env["CUA_DRIVER_RS_ENABLE_WAYLAND"] == "1" + assert env["DO_NOT_TRACK"] == "1"