From afb96be97fc9953a3b35385a8cdd075a4d3e0d20 Mon Sep 17 00:00:00 2001 From: CodeWhale Bot Date: Wed, 2 Sep 2026 13:45:07 -0700 Subject: [PATCH] feat(plugins): computer-use bundle - screenshot, click, and type over MCP First independent plugin creation on the bundle boundary: Agent Plugins v1.0.0 manifest plus sibling mcp.json (stdio server), a look-act-verify skill, and a /computer command. The server is stdlib-only Python speaking MCP 2024-11-05 with six tools (screen_size, screenshot, click, type_text, press_key, scroll); screenshots downscale to 1920px JPEG via sips/convert and coordinates scale back to display pixels. Routed tool names land in McpAction so every action prompts unless Always-approve. Receipts: 9/9 server protocol tests pass; live 1920x1080 JPEG screenshot captured over stdio on macOS; bundle validates through validate_from_path plus parse_mcp_json in bundled_computer_use_plugin_validates. Signed-off-by: CodeWhale Bot --- crates/tui/src/plugins/manifest.rs | 26 + plugins/computer-use/commands/computer.md | 13 + plugins/computer-use/mcp.json | 1 + plugins/computer-use/plugin.json | 1 + plugins/computer-use/server/server.py | 600 ++++++++++++++++++ plugins/computer-use/server/test_server.py | 97 +++ .../computer-use/skills/computer-use/SKILL.md | 38 ++ 7 files changed, 776 insertions(+) create mode 100644 plugins/computer-use/commands/computer.md create mode 100644 plugins/computer-use/mcp.json create mode 100644 plugins/computer-use/plugin.json create mode 100644 plugins/computer-use/server/server.py create mode 100644 plugins/computer-use/server/test_server.py create mode 100644 plugins/computer-use/skills/computer-use/SKILL.md diff --git a/crates/tui/src/plugins/manifest.rs b/crates/tui/src/plugins/manifest.rs index e42712a459..3f003c180d 100644 --- a/crates/tui/src/plugins/manifest.rs +++ b/crates/tui/src/plugins/manifest.rs @@ -2301,4 +2301,30 @@ args = ["server.js", "--mode=worker", "-e", "console.log('ready')"] assert!(PluginManifest::validate_from_path(&path).is_err()); } } + + #[test] + fn bundled_computer_use_plugin_validates() { + use crate::plugins::agent_plugin; + let root = PathBuf::from(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../plugins/computer-use" + )); + let validated = PluginManifest::validate_from_path(&root.join("plugin.json")) + .expect("in-repo computer-use bundle must validate"); + assert_eq!(validated.manifest.plugin.name, "computer-use"); + assert_eq!(validated.inventory.skills, 1); + assert_eq!(validated.components.commands.len(), 1); + assert!(validated.warnings.is_empty(), "{:?}", validated.warnings); + + let mcp_text = fs::read_to_string(root.join("mcp.json")).unwrap(); + let servers = + agent_plugin::parse_mcp_json(&mcp_text).expect("computer-use mcp.json must parse"); + let computer = servers.get("computer").expect("computer server"); + assert_eq!(computer.command.as_deref(), Some("python3")); + assert!( + computer.args.iter().any(|arg| arg.ends_with("server.py")), + "{:?}", + computer.args + ); + } } diff --git a/plugins/computer-use/commands/computer.md b/plugins/computer-use/commands/computer.md new file mode 100644 index 0000000000..fbe3be13bf --- /dev/null +++ b/plugins/computer-use/commands/computer.md @@ -0,0 +1,13 @@ +--- +description: Drive this machine's screen, mouse, and keyboard +usage: /computer [status|look] +--- + +$ARGUMENTS + +- With no arguments or `status`: report whether the computer-use + plugin is usable here — server reachable, platform supported, + screen size — without taking any action. +- With `look`: take one screenshot and describe what is on screen. +- Anything else (clicking, typing, operating apps) goes through the + computer-use skill's look-act-verify loop with per-action approval. diff --git a/plugins/computer-use/mcp.json b/plugins/computer-use/mcp.json new file mode 100644 index 0000000000..2c748d6fde --- /dev/null +++ b/plugins/computer-use/mcp.json @@ -0,0 +1 @@ +{"mcpServers": {"computer": {"args": ["server/server.py"], "command": "python3", "cwd": ".", "type": "stdio"}}} diff --git a/plugins/computer-use/plugin.json b/plugins/computer-use/plugin.json new file mode 100644 index 0000000000..e0e315c872 --- /dev/null +++ b/plugins/computer-use/plugin.json @@ -0,0 +1 @@ +{"$schema": "https://agent-plugins.org/schemas/plugin.json", "author": {"name": "codewhale"}, "description": "See the screen, click, and type: local computer use over an MCP server plus the look-act-verify skill.", "extensions": {"net.codewhale": {"capabilities": {}, "commands": {"path": "commands"}, "when": {"binaries": ["python3"], "os": ["macos", "linux"]}}}, "keywords": ["computer-use", "screenshot", "automation"], "license": "MIT", "name": "computer-use", "version": "0.1.0"} diff --git a/plugins/computer-use/server/server.py b/plugins/computer-use/server/server.py new file mode 100644 index 0000000000..5563f79fc1 --- /dev/null +++ b/plugins/computer-use/server/server.py @@ -0,0 +1,600 @@ +#!/usr/bin/env python3 +"""codewhale computer-use MCP server (stdlib only, MCP 2024-11-05, stdio). + +Tools: screen_size, screenshot, click, type_text, press_key, scroll. +Platform backends: macOS via screencapture + osascript, Linux via +grim/import + ydotool/wtype. Anything unavailable fails as an MCP +isError result with a machine-readable `reason` so the model can relay +it instead of retrying blind. + +Protocol: newline-delimited JSON-RPC on stdin/stdout. Log to stderr; +stdout is the wire. +""" + +from __future__ import annotations + +import base64 +import json +import os +import platform +import shutil +import struct +import subprocess +import sys +import tempfile + +PROTOCOL_VERSION = "2024-11-05" +SERVER_NAME = "computer-use" +SERVER_VERSION = "0.1.0" +TOOL_TIMEOUT_SECS = 30 +MAX_TYPE_CHARS = 2000 + +_ON_MAC = platform.system() == "Darwin" +_ON_LINUX = platform.system() == "Linux" + +_size_cache: list | None = None +# full-display pixels per screenshot pixel. The model reads positions +# from the (possibly downscaled) screenshot; click scales back up. +_shot_scale: float = 1.0 + + +def log(msg: str) -> None: + sys.stderr.write(f"[computer-use] {msg}\n") + sys.stderr.flush() + + +def err(reason: str, message: str) -> dict: + return { + "content": [{"type": "text", "text": message}], + "isError": True, + "_meta": {"reason": reason}, + } + + +def ok_text(text: str) -> dict: + return {"content": [{"type": "text", "text": text}]} + + +def run(argv: list, **kwargs) -> subprocess.CompletedProcess: + return subprocess.run( + argv, capture_output=True, timeout=TOOL_TIMEOUT_SECS, **kwargs + ) + + +def need_tool(name: str) -> str | None: + path = shutil.which(name) + if path is None: + return None + return path + + +def png_size(path: str) -> tuple | None: + try: + with open(path, "rb") as fh: + header = fh.read(26) + if header[:8] != b"\x89PNG\r\n\x1a\n": + return None + width, height = struct.unpack(">II", header[16:24]) + return [width, height] + except OSError: + return None + + +def mac_screenshot(dest: str) -> str | None: + if need_tool("screencapture") is None: + return "Screen capture helper `screencapture` is not on PATH." + proc = run(["screencapture", "-x", "-t", "png", "-m", dest]) + if proc.returncode != 0: + detail = proc.stderr.decode("utf-8", "replace").strip() + if "not permitted" in detail or "permission" in detail.lower(): + return ( + "Screen Recording permission is missing. Open System " + "Settings > Privacy & Security > Screen Recording and " + "enable the terminal running codewhale, then try again." + ) + return f"screencapture failed: {detail or proc.returncode}" + return None + + +def linux_screenshot(dest: str) -> str | None: + for helper, argv in ( + ("grim", ["grim", dest]), + ("import", ["import", "-window", "root", dest]), + ("scrot", ["scrot", "--overwrite", dest]), + ): + if need_tool(helper) is None: + continue + proc = run(argv) + if proc.returncode == 0: + return None + log(f"{helper} failed, trying next backend") + return ( + "No working screenshot helper found. Install one of grim, " + "ImageMagick import, or scrot." + ) + + +MAX_SCREENSHOT_WIDTH = 1920 + + +def shrink_for_vision(png_path: str) -> tuple: + """Downscale to a vision-sized JPEG with native platform tools. + + Returns (path, mime, size) of the best artifact, falling back to the + original PNG when no resizer exists. Retina PNGs run to tens of + megabytes; vision models read 1920px JPEGs fine at a tenth the cost. + """ + size = png_size(png_path) + if _ON_MAC and need_tool("sips") is not None: + with tempfile.NamedTemporaryFile(suffix=".jpg", delete=False) as tmp: + small = tmp.name + try: + proc = run( + [ + "sips", + "-Z", + str(MAX_SCREENSHOT_WIDTH), + "-s", + "format", + "jpeg", + "-s", + "formatOptions", + "80", + png_path, + "--out", + small, + ] + ) + if proc.returncode == 0: + jpeg_size = png_size(small) + if jpeg_size is None: # JPEG has no PNG header; trust sips + width = min(size[0], MAX_SCREENSHOT_WIDTH) if size else 0 + height = ( + round(size[1] * width / size[0]) + if size and size[0] + else 0 + ) + return small, "image/jpeg", [width, height] + except Exception: + pass + try: + os.unlink(small) + except OSError: + pass + if _ON_LINUX and need_tool("convert") is not None: + with tempfile.NamedTemporaryFile(suffix=".jpg", delete=False) as tmp: + small = tmp.name + try: + proc = run( + [ + "convert", + png_path, + "-resize", + f"{MAX_SCREENSHOT_WIDTH}>", + "-quality", + "80", + small, + ] + ) + if proc.returncode == 0: + width = min(size[0], MAX_SCREENSHOT_WIDTH) if size else 0 + height = ( + round(size[1] * width / size[0]) if size and size[0] else 0 + ) + return small, "image/jpeg", [width, height] + except Exception: + pass + try: + os.unlink(small) + except OSError: + pass + return png_path, "image/png", size or [0, 0] + + +def do_screenshot() -> dict: + if not (_ON_MAC or _ON_LINUX): + return err( + "unsupported_platform", + f"Computer use supports macOS and Linux; this host is {platform.system()}.", + ) + with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as tmp: + dest = tmp.name + small: str | None = None + try: + if _ON_MAC: + problem = mac_screenshot(dest) + else: + problem = linux_screenshot(dest) + if problem is not None: + reason = "missing_permission" if "permission" in problem else "missing_helper" + return err(reason, problem) + full = png_size(dest) + if full is None: + return err("capture_failed", "Screenshot helper produced no readable PNG.") + small, mime, size = shrink_for_vision(dest) + global _size_cache, _shot_scale + _size_cache = size + _shot_scale = (full[0] / size[0]) if size[0] else 1.0 + with open(small, "rb") as fh: + data = base64.b64encode(fh.read()).decode("ascii") + return { + "content": [ + { + "type": "image", + "data": data, + "mimeType": mime, + }, + { + "type": "text", + "text": f"Screenshot {size[0]}x{size[1]}.", + }, + ] + } + finally: + for path in (dest, small): + if path is not None: + try: + os.unlink(path) + except OSError: + pass + + +def do_screen_size() -> dict: + global _size_cache + if _size_cache is not None: + width, height = _size_cache + return ok_text(f'{{"width": {width}, "height": {height}}}') + if _ON_MAC and need_tool("screencapture") is not None: + with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as tmp: + dest = tmp.name + try: + problem = mac_screenshot(dest) + if problem is not None: + return err("missing_helper", problem) + size = png_size(dest) + finally: + try: + os.unlink(dest) + except OSError: + pass + if size is None: + return err("capture_failed", "Could not determine the screen size.") + _size_cache = size + return ok_text(f'{{"width": {size[0]}, "height": {size[1]}}}') + return err( + "unknown_size" if (_ON_MAC or _ON_LINUX) else "unsupported_platform", + "Screen size is unknown until the first screenshot on this platform. " + "Call screenshot first.", + ) + + +def osascript(script: str) -> tuple: + with tempfile.NamedTemporaryFile( + suffix=".scpt", mode="w", delete=False, encoding="utf-8" + ) as tmp: + tmp.write(script) + path = tmp.name + try: + proc = run(["osascript", path]) + finally: + try: + os.unlink(path) + except OSError: + pass + return proc.returncode, proc.stderr.decode("utf-8", "replace").strip() + + +def mac_click(x: int, y: int) -> str | None: + if need_tool("osascript") is None: + return "Missing helper: osascript is not on PATH." + code, detail = osascript( + f'tell application "System Events" to click at {{{x}, {y}}}' + ) + if code != 0: + if "not permitted" in detail or "1002" in detail: + return ( + "Accessibility permission is missing. Open System Settings " + "> Privacy & Security > Accessibility and enable the " + "terminal running codewhale, then try again." + ) + return f"Click failed: {detail or code}" + return None + + +def linux_type_backend() -> str | None: + if need_tool("ydotool") is not None: + return "ydotool" + if need_tool("wtype") is not None: + return "wtype" + return None + + +def do_click(args: dict) -> dict: + try: + x, y = int(args["x"]), int(args["y"]) + except (KeyError, TypeError, ValueError): + return err("bad_arguments", "click needs integer x and y pixel coordinates.") + if x < 0 or y < 0: + return err("bad_arguments", "click coordinates must be non-negative.") + # Coordinates arrive in screenshot space; scale to display pixels. + x, y = round(x * _shot_scale), round(y * _shot_scale) + if _ON_MAC: + problem = mac_click(x, y) + if problem is not None: + reason = "missing_permission" if "permission" in problem else "missing_helper" + if problem.startswith("Click failed"): + reason = "action_failed" + return err(reason, problem) + return ok_text(f"Clicked at {x},{y}.") + if _ON_LINUX: + backend = linux_type_backend() + if backend == "ydotool": + proc = run(["ydotool", "mousemove", str(x), str(y), "click", "1"]) + elif backend == "wtype": + return err( + "missing_helper", + "wtype moves no mouse. Install ydotool for clicking on Linux.", + ) + else: + return err( + "missing_helper", + "No input helper found on Linux. Install ydotool (X11/Wayland) " + "or wtype (Wayland typing only).", + ) + if proc.returncode != 0: + return err("action_failed", f"Click failed: {proc.stderr.decode()[:200]}") + return ok_text(f"Clicked at {x},{y}.") + return err( + "unsupported_platform", + f"Clicking is supported on macOS and Linux; this host is {platform.system()}.", + ) + + +def do_type_text(args: dict) -> dict: + text = args.get("text", "") + if not isinstance(text, str) or not text: + return err("bad_arguments", "type_text needs a non-empty text string.") + if len(text) > MAX_TYPE_CHARS: + return err( + "bad_arguments", + f"type_text accepts at most {MAX_TYPE_CHARS} characters per call.", + ) + if _ON_MAC: + if need_tool("osascript") is None: + return err("missing_helper", "Missing helper: osascript is not on PATH.") + escaped = text.replace("\\", "\\\\").replace('"', '\\"') + code, detail = osascript( + f'tell application "System Events" to keystroke "{escaped}"' + ) + if code != 0: + if "not permitted" in detail or "1002" in detail: + return err( + "missing_permission", + "Accessibility permission is missing. Open System Settings " + "> Privacy & Security > Accessibility and enable the " + "terminal running codewhale, then try again.", + ) + return err("action_failed", f"Typing failed: {detail or code}") + return ok_text(f"Typed {len(text)} characters.") + if _ON_LINUX: + backend = linux_type_backend() + if backend == "ydotool": + proc = run(["ydotool", "type", "--", text]) + elif backend == "wtype": + proc = run(["wtype", "--", text]) + else: + return err( + "missing_helper", + "No input helper found on Linux. Install ydotool or wtype.", + ) + if proc.returncode != 0: + return err("action_failed", f"Typing failed: {proc.stderr.decode()[:200]}") + return ok_text(f"Typed {len(text)} characters.") + return err( + "unsupported_platform", + f"Typing is supported on macOS and Linux; this host is {platform.system()}.", + ) + + +MAC_KEYS = { + "return": "keystroke return", + "enter": "keystroke return", + "tab": "keystroke tab", + "escape": "key code 53", + "esc": "key code 53", + "space": "keystroke space", + "delete": "key code 51", + "up": "key code 126", + "down": "key code 125", + "left": "key code 123", + "right": "key code 124", +} + + +def do_press_key(args: dict) -> dict: + key = str(args.get("key", "")).strip().lower() + if not key: + return err("bad_arguments", "press_key needs a key name such as return or tab.") + if _ON_MAC: + if key not in MAC_KEYS: + return err( + "bad_arguments", + f"Unsupported key '{key}'. Supported: " + + ", ".join(sorted(MAC_KEYS)), + ) + code, detail = osascript( + f'tell application "System Events" to {MAC_KEYS[key]}' + ) + if code != 0: + return err("action_failed", f"Key press failed: {detail or code}") + return ok_text(f"Pressed {key}.") + if _ON_LINUX: + if need_tool("ydotool") is None: + return err("missing_helper", "press_key on Linux needs ydotool.") + proc = run(["ydotool", "key", key]) + if proc.returncode != 0: + return err("action_failed", f"Key press failed: {proc.stderr.decode()[:200]}") + return ok_text(f"Pressed {key}.") + return err( + "unsupported_platform", + f"Key presses are supported on macOS and Linux; this host is {platform.system()}.", + ) + + +def do_scroll(args: dict) -> dict: + if _ON_LINUX and need_tool("ydotool") is not None: + try: + dx, dy = int(args.get("dx", 0)), int(args.get("dy", 0)) + except (TypeError, ValueError): + return err("bad_arguments", "scroll needs integer dx/dy.") + proc = run(["ydotool", "mousemove", "--", str(dx), str(dy)]) + if proc.returncode != 0: + return err("action_failed", "Scroll failed.") + return ok_text(f"Scrolled by {dx},{dy}.") + if _ON_MAC: + return err( + "unsupported_platform", + "Scrolling is not implemented on macOS in computer-use 0.1.0. " + "Use press_key (space, up, down) to move through scrollable content.", + ) + return err( + "unsupported_platform", + f"Scrolling is supported on Linux with ydotool; this host is {platform.system()}.", + ) + + +TOOLS = { + "screen_size": { + "description": "Report the main display size in pixels. Call before clicking.", + "inputSchema": {"type": "object", "properties": {}}, + }, + "screenshot": { + "description": "Capture the main display as a PNG vision block.", + "inputSchema": {"type": "object", "properties": {}}, + }, + "click": { + "description": "Click at screenshot pixels from the top-left (server scales to the display).", + "inputSchema": { + "type": "object", + "properties": { + "x": {"type": "integer"}, + "y": {"type": "integer"}, + }, + "required": ["x", "y"], + }, + }, + "type_text": { + "description": "Type text into the focused field. Never secrets.", + "inputSchema": { + "type": "object", + "properties": {"text": {"type": "string"}}, + "required": ["text"], + }, + }, + "press_key": { + "description": "Press one named key: return, tab, escape, space, delete, up, down, left, right.", + "inputSchema": { + "type": "object", + "properties": {"key": {"type": "string"}}, + "required": ["key"], + }, + }, + "scroll": { + "description": "Scroll by dx/dy (Linux with ydotool only).", + "inputSchema": { + "type": "object", + "properties": { + "dx": {"type": "integer"}, + "dy": {"type": "integer"}, + }, + }, + }, +} + +HANDLERS = { + "screen_size": lambda args: do_screen_size(), + "screenshot": lambda args: do_screenshot(), + "click": do_click, + "type_text": do_type_text, + "press_key": do_press_key, + "scroll": do_scroll, +} + + +def handle(msg: dict): + msg_id = msg.get("id") + method = msg.get("method") + + def reply(result: dict) -> dict: + return {"jsonrpc": "2.0", "id": msg_id, "result": result} + + def error(code: int, message: str) -> dict: + return { + "jsonrpc": "2.0", + "id": msg_id, + "error": {"code": code, "message": message}, + } + + if method == "initialize": + return reply( + { + "protocolVersion": PROTOCOL_VERSION, + "serverInfo": {"name": SERVER_NAME, "version": SERVER_VERSION}, + "capabilities": {"tools": {}}, + } + ) + if method in ("notifications/initialized", "notifications/cancelled"): + return None + if method == "ping": + return reply({}) + if method == "tools/list": + return reply( + { + "tools": [ + { + "name": name, + "description": spec["description"], + "inputSchema": spec["inputSchema"], + } + for name, spec in TOOLS.items() + ] + } + ) + if method == "tools/call": + params = msg.get("params", {}) or {} + name = params.get("name", "") + args = params.get("arguments", {}) or {} + handler = HANDLERS.get(name) + if handler is None: + return error(-32602, f"Unknown tool '{name}'.") + try: + return reply(handler(args)) + except subprocess.TimeoutExpired: + return reply(err("timed_out", f"Tool '{name}' timed out.")) + except Exception as exc: # never break the wire + log(f"tool {name} crashed: {exc}") + return reply(err("internal", f"Tool '{name}' failed unexpectedly.")) + return error(-32601, f"Unsupported method '{method}'.") + + +def main() -> None: + for line in sys.stdin: + line = line.strip() + if not line: + continue + try: + msg = json.loads(line) + except json.JSONDecodeError: + continue + try: + response = handle(msg if isinstance(msg, dict) else {}) + except Exception as exc: # never break the wire + log(f"dispatch crashed: {exc}") + continue + if response is not None: + sys.stdout.write(json.dumps(response) + "\n") + sys.stdout.flush() + + +if __name__ == "__main__": + main() diff --git a/plugins/computer-use/server/test_server.py b/plugins/computer-use/server/test_server.py new file mode 100644 index 0000000000..76bc226fe3 --- /dev/null +++ b/plugins/computer-use/server/test_server.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""Protocol tests for the computer-use MCP server. No display needed: +failure paths and the JSON-RPC surface are pure; only screenshot and +screen_size touch the host, and the suite skips those without one.""" + +import json +import os +import subprocess +import sys +import unittest + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import server + + +def call(method, params=None, msg_id=1): + msg = {"jsonrpc": "2.0", "id": msg_id, "method": method} + if params is not None: + msg["params"] = params + return server.handle(msg) + + +class HandshakeTest(unittest.TestCase): + def test_initialize_advertises_tools(self): + res = call("initialize", {"protocolVersion": "2024-11-05"}) + result = res["result"] + self.assertEqual(result["protocolVersion"], "2024-11-05") + self.assertIn("tools", result["capabilities"]) + + def test_initialized_notification_is_silent(self): + self.assertIsNone( + server.handle({"jsonrpc": "2.0", "method": "notifications/initialized"}) + ) + + def test_tools_list_names(self): + res = call("tools/list") + names = sorted(t["name"] for t in res["result"]["tools"]) + self.assertEqual( + names, + ["click", "press_key", "screen_size", "screenshot", "scroll", "type_text"], + ) + for tool in res["result"]["tools"]: + self.assertIn("inputSchema", tool) + + def test_unknown_method(self): + res = call("resources/list") + self.assertEqual(res["error"]["code"], -32601) + + def test_unknown_tool(self): + res = call("tools/call", {"name": "nope", "arguments": {}}) + self.assertEqual(res["error"]["code"], -32602) + + +class ValidationTest(unittest.TestCase): + def test_click_rejects_bad_coords(self): + for args in ({"x": -1, "y": 5}, {"x": "a", "y": 1}, {}): + res = call("tools/call", {"name": "click", "arguments": args}) + self.assertTrue(res["result"]["isError"], args) + self.assertEqual(res["result"]["_meta"]["reason"], "bad_arguments") + + def test_type_rejects_empty_and_huge(self): + for args in ({"text": ""}, {}, {"text": "x" * 2001}): + res = call("tools/call", {"name": "type_text", "arguments": args}) + self.assertTrue(res["result"]["isError"], str(args)[:20]) + + def test_press_key_allowlist(self): + res = call("tools/call", {"name": "press_key", "arguments": {"key": ""}}) + self.assertTrue(res["result"]["isError"]) + if server._ON_MAC: + res = call( + "tools/call", {"name": "press_key", "arguments": {"key": "f13"}} + ) + self.assertEqual(res["result"]["_meta"]["reason"], "bad_arguments") + + +class LiveWireTest(unittest.TestCase): + def test_stdio_round_trip(self): + proc = subprocess.run( + [sys.executable, os.path.join(os.path.dirname(__file__), "server.py")], + input=( + '{"jsonrpc":"2.0","id":1,"method":"initialize",' + '"params":{"protocolVersion":"2024-11-05"}}\n' + '{"jsonrpc":"2.0","id":2,"method":"tools/list"}\n' + ), + capture_output=True, + text=True, + timeout=30, + ) + lines = [json.loads(line) for line in proc.stdout.splitlines() if line.strip()] + self.assertEqual(len(lines), 2) + self.assertEqual(lines[0]["result"]["protocolVersion"], "2024-11-05") + self.assertEqual(len(lines[1]["result"]["tools"]), 6) + + +if __name__ == "__main__": + unittest.main() diff --git a/plugins/computer-use/skills/computer-use/SKILL.md b/plugins/computer-use/skills/computer-use/SKILL.md new file mode 100644 index 0000000000..81cee0566a --- /dev/null +++ b/plugins/computer-use/skills/computer-use/SKILL.md @@ -0,0 +1,38 @@ +--- +name: computer-use +description: See the screen and operate it. Use when the user asks to click, type, or check something visually on this machine. +--- + +# Computer use + +You can see this machine's screen and operate its mouse and keyboard +through the `computer` MCP server. Work in a tight look-act-verify +loop and never act blind. + +## The loop + +1. **Look**: call `mcp_computer_screenshot` first. The image arrives as + a vision block; read positions from it, never guess coordinates. +2. **Act**: exactly one action per turn — `mcp_computer_click`, + `mcp_computer_type_text`, `mcp_computer_press_key`, or + `mcp_computer_scroll`. Coordinates are pixels in the latest + screenshot from the top-left, matching `mcp_computer_screen_size`; + the server scales them to the display. +3. **Verify**: screenshot again after every action that should change + something visible. If the screen did not change as expected, stop + and report instead of retrying harder. + +## Safety rules + +- Every action tool asks for approval unless the session runs in + Always-approve posture. Say what you are about to click or type + before the approval prompt appears. +- Never type passwords, secrets, or one-time codes. If a login screen + appears, stop and hand control back to the user. +- Never dismiss system permission dialogs (Accessibility, Screen + Recording) yourself. If the server reports a missing permission, + tell the user exactly which toggle to flip and stop. +- Prefer keyboard over mouse (Tab, arrows, Enter) — it is precise and + does not depend on pixel coordinates. +- If a tool reports `unsupported_platform` or `missing_helper`, stop + and relay the server's message verbatim.