diff --git a/src/agent_env/drivers/README.md b/src/agent_env/drivers/README.md index b24c86cd6..51f57f3ba 100644 --- a/src/agent_env/drivers/README.md +++ b/src/agent_env/drivers/README.md @@ -125,5 +125,30 @@ cuts a server's instructions at 4,096 characters, so the skill itself isn't in t (`DO_NOT_TRACK`). `computer-ready` checks its version, that it prints the pinned skill, and the TCC permissions. Name it as `macos-harness`. +`envs/codex-computer-use/` is OpenAI's Codex Computer Use as a driver, for the same macOS computers. It is the ChatGPT +app's `unified-computer-use` plugin, whose MCP server, `cua_repl`, runs the JavaScript an agent sends against OpenAI's +`cua` API, which reads apps' accessibility trees and acts on them in the background through the plugin's helper app. The +ChatGPT version `driver.py` pins as `VERSION`, OpenAI's versioned download checked by sha256, is unpacked beside the +driver by `install.sh` and never run, so it never updates itself, and its helper app is copied to +`/opt/agentenv/codex-computer-use/`, where `[install.macos]` grants it Accessibility, Post Event and Screen Recording, +as it grants the driver's own Python. The driver starts `cua_repl` as the app would, on the app's own Node with the +environment the app builds and the computer surface alone, one server for each agent session, since the REPL keeps its +bindings between calls; one that stops is started afresh on the next call, and nothing says when a session ends, so one +left unused for 30 minutes, or past the eight kept, is closed, the least recently used first. Agents get the server's +instructions and its `js` and `js_reset` tools, as `mcp.json` pins them, `js` marked as acting on apps where Codex marks +it read-only; `js_add_node_module_dir` and `turn_ended`, which it serves the ChatGPT app, are hidden. Codex asks its +host before each action on an app, and the driver answers: it approves the apps its `allowed_apps` setting names, by +name or bundle identifier, case aside, as the request's metadata or its message gives them, or every app when it names +none, and nothing else, recording the computer's audio included, a request that names no app logged as a warning. Each +call runs under Codex's own sandbox, read-only but for `~/.codex` and the temporary folders and with no network, unless +its `js_sandbox` setting is off; the apps it drives are outside it, and it still lets the JavaScript read files and run +commands, so a bench that wants the GUI alone guards for that. No ChatGPT sign-in is needed. Its `computer_*` tools are +agent-desktop's, from `MacOSDriver`. `computer-ready` checks the executables' sha256, that `cua_repl` gives the +instructions and lists the tools `mcp.json` pins and answers `cua.getState()`, and the TCC permissions. To move it to +another ChatGPT version, set `VERSION`, pin the zip's URL and sha256 in `env.toml` and the executables' in `driver.py`'s +`EXECUTABLES` (a unit test checks that `env.toml`, `install.sh` and `driver.py` agree on the version and the helper's +path), and take `mcp.json` again from what that version's `cua_repl` says at connect and lists. Name it as +`codex-computer-use`. + This bundle has no tasks of its own: other bundles pull its drivers by alias, and `agent-env run drivers` has nothing to run. diff --git a/src/agent_env/drivers/aliases.toml b/src/agent_env/drivers/aliases.toml index f53cc16c2..a93822b44 100644 --- a/src/agent_env/drivers/aliases.toml +++ b/src/agent_env/drivers/aliases.toml @@ -5,3 +5,4 @@ cua-driver = ["cua-driver"] page = ["page"] agent-desktop = ["agent-desktop"] macos-harness = ["macos-harness"] +codex-computer-use = ["codex-computer-use"] diff --git a/src/agent_env/drivers/envs/codex-computer-use/driver.py b/src/agent_env/drivers/envs/codex-computer-use/driver.py new file mode 100644 index 000000000..ea6773c42 --- /dev/null +++ b/src/agent_env/drivers/envs/codex-computer-use/driver.py @@ -0,0 +1,352 @@ +"""OpenAI's Codex Computer Use as a computer driver on macOS: the ChatGPT app's ``unified-computer-use`` plugin, whose +MCP server, ``cua_repl``, runs the JavaScript an agent sends against OpenAI's ``cua`` API, which reads apps' +accessibility trees and acts on them in the background through the plugin's helper app. The pinned app is unpacked +beside the driver and never run, so it never updates itself; the driver starts ``cua_repl`` as the app would, on the +app's own Node with the environment the app builds, one server for each agent session, since the REPL keeps its +bindings between calls, closed once it's left unused or past the most kept, since nothing says when a session ends. +Agents get the server's instructions and its ``js`` and ``js_reset`` tools as pinned (``mcp.json``), ``js`` marked as +acting on apps where Codex marks it read-only; the two it serves only the ChatGPT app are hidden. Codex asks its host +before each action on an app, and the driver answers: it approves the apps the ``allowed_apps`` setting names, by name +or bundle identifier, or every app when it names none, and nothing else, recording the computer's audio included. Each +call runs under Codex's own sandbox, read-only but for Codex's folder and the temporary ones and with no network, unless +the ``js_sandbox`` setting is off; the apps it drives are outside it. No ChatGPT sign-in is needed. The computer/v1 +operations are MacOSDriver's; this process's Python is granted Accessibility, Post Event and Screen Recording, as the +helper app is. It needs ``agentenv-framework-protocol[computer,macos]`` and the pinned ChatGPT app.""" +from __future__ import annotations + +import asyncio +import contextlib +import functools +import hashlib +import json +import logging +import os +import re +import time +from pathlib import Path +from typing import Any, AsyncIterator, Callable + +from mcp import ClientSession, McpError, StdioServerParameters +from mcp.client.stdio import stdio_client +from mcp.shared.context import RequestContext +from mcp.types import (CONNECTION_CLOSED, CallToolResult, ElicitRequestParams, ElicitResult, InitializeResult, + TextContent) + +from agentenv_protocol import EnvironmentTool, ForwardedTool, ToolContext, server_instructions +from agentenv_protocol.computer import ComputerSettings, MacOSDriver, ReadyCheck, computer_driver + +logger = logging.getLogger(__name__) + +HERE = Path(__file__).resolve().parent +#: The pinned ChatGPT app, as install.sh unpacks it, and where in it what starts Codex's REPL is. +VERSION = "26.930.51102" +APP = HERE / "ChatGPT.app" +CUA_NODE = Path("Contents", "Resources", "cua_node") +NODE = CUA_NODE / "bin" / "node" +NODE_REPL = CUA_NODE / "bin" / "node_repl" +NODE_MODULES = CUA_NODE / "lib" / "node_modules" +LAUNCHER = NODE_MODULES / "@oai" / "cua-repl" / "bin" / "cua-repl.mjs" +CODEX_CLI = Path("Contents", "Resources", "codex-cli", "CodexCLI.app", "Contents", "MacOS", "codex") +#: Codex Computer Use's helper app, where install.sh copies it and the layer's grant names it. +HELPER = Path("/opt/agentenv/codex-computer-use/Codex Computer Use.app") +#: The sha256 of each executable at the pinned version, by its path under the app or the helper. +EXECUTABLES = { + "ChatGPT.app/Contents/MacOS/ChatGPT": "1f2989ac3d702e48c740fe4c54b1b6279f9e48e0e4c0ead7d4501464e2ea976e", + "ChatGPT.app/Contents/Resources/cua_node/bin/node": + "59f603e9e6d81c8b6bdaf6bb35b1990e7b8048762ca07a524ac7e87ade6fea15", + "ChatGPT.app/Contents/Resources/cua_node/bin/node_repl": + "bbd1640851593a724e2a6bc558f3a16078f40b09d4fe76f2f6d89f8b895eb5ed", + "Codex Computer Use.app/Contents/MacOS/SkyComputerUseService": + "fa5b5d685f550af82902cff1a345db558a1da3eb526cf78cd4659fa0dcde01a5", +} +#: What ``cua_repl`` says at connect and lists at the pinned version: its instructions and its tools. +MANIFEST = HERE / "mcp.json" +#: The tools agents get; the others are the ChatGPT app's own. +FORWARDED = ("js", "js_reset") +#: The hints agents get where Codex's own undersell a tool: ``js`` acts on apps, and through them on anything. +ANNOTATIONS = {"js": {"readOnlyHint": False, "destructiveHint": True, "openWorldHint": True}} +#: How long a session's REPL is kept unused before it's closed, since nothing says when a session ends; how many are +#: kept at most, the least recently used closed first; and how often the idle ones are looked for. +IDLE_S, MAX_REPLS, SWEEP_S = 1800, 8, 60 +#: How long a closed REPL's server has to end before it's cancelled. +CLOSE_TIMEOUT_S = 10 +#: What of the driver's environment ``cua_repl`` gets: the session user's own; none of a deploy's values. +KEPT = ("HOME", "USER", "LOGNAME", "TMPDIR", "LANG") +#: The PATH it runs, and launches apps, with. +PATH = "/usr/bin:/bin:/usr/sbin:/sbin" +#: The JavaScript the readiness check runs: an inventory, which needs no app's approval. +PROBE = "await cua.getState({ emit: false }); nodeRepl.write('ready');" +#: The message of the approval Codex asks of its host before it acts on an app, which names it as the request's +#: metadata does. +APPROVAL =re.compile(r'Allow (?:Computer Use|Codex) to use "?(?P[^"?]+)"?\?') +#: The key of the metadata Codex reads its sandbox from on each call. +SANDBOX_KEY = "codex/sandbox-state-meta" + + +@computer_driver(name="codex-computer-use", + supports=[{"os": {"type": "macos", "display": ["aqua"]}, "arch": ["arm64"]}]) +class CodexComputerUse(MacOSDriver): + class Settings(ComputerSettings): + #: The apps Codex may act on, by name or bundle identifier, case aside; None for every app. + allowed_apps: list[str] | None = None + js_sandbox: bool = True + + def __init__(self) -> None: + super().__init__() + self._pinned: list[ReadyCheck] | None = None + #: Each session's REPL, the least recently used first. + self._repls: dict[str | None, _Repl] = {} + self._sweeper: asyncio.Task | None = None + #: The closes under way, held so they aren't collected before they end. + self._closing: set[asyncio.Task] = set() + + def forwarded_tools(self) -> list[ForwardedTool]: + return [ForwardedTool(EnvironmentTool(name=t["name"], description=t["description"], + inputSchema=t["inputSchema"]), + functools.partial(self._forward, t["name"]), + {**(t.get("annotations") or {}), **ANNOTATIONS.get(t["name"], {})}) + for t in _manifest()["tools"] if t["name"] in FORWARDED] + + @server_instructions + def codex_instructions(self) -> str: + return _manifest()["instructions"] + + async def ready_checks(self) -> list[ReadyCheck]: + """The pinned executables, the tools ``cua_repl`` lists and that it answers, checked until they pass once, and + MacOSDriver's TCC permissions every time.""" + if self._pinned is None or not all(check.ok for check in self._pinned): + self._pinned = await self._pinned_checks() + return [*self._pinned, *await super().ready_checks()] + + async def _forward(self, tool: str, /, **arguments: Any) -> CallToolResult: + """A call to one of Codex's tools on the caller's own REPL, answered as Codex answers it. Other sessions' REPLs + left idle, or past the most kept, are closed. Nothing here awaits before the call is queued, so calls a session + makes together find one REPL.""" + repl = self._repls.pop(caller := ToolContext.current().caller.session, None) or _Repl(self._server) + self._evict(keep=MAX_REPLS - 1) + self._repls[caller] = repl + if self._sweeper is None or self._sweeper.done(): + self._sweeper = asyncio.create_task(self._sweep()) + try: + return await repl.call(tool, arguments, self._sandbox()) + except Exception as e: + return CallToolResult(isError=True, content=[TextContent(type="text", text=( + f"Codex's cua_repl failed before it answered, so {tool} may have run: {type(e).__name__}: {e}"))]) + + def _evict(self, keep: int = MAX_REPLS) -> None: + """Take out the REPLs idle for ``IDLE_S`` and, past ``keep``, the least recently used, none of them in a call, + and close them in the background.""" + now = time.monotonic() + free = [caller for caller, repl in self._repls.items() if not repl.busy] + idle = [caller for caller in free if now - self._repls[caller].used > IDLE_S] + over = max(0, len(self._repls) - len(idle) - keep) + spare = [caller for caller in free if caller not in idle][:over] + for caller in [*idle, *spare]: + closing = asyncio.create_task(self._repls.pop(caller).close()) + self._closing.add(closing) + closing.add_done_callback(self._closing.discard) + + async def _sweep(self) -> None: + """Close idle REPLs every ``SWEEP_S`` while there are any.""" + while self._repls: + await asyncio.sleep(SWEEP_S) + self._evict() + + @contextlib.asynccontextmanager + async def _server(self) -> AsyncIterator[tuple[ClientSession, InitializeResult]]: + """``cua_repl`` connected and initialized, with what it said at connect, its approval requests answered; it + ends with the block.""" + server = StdioServerParameters(command=str(APP / NODE), args=[str(APP / LAUNCHER)], env=_environment(), + cwd=str(_workspace())) + async with (stdio_client(server) as (read, write), + ClientSession(read, write, elicitation_callback=self._approve) as session): + yield session, await session.initialize() + + async def _approve(self, context: RequestContext, params: ElicitRequestParams) -> ElicitResult: + """Codex's request to act on an app, approved when ``allowed_apps`` names the app or names none; anything else + it asks, to record the computer's audio say, is declined.""" + message = getattr(params, "message", "") + names = _app_names(message, params.meta.model_dump() if params.meta else {}) + allowed = self.settings.allowed_apps + approved = bool(names) and (allowed is None or bool(names & {app.lower() for app in allowed})) + if names: + logger.info("%s %r", "approved" if approved else "declined", message) + else: + logger.warning("declined %r, which names no app", message) + return ElicitResult(action="accept", content={}) if approved else ElicitResult(action="decline") + + def _sandbox(self) -> dict: + """The metadata that sets the sandbox Codex runs the call's JavaScript in.""" + profile = _read_only() if self.settings.js_sandbox else {"type": "disabled"} + return {SANDBOX_KEY: {"sandboxCwd": _workspace().as_uri(), "permissionProfile": profile}} + + async def _pinned_checks(self) -> list[ReadyCheck]: + digests = await asyncio.to_thread(_digests) + differ = sorted(path for path, digest in digests.items() if digest != EXECUTABLES[path]) + checks = [ReadyCheck("Codex Computer Use", not differ, f"ChatGPT {VERSION}, as pinned" if not differ else + f"these differ from the pinned ChatGPT {VERSION}: {', '.join(differ)}")] + if differ: + return checks + try: + async with self._server() as (session, initialized): + listed = (await session.list_tools()).tools + probed = await session.call_tool("js", {"code": PROBE}, meta=self._sandbox()) + except Exception as e: + return [*checks, ReadyCheck("Codex tools", False, f"cua_repl didn't answer: {type(e).__name__}: {e}")] + running = {t.name: t.model_dump(mode="json", exclude_none=True) for t in listed} + pinned = {t["name"]: t for t in _manifest()["tools"]} + changed = ["instructions"] if initialized.instructions != _manifest()["instructions"] else [] + changed += sorted(name for name in running.keys() | pinned.keys() if name not in running or name not in pinned + or any(running[name].get(key) != value for key, value in pinned[name].items())) + answer = " ".join(c.text for c in probed.content if isinstance(c, TextContent)) + return [*checks, + ReadyCheck("Codex tools", not changed, f"its instructions and {len(running)} tools, as pinned" + if not changed else f"these differ from {MANIFEST.name}: {', '.join(changed)}"), + ReadyCheck("Codex answers", not probed.isError and "ready" in answer, + "it read the computer's apps" if not probed.isError else f"cua.getState() failed: {answer}")] + + +class _Repl: + """A session's ``cua_repl``, one server at a time, each held open by a task of its own, since a stdio client is + entered and left in one task, and reached from any through that server's queue, one call at a time. A server that + stops is let go at once, while it still ends, and the next call starts one afresh, without the last one's bindings; + ``close`` ends them for good.""" + + def __init__(self, open_server: Callable[[], contextlib.AbstractAsyncContextManager[ + tuple[ClientSession, InitializeResult]]]) -> None: + self._open = open_server + #: The queue of the server taking calls, None when none is; and the tasks of every server not yet ended. + self._calls: asyncio.Queue | None = None + self._serving: set[asyncio.Task] = set() + #: When a call last ended, on the monotonic clock, and how many are queued or running. + self.used, self.busy = time.monotonic(), 0 + + async def call(self, tool: str, arguments: dict, meta: dict) -> CallToolResult: + if self._calls is None: + self._calls = asyncio.Queue() + serving = asyncio.create_task(self._serve(self._calls)) + self._serving.add(serving) + serving.add_done_callback(self._serving.discard) + answer = asyncio.get_running_loop().create_future() + self.busy += 1 + try: + self._calls.put_nowait((tool, arguments, meta, answer)) + return await answer + finally: + self.busy -= 1 + self.used = time.monotonic() + + async def close(self) -> None: + """Every server ended, its process with it, waiting at most ``CLOSE_TIMEOUT_S`` before cancelling one.""" + if self._calls is not None: + self._calls.put_nowait(None) + self._calls = None + if serving := set(self._serving): + _, late = await asyncio.wait(serving, timeout=CLOSE_TIMEOUT_S) + for task in late: + task.cancel() + + async def _serve(self, calls: asyncio.Queue) -> None: + stopped: BaseException = ConnectionError("cua_repl stopped") + try: + async with self._open() as (session, _): + try: + while (call := await calls.get()) is not None: + tool, arguments, meta, answer = call + try: + result = await session.call_tool(tool, arguments, meta=meta) + except McpError as e: + _settle(answer, error=e) + if e.error.code == CONNECTION_CLOSED: + raise + continue + except Exception as e: + _settle(answer, error=e) + raise + _settle(answer, result=result) + finally: + self._let_go(calls) + except Exception as e: + stopped = e + finally: + self._let_go(calls) + while not calls.empty(): + if (call := calls.get_nowait()) is not None: + _settle(call[-1], error=stopped) + + def _let_go(self, calls: asyncio.Queue) -> None: + """The next call to a new server, not to this one, which takes no more.""" + if self._calls is calls: + self._calls = None + + +def _settle(answer: asyncio.Future, *, result: Any = None, error: BaseException | None = None) -> None: + if answer.done(): + return + if error is not None: + answer.set_exception(error) + else: + answer.set_result(result) + + +@functools.cache +def _manifest() -> dict: + return json.loads(MANIFEST.read_text()) + + +def _app_names(message: str, meta: dict) -> set[str]: + """The app an approval request is for, case aside: its bundle identifier and name as the request's metadata gives + them, and the name its message quotes; none for a request that isn't for an app.""" + shown = [field.get("value") for field in meta.get("tool_params_display") or [] if field.get("name") == "app"] + named = APPROVAL.fullmatch(message.strip()) + names = [(meta.get("tool_params") or {}).get("app"), *shown, named and named["app"]] + return {str(name).strip().lower() for name in names if name} + + +def _codex_home() -> Path: + return Path.home() / ".codex" + + +def _read_only() -> dict: + """Codex's managed sandbox profile: everything readable, only Codex's folder and the temporary ones writable, and + no network.""" + special = [{"path": {"type": "special", "value": {"kind": kind}}, "access": access} + for kind, access in (("root", "read"), ("tmpdir", "write"), ("slash_tmp", "write"))] + codex = {"path": {"type": "path", "path": str(_codex_home())}, "access": "write"} + return {"type": "managed", "network": "restricted", + "file_system": {"type": "restricted", "entries": [*special, codex]}} + + +def _workspace() -> Path: + """The folder ``cua_repl`` runs in, Codex's sandbox's working folder, inside the one it may write.""" + folder = _codex_home() / "agentenv" + folder.mkdir(parents=True, exist_ok=True) + return folder + + +def _environment() -> dict[str, str]: + """``cua_repl``'s environment, as the ChatGPT app builds it for the plugin, with the computer surface alone.""" + codex, modules = str(_codex_home()), str(APP / NODE_MODULES) + return {**{name: os.environ[name] for name in KEPT if name in os.environ}, "PATH": PATH, + "CODEX_HOME": codex, + "CODEX_CLI_PATH": str(APP / CODEX_CLI), + "CUA_REPL_ENABLED_SURFACES": "computer", + "CUA_REPL_NODE_REPL_PATH": str(APP / NODE_REPL), + "NODE_REPL_INSTRUCTIONS_USE_CASE_COMPUTER_USE": "Control desktop apps on macOS through Computer Use.", + "NODE_REPL_NATIVE_PIPE_CONNECT_TIMEOUT_MS": "1000", + "NODE_REPL_NODE_MODULE_DIRS": modules, + "NODE_REPL_NODE_PATH": str(APP / NODE), + "NODE_REPL_TRUSTED_CODE_PATHS": f"{codex}:{modules}", + "NODE_REPL_TRUSTED_SERVICES": json.dumps({"sky": "@oai/sky/service"}), + "SKY_CUA_SERVICE_PATH": str(HELPER)} + + +def _digests() -> dict[str, str | None]: + """Each pinned executable's sha256, None for one that isn't there.""" + roots = {"ChatGPT.app": APP, "Codex Computer Use.app": HELPER} + found = {} + for path in EXECUTABLES: + root, _, rest = path.partition("/") + file = roots[root] / rest + found[path] = hashlib.sha256(file.read_bytes()).hexdigest() if file.is_file() else None + return found diff --git a/src/agent_env/drivers/envs/codex-computer-use/env.toml b/src/agent_env/drivers/envs/codex-computer-use/env.toml new file mode 100644 index 000000000..fa5622d76 --- /dev/null +++ b/src/agent_env/drivers/envs/codex-computer-use/env.toml @@ -0,0 +1,15 @@ +type = "computer_driver" +module = "driver.py:CodexComputerUse" +port = 18765 + +[install.macos] +python = "requirements-macos.txt" +downloads = [ + { url = "https://persistent.oaistatic.com/codex-app-prod/ChatGPT-darwin-arm64-26.930.51102.zip", sha256 = "7001df293fd478b2a4e2d8b5f4ea3aae43a2e21d168960256b2f31ac4ee43f6f", to = "ChatGPT.zip", arch = "arm64" }, +] +run = [{ script = "install.sh", scope = "session" }] +grants = [ + { executable = "venv/bin/python", services = ["accessibility", "post_event", "screen_capture"] }, + { app = "/opt/agentenv/codex-computer-use/Codex Computer Use.app", services = ["accessibility", "post_event", "screen_capture"] }, +] +services = [{ name = "driver", command = "venv/bin/python -m agentenv_protocol.computer.serve driver.py:CodexComputerUse" }] diff --git a/src/agent_env/drivers/envs/codex-computer-use/install.sh b/src/agent_env/drivers/envs/codex-computer-use/install.sh new file mode 100755 index 000000000..ecdd32d1b --- /dev/null +++ b/src/agent_env/drivers/envs/codex-computer-use/install.sh @@ -0,0 +1,13 @@ +#!/bin/sh +# The pinned ChatGPT app, from the zip the install downloaded and checked, unpacked beside the driver and never run, so it +# never updates itself; and Codex Computer Use's helper app copied out of it to where the layer's grant names it, a path +# that doesn't depend on this folder's name. As the session user, whom the helper must belong to. +set -eu +ditto -x -k ChatGPT.zip . +rm -f ChatGPT.zip +helper="Codex Computer Use.app" +dest=/opt/agentenv/codex-computer-use +mkdir -p "$dest" +rm -rf "$dest/$helper" +ditto "ChatGPT.app/Contents/Resources/cua_node/lib/node_modules/@oai/sky/$helper" "$dest/$helper" +xattr -dr com.apple.quarantine ChatGPT.app "$dest" || true diff --git a/src/agent_env/drivers/envs/codex-computer-use/mcp.json b/src/agent_env/drivers/envs/codex-computer-use/mcp.json new file mode 100644 index 000000000..a919f71c3 --- /dev/null +++ b/src/agent_env/drivers/envs/codex-computer-use/mcp.json @@ -0,0 +1,105 @@ +{ + "instructions": "UI automation through cua_repl using the initialized cua API.", + "tools": [ + { + "name": "js", + "description": "Control native apps or browsers on the user’s computer by reading or operating UI. Prefer purpose-built skills, connectors, APIs, or CLIs when available.\n\nOn the first invocation of `cua_repl`, or after resetting it, execute exactly one of the API calls shown below, optionally assigning its result to a variable. Do not add other API calls, waits, or snapshots to that invocation.\nThe tool result will include documentation and, when creating or selecting a tab or selecting an app, its initial UI state. Selecting a browser does not open a tab. Read that result before continuing.\nUse only APIs described in the tool instructions or returned documentation.\n\nWhen you need an inventory of available apps, browsers, and tabs, get a snapshot of all enabled surfaces. Otherwise, use the relevant entry point below:\n\n```javascript\nawait cua.getState();\n```\n\nBrowser APIs are disabled.\n\nIf the user specifies an app to use, get the app by name, bundle ID, or path:\n\n```javascript\nlet app = await cua.getApp(\"Example App\");\n```\n\nTo add other content to the tool result, use `nodeRepl.write(value)` for text or other values and `await nodeRepl.emitImage(image)` for images. The APIs listed above already display their documentation or UI state; do not wrap their results in `write` or `emitImage`.\n\nIf your context begins with a summary of an existing computer use task, call `await cua.rewriteDocumentation()` before continuing the computer use task to ensure you have a complete view of the necessary documentation.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "code": { + "description": "JavaScript to execute using the initialized cua_repl runtime.", + "type": "string" + }, + "timeout_ms": { + "description": "Optional execution timeout in milliseconds. Defaults to 30000 (30 seconds) when omitted.", + "minimum": 1, + "type": "integer" + }, + "title": { + "description": "Short user-facing description of what the code does.", + "maxLength": 80, + "minLength": 1, + "type": "string" + } + }, + "required": [ + "code" + ], + "type": "object" + }, + "annotations": { + "readOnlyHint": true, + "destructiveHint": false, + "openWorldHint": false + } + }, + { + "name": "js_add_node_module_dir", + "description": "Add an absolute `node_modules` directory for package imports. The directory remains available after `js_reset`.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "path": { + "description": "Absolute path to a node_modules directory to add to Node package resolution.", + "minLength": 1, + "type": "string" + } + }, + "required": [ + "path" + ], + "type": "object" + }, + "annotations": { + "readOnlyHint": true, + "destructiveHint": false, + "openWorldHint": false + } + }, + { + "name": "js_reset", + "description": "Reset the persistent cua_repl JavaScript session. All JavaScript bindings are discarded. The next cua_repl.js call initializes a fresh runtime for the enabled surfaces. This does not close browser tabs or native apps, or erase their state.", + "inputSchema": { + "additionalProperties": false, + "properties": {}, + "type": "object" + }, + "annotations": { + "readOnlyHint": true, + "destructiveHint": false, + "openWorldHint": false + } + }, + { + "name": "turn_ended", + "description": "Notify trusted libraries that a Codex turn ended. Repeated notifications for the same session and turn are ignored.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "hook_event_name": { + "minLength": 1, + "type": "string" + }, + "session_id": { + "minLength": 1, + "type": "string" + }, + "turn_id": { + "minLength": 1, + "type": "string" + } + }, + "required": [ + "hook_event_name", + "session_id", + "turn_id" + ], + "type": "object" + }, + "annotations": { + "idempotentHint": true + } + } + ] +} diff --git a/src/agent_env/drivers/envs/codex-computer-use/requirements-macos.txt b/src/agent_env/drivers/envs/codex-computer-use/requirements-macos.txt new file mode 100644 index 000000000..839959600 --- /dev/null +++ b/src/agent_env/drivers/envs/codex-computer-use/requirements-macos.txt @@ -0,0 +1,4 @@ +# What a composed macOS computer installs into the driver's venv: the protocol with MacOSDriver, whose screen and input +# need its macos extra, and the MCP client the forwarded tools reach Codex's cua_repl server with. A release pins it to +# the version it ships. The ChatGPT app is the pinned download, whose executables' sha256 the driver checks. +agentenv-framework-protocol[computer,macos]>=0.1.459 diff --git a/tst/unit/bundle/shipped_bundles_test.py b/tst/unit/bundle/shipped_bundles_test.py index 5e069f1d3..357d5708f 100644 --- a/tst/unit/bundle/shipped_bundles_test.py +++ b/tst/unit/bundle/shipped_bundles_test.py @@ -63,6 +63,7 @@ def test_the_shipped_env_aliases_name_the_mail_env_the_validator_deploys_the_dri "cua-driver": "@local/agentenv-framework/drivers/cua-driver", "page": "@local/agentenv-framework/drivers/page", "agent-desktop": "@local/agentenv-framework/drivers/agent-desktop", "macos-harness": "@local/agentenv-framework/drivers/macos-harness", + "codex-computer-use": "@local/agentenv-framework/drivers/codex-computer-use", "chromium": "@local/agentenv-framework/browsers/chromium"} assert A2AAgent.VALIDATION_ENV_ID == "mail" diff --git a/tst/unit/drivers/codex_computer_use_driver_test.py b/tst/unit/drivers/codex_computer_use_driver_test.py new file mode 100644 index 000000000..d7efc5bad --- /dev/null +++ b/tst/unit/drivers/codex_computer_use_driver_test.py @@ -0,0 +1,360 @@ +"""The codex-computer-use driver against a fake cua_repl: agents get its js and js_reset as pinned, with its +instructions, and not the tools it serves only the ChatGPT app; each call carries Codex's read-only sandbox unless the +setting turns it off; Codex's approval requests are answered from allowed_apps by name or bundle identifier, from the +request's metadata or its message, every app approved when it names none, and nothing else; each agent session keeps a +REPL of its own across calls, one that stops starting afresh; cua_repl gets the app's environment and none of the +driver's; readiness checks the pinned executables, what cua_repl says and lists and that it answers; and the driver's +files agree on the pinned version.""" +from __future__ import annotations + +import asyncio +import contextlib +import hashlib +import importlib.util +import json +import os +import sys +import textwrap +import tomllib +from pathlib import Path + +import pytest +from mcp.shared.memory import create_connected_server_and_client_session + +import agent_env +from agentenv_protocol import SESSION_META_KEY +from agentenv_protocol.computer import Platform, _macos + +DRIVER = Path(agent_env.__file__).parent / "drivers" / "envs" / "codex-computer-use" / "driver.py" +FAKE = textwrap.dedent('''\ + import json, os + from pathlib import Path + import anyio + import mcp.types as types + from mcp.server.lowlevel import Server + from mcp.server.stdio import stdio_server + from mcp.shared.message import ServerMessageMetadata + state = Path(os.environ["HOME"]) / "fake" + (state / f"env-{os.getpid()}.json").write_text(json.dumps(dict(os.environ))) + manifest = json.loads((state / "tools.json").read_text()) + server = Server("cua_repl", instructions=manifest["instructions"]) + bindings = {} + + @server.list_tools() + async def tools(): + return [types.Tool(**t) for t in manifest["tools"]] + + async def ask(message, tool=None, app=None, shown=None): + context = server.request_context + meta = types.RequestParams.Meta(**({} if tool is None else { + "codex_approval_kind": "mcp_tool_call", "connector_id": "computer-use", "tool_name": tool, + "tool_params": {"app": app} if app else {}, + "tool_params_display": [{"display_name": "App", "name": "app", "value": shown}] if shown else []})) + result = await context.session.send_request( + types.ServerRequest(types.ElicitRequest(params=types.ElicitRequestFormParams( + message=message, requestedSchema={"type": "object", "properties": {}}, _meta=meta))), + types.ElicitResult, metadata=ServerMessageMetadata(related_request_id=context.request_id)) + return result.action + + @server.call_tool(validate_input=False) + async def call(name, arguments): + context = server.request_context + with open(state / "calls.jsonl", "a") as out: + out.write(json.dumps({"pid": os.getpid(), "name": name, "arguments": arguments, + "meta": context.meta.model_dump() if context.meta else {}}) + "\\n") + code = arguments.get("code", "") + if name == "js_reset": + bindings.clear() + text = "reset" + elif code.startswith("app "): + _, shown, bundle = code.split(" ") + text = await ask(f'Allow Computer Use to use "{shown}"?', "get_app_state", bundle, shown) + elif code.startswith("reworded "): + _, shown, bundle = code.split(" ") + text = await ask(f"Codex would like to control {shown}.", "get_app_state", bundle, shown) + elif code.startswith("bare "): + text = await ask(f'Allow Computer Use to use "{code[5:]}"?') + elif code == "audio": + text = await ask("Allow Computer Use to record computer audio?", "start_audio_recording") + elif code.startswith("set "): + bindings["x"] = code[4:] + text = "set" + elif code == "get": + text = f"{bindings.get('x')} in {os.getpid()}" + elif code == "die": + os._exit(1) + elif "cua.getState" in code: + text = "ready" if not (state / "unready").exists() else "no apps" + else: + text = "ran" + return [types.TextContent(type="text", text=text)] + + async def main(): + async with stdio_server() as (read, write): + await server.run(read, write, server.create_initialization_options()) + + anyio.run(main) +''') +MACOS = Platform(os="macos", version="26.5", arch="arm64", display="aqua") + + +class Granted: + def missing(self) -> list[str]: + return [] + + +@pytest.fixture +def pinned(monkeypatch, tmp_path): + """The driver's module, as it ships.""" + monkeypatch.delenv("AGENTENV_COMPUTER_SETTINGS", raising=False) + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("AGENTENV_CONTROL_TOKEN", "a-deploys-secret") + monkeypatch.setattr(_macos, "Quartz", Granted) + monkeypatch.setattr(sys, "platform", "darwin") + monkeypatch.setattr("agentenv_protocol.computer.driver.detect", lambda: MACOS) + spec = importlib.util.spec_from_file_location("codex_computer_use_driver", DRIVER) + loaded = importlib.util.module_from_spec(spec) + spec.loader.exec_module(loaded) + return loaded + + +@pytest.fixture +def module(pinned, monkeypatch, tmp_path): + """The driver's module on a fake ChatGPT app whose Node is the fake cua_repl.""" + (tmp_path / "fake").mkdir() + (tmp_path / "fake" / "tools.json").write_text(pinned.MANIFEST.read_text()) + app, helper = tmp_path / "ChatGPT.app", tmp_path / "Codex Computer Use.app" + node = app / pinned.NODE + node.parent.mkdir(parents=True) + node.write_text(f"#!{sys.executable}\n{FAKE}") + node.chmod(0o755) + digests = {} + for path in pinned.EXECUTABLES: + root, _, rest = path.partition("/") + file = (app if root == "ChatGPT.app" else helper) / rest + if not file.exists(): + file.parent.mkdir(parents=True, exist_ok=True) + file.write_text(f"the pinned {file.name}") + digests[path] = hashlib.sha256(file.read_bytes()).hexdigest() + monkeypatch.setattr(pinned, "APP", app) + monkeypatch.setattr(pinned, "HELPER", helper) + monkeypatch.setattr(pinned, "EXECUTABLES", digests) + return pinned + + +def _driver(module, monkeypatch, **settings): + monkeypatch.setenv("AGENTENV_COMPUTER_SETTINGS", json.dumps(settings)) + return module.CodexComputerUse() + + +@contextlib.asynccontextmanager +async def mcp_session(driver): + async with create_connected_server_and_client_session(driver.mcp or driver.create_app()) as session: + yield session + + +def _calls(home: Path) -> list[dict]: + path = home / "fake" / "calls.jsonl" + return [json.loads(line) for line in path.read_text().splitlines()] if path.exists() else [] + + +def _text(result) -> str: + return " ".join(c.text for c in result.content if c.type == "text") + + +async def _gone(pid: int) -> bool: + for _ in range(200): + try: + os.kill(pid, 0) + except ProcessLookupError: + return True + await asyncio.sleep(0.01) + return False + + +@pytest.mark.asyncio +async def test_agents_get_codexs_js_and_js_reset_as_pinned_with_its_instructions_and_not_the_apps_own_tools( + module, monkeypatch): + driver = _driver(module, monkeypatch) + async with mcp_session(driver) as session: + listed = {t.name: t for t in (await session.list_tools()).tools} + + pinned = {t["name"]: t for t in module._manifest()["tools"]} + assert {"js", "js_reset"} <= set(listed) and not {"js_add_node_module_dir", "turn_ended"} & set(listed) + assert listed["js"].inputSchema == pinned["js"]["inputSchema"] + assert listed["js"].annotations.readOnlyHint is False and listed["js"].annotations.destructiveHint is True + assert listed["js_reset"].annotations.readOnlyHint is True + assert listed["js"].description == pinned["js"]["description"] + assert driver.mcp.instructions.startswith(module._manifest()["instructions"]) + + +@pytest.mark.asyncio +async def test_each_call_runs_in_codexs_read_only_sandbox_unless_the_setting_turns_it_off( + module, monkeypatch, tmp_path): + async with mcp_session(_driver(module, monkeypatch)) as session: + await session.call_tool("js", {"code": "1 + 1"}) + async with mcp_session(_driver(module, monkeypatch, js_sandbox=False)) as session: + await session.call_tool("js", {"code": "1 + 1"}) + + sandboxed, open_ = (call["meta"]["codex/sandbox-state-meta"] for call in _calls(tmp_path)) + profile = sandboxed["permissionProfile"] + entries = {json.dumps(e["path"], sort_keys=True): e["access"] for e in profile["file_system"]["entries"]} + assert profile["type"] == "managed" and profile["network"] == "restricted" + assert entries == {'{"type": "special", "value": {"kind": "root"}}': "read", + '{"type": "special", "value": {"kind": "tmpdir"}}': "write", + '{"type": "special", "value": {"kind": "slash_tmp"}}': "write", + json.dumps({"path": str(tmp_path / ".codex"), "type": "path"}, sort_keys=True): "write"} + assert sandboxed["sandboxCwd"] == (tmp_path / ".codex" / "agentenv").as_uri() + assert open_["permissionProfile"] == {"type": "disabled"} + + +@pytest.mark.asyncio +async def test_approvals_follow_allowed_apps_by_name_or_bundle_identifier_and_nothing_else_is_approved( + module, monkeypatch, caplog): + async with mcp_session(_driver(module, monkeypatch, allowed_apps=["calculator", "com.apple.TextEdit"])) as session: + async def js(code: str) -> str: + return _text(await session.call_tool("js", {"code": code})) + + named, by_id = await js("app Calculator com.apple.calculator"), await js("app TextEdit com.apple.TextEdit") + other, audio = await js("app Finder com.apple.finder"), await js("audio") + reworded, bare = await js("reworded Calculator com.apple.calc"), await js("bare Calculator") + async with mcp_session(_driver(module, monkeypatch)) as session: + any_app = _text(await session.call_tool("js", {"code": "app Finder com.apple.finder"})) + no_audio = _text(await session.call_tool("js", {"code": "audio"})) + + assert [named, by_id, other, audio] == ["accept", "accept", "decline", "decline"] + assert [reworded, bare] == ["accept", "accept"] + assert [any_app, no_audio] == ["accept", "decline"] + warned = [r.getMessage() for r in caplog.records if r.name == module.logger.name and r.levelname == "WARNING"] + assert len(warned) == 2 and all("record computer audio" in message for message in warned) + + +@pytest.mark.asyncio +async def test_each_agent_session_keeps_a_repl_of_its_own_and_one_that_stops_starts_afresh_while_it_still_ends( + module, monkeypatch, tmp_path): + """A server that stops takes a while to end; the session's next call starts a new one rather than wait on it.""" + serve = module.CodexComputerUse._server + + @contextlib.asynccontextmanager + async def slow_to_end(self): + async with serve(self) as opened: + try: + yield opened + finally: + await asyncio.sleep(0.5) + + monkeypatch.setattr(module.CodexComputerUse, "_server", slow_to_end) + async with mcp_session(_driver(module, monkeypatch)) as session: + for caller, value in (("a", "1"), ("b", "2")): + await session.call_tool("js", {"code": f"set {value}"}, meta={SESSION_META_KEY: caller}) + a = _text(await session.call_tool("js", {"code": "get"}, meta={SESSION_META_KEY: "a"})) + b = _text(await session.call_tool("js", {"code": "get"}, meta={SESSION_META_KEY: "b"})) + died = await session.call_tool("js", {"code": "die"}, meta={SESSION_META_KEY: "a"}) + after = _text(await session.call_tool("js", {"code": "get"}, meta={SESSION_META_KEY: "a"})) + + (a_value, _, a_pid), (b_value, _, b_pid) = a.partition(" in "), b.partition(" in ") + assert (a_value, b_value) == ("1", "2") and a_pid != b_pid + assert died.isError and "may have run" in _text(died) + assert after.startswith("None in ") and after != f"None in {a_pid}" + + +@pytest.mark.asyncio +async def test_cua_repl_gets_the_apps_environment_and_none_of_the_drivers(module, monkeypatch, tmp_path): + async with mcp_session(_driver(module, monkeypatch)) as session: + await session.call_tool("js", {"code": "1 + 1"}) + + (env_file,) = (tmp_path / "fake").glob("env-*.json") + env = json.loads(env_file.read_text()) + modules = str(module.APP / module.NODE_MODULES) + assert "AGENTENV_CONTROL_TOKEN" not in env and env["PATH"] == "/usr/bin:/bin:/usr/sbin:/sbin" + assert env["CUA_REPL_ENABLED_SURFACES"] == "computer" and env["SKY_CUA_SERVICE_PATH"] == str(module.HELPER) + assert env["NODE_REPL_TRUSTED_SERVICES"] == '{"sky": "@oai/sky/service"}' + assert env["NODE_REPL_TRUSTED_CODE_PATHS"] == f"{tmp_path / '.codex'}:{modules}" + assert env["CODEX_HOME"] == str(tmp_path / ".codex") and env["NODE_REPL_NODE_PATH"] == str(module.APP / module.NODE) + + +@pytest.mark.asyncio +async def test_readiness_checks_the_pinned_executables_what_cua_repl_says_and_lists_and_that_it_answers( + module, monkeypatch, tmp_path): + driver = _driver(module, monkeypatch) + manifest = module._manifest() + + async def tools_check(served: dict): + (tmp_path / "fake" / "tools.json").write_text(json.dumps(served)) + driver._pinned = None + return (await driver.ready_checks())[1] + + ready = await driver.ready_checks() + fewer = await tools_check({**manifest, "tools": manifest["tools"][:3]}) + described = await tools_check({"instructions": "other", "tools": [ + {**t, "description": "other"} if t["name"] == "js" else t for t in manifest["tools"]]}) + hinted = await tools_check({**manifest, "tools": [ + {**t, "annotations": {}} if t["name"] == "js_reset" else t for t in manifest["tools"]]}) + (module.HELPER / "Contents" / "MacOS" / "SkyComputerUseService").write_text("another build") + driver._pinned = None + (changed, *_) = await driver.ready_checks() + + assert [(c.name, c.ok) for c in ready] == [("Codex Computer Use", True), ("Codex tools", True), + ("Codex answers", True), ("macOS permissions", True)] + assert (fewer.ok, fewer.detail) == (False, "these differ from mcp.json: turn_ended") + assert (described.ok, described.detail) == (False, "these differ from mcp.json: instructions, js") + assert (hinted.ok, hinted.detail) == (False, "these differ from mcp.json: js_reset") + assert not changed.ok and "Codex Computer Use.app/Contents/MacOS/SkyComputerUseService" in changed.detail + + +def test_env_toml_install_sh_and_the_driver_agree_on_the_pinned_version_and_the_helpers_path(pinned): + """A move to another ChatGPT version, or the helper to another folder, has to change them together.""" + folder = DRIVER.parent + macos = tomllib.loads((folder / "env.toml").read_text())["install"]["macos"] + install = (folder / "install.sh").read_text() + + (download,) = macos["downloads"] + assert download["url"].endswith(f"/ChatGPT-darwin-arm64-{pinned.VERSION}.zip") + assert [grant["app"] for grant in macos["grants"] if "app" in grant] == [str(pinned.HELPER)] + assert f'helper="{pinned.HELPER.name}"\n' in install and f"dest={pinned.HELPER.parent}\n" in install + assert pinned.APP == folder / "ChatGPT.app" and all( + path.startswith(("ChatGPT.app/", f"{pinned.HELPER.name}/")) for path in pinned.EXECUTABLES) + + +@pytest.mark.asyncio +async def test_past_the_most_repls_kept_the_least_recently_used_is_closed_and_the_others_keep_their_bindings( + module, monkeypatch): + monkeypatch.setattr(module, "MAX_REPLS", 2) + async with mcp_session(_driver(module, monkeypatch)) as session: + async def js(caller: str, code: str) -> str: + return _text(await session.call_tool("js", {"code": code}, meta={SESSION_META_KEY: caller})) + + for caller in "abc": + await js(caller, f"set {caller}") + b, c, a = await js("b", "get"), await js("c", "get"), await js("a", "get") + + assert a.startswith("None in ") and b.startswith("b in ") and c.startswith("c in ") + + +@pytest.mark.asyncio +async def test_a_repl_left_idle_is_closed_without_another_call_and_its_session_starts_afresh(module, monkeypatch): + monkeypatch.setattr(module, "SWEEP_S", 0.05) + monkeypatch.setattr(module, "IDLE_S", 0.2) + async with mcp_session(_driver(module, monkeypatch)) as session: + await session.call_tool("js", {"code": "set 1"}, meta={SESSION_META_KEY: "a"}) + before = _text(await session.call_tool("js", {"code": "get"}, meta={SESSION_META_KEY: "a"})) + closed = await _gone(int(before.rpartition(" in ")[2])) + after = _text(await session.call_tool("js", {"code": "get"}, meta={SESSION_META_KEY: "a"})) + + assert before.startswith("1 in ") and closed and after.startswith("None in ") + + +@pytest.mark.asyncio +async def test_calls_a_session_makes_together_while_another_repl_closes_share_one_repl( + module, monkeypatch, tmp_path): + monkeypatch.setattr(module, "MAX_REPLS", 1) + async with mcp_session(_driver(module, monkeypatch)) as session: + async def js(caller: str, code: str) -> str: + return _text(await session.call_tool("js", {"code": code}, meta={SESSION_META_KEY: caller})) + + await js("a", "set a") + await asyncio.gather(js("b", "set b"), js("b", "get")) + after = await js("b", "get") + + pids = {call["pid"] for call in _calls(tmp_path) if call["arguments"]["code"] in ("set b", "get")} + assert pids == {int(after.rpartition(" in ")[2])} and after.startswith("b in ")