diff --git a/apps/code/electron.vite.config.ts b/apps/code/electron.vite.config.ts index 2e8172dd17..aab4cb738c 100644 --- a/apps/code/electron.vite.config.ts +++ b/apps/code/electron.vite.config.ts @@ -134,6 +134,10 @@ export default defineConfig(({ mode }) => { }, rollupOptions: { input: { + "adapters/codex-app-server/local-tools-mcp-server": path.resolve( + __dirname, + "../../packages/agent/src/adapters/codex-app-server/local-tools-mcp-server.ts", + ), bootstrap: path.resolve(__dirname, "src/main/bootstrap.ts"), "workspace-server": require.resolve( "@posthog/workspace-server/serve", diff --git a/apps/code/package.json b/apps/code/package.json index a0a50e5b15..43d8b772bc 100644 --- a/apps/code/package.json +++ b/apps/code/package.json @@ -15,7 +15,7 @@ "start": "electron-vite dev --watch", "start:debug": "electron-vite dev --watch --inspect=5858", "compile": "electron-vite build", - "package": "electron-vite build && electron-builder build --dir --config electron-builder.ts", + "package": "electron-vite build && node ../../packages/agent/build/verify-local-tools-mcp-server.mjs .vite/build/adapters/codex-app-server/local-tools-mcp-server.js && electron-builder build --dir --config electron-builder.ts", "package:dev": "FORCE_DEV_MODE=1 SKIP_NOTARIZE=1 electron-vite build && electron-builder build --dir --config electron-builder.ts", "make": "electron-vite build && electron-builder build --config electron-builder.ts", "make:linux": "bash scripts/build-linux-docker.sh", diff --git a/docs/BROWSER-USE.md b/docs/BROWSER-USE.md index c9ed66cd21..0a4e64ea7a 100644 --- a/docs/BROWSER-USE.md +++ b/docs/BROWSER-USE.md @@ -1,6 +1,6 @@ -# Browser use +# Browser automation PostHog Code can give local agent sessions browser automation tools through an isolated Playwright MCP server. @@ -8,14 +8,14 @@ PostHog Code can give local agent sessions browser automation tools through an i 1. Install Google Chrome. 2. Open **Settings → Advanced**. -3. Enable **Browser use**. +3. Enable **Browser automation**. 4. Start a new local session. The setting applies when a session starts. Existing sessions are unchanged. ## Behavior -- Browser use is opt-in and disabled by default. +- Browser automation is opt-in and disabled by default. - It is available only to local sessions. - Each session launches an isolated Chrome profile, so it does not inherit cookies or logins from the user's normal browser profile. - Tool calls and screenshots use the existing MCP tool-call pipeline. diff --git a/docs/COMPUTER-USE.md b/docs/COMPUTER-USE.md new file mode 100644 index 0000000000..53a91e085e --- /dev/null +++ b/docs/COMPUTER-USE.md @@ -0,0 +1,38 @@ + + +# Computer use + +PostHog Code can give local agent sessions tools to see and control the macOS desktop across supported agent adapters. + +## Enable it + +1. Open **Settings → Advanced**. +2. Enable **Computer use**. +3. Start a new local session. +4. Grant Screen Recording and Accessibility access when macOS requests it. + +The setting applies when a session starts. Existing sessions are unchanged. + +## Behavior + +- Computer use is opt-in and disabled by default. +- It is available only to local sessions on macOS. +- Agents can capture the desktop, list visible applications, open or focus applications, click coordinates, type text, and press keys. +- Claude, Codex, and future adapters receive the same PostHog-owned MCP tools rather than adapter-specific computer-control implementations. +- Cloud sessions and unsupported operating systems do not receive the tools. +- Tool calls use the existing MCP tool-call and approval pipeline. + +## Safety + +- Review the screen before approving an action and verify the result after it runs. +- Do not ask an agent to enter passwords, tokens, recovery codes, or other secrets. +- Disable the setting and start a new session to remove the tools. +- Revoke access in **System Settings → Privacy & Security → Screen Recording** or **Accessibility** to prevent native control. + +## Scope + +The initial implementation uses macOS system utilities for screenshots, application launching, mouse input, and keyboard input. It does not provide computer control to cloud tasks because those tasks do not run on the user's computer. Remote computer use would require an explicit device connection, user-presence controls, and a separate security design. + +## Implementation + +Computer actions are registered in the shared local-tools MCP registry. The Claude adapter exposes that registry through its in-process MCP server, while the Codex adapter exposes the same registry through its stdio MCP server. Session metadata gates the tools by environment, operating system, and the user's opt-in setting. diff --git a/packages/agent/build/verify-local-tools-mcp-server.mjs b/packages/agent/build/verify-local-tools-mcp-server.mjs index 89fafa1e0e..d239afe4b6 100644 --- a/packages/agent/build/verify-local-tools-mcp-server.mjs +++ b/packages/agent/build/verify-local-tools-mcp-server.mjs @@ -8,10 +8,12 @@ import { spawn } from "node:child_process"; import { resolve } from "node:path"; import { fileURLToPath } from "node:url"; -const script = resolve( - fileURLToPath(new URL(".", import.meta.url)), - "../dist/adapters/codex-app-server/local-tools-mcp-server.js", -); +const script = process.argv[2] + ? resolve(process.argv[2]) + : resolve( + fileURLToPath(new URL(".", import.meta.url)), + "../dist/adapters/codex-app-server/local-tools-mcp-server.js", + ); const ctx = Buffer.from( JSON.stringify({ cwd: "/tmp", taskId: "smoke", token: "smoke-token" }), diff --git a/packages/agent/src/adapters/claude/claude-agent.ts b/packages/agent/src/adapters/claude/claude-agent.ts index 402d8dbdd4..dcd794c488 100644 --- a/packages/agent/src/adapters/claude/claude-agent.ts +++ b/packages/agent/src/adapters/claude/claude-agent.ts @@ -1892,6 +1892,7 @@ export class ClaudeAcpAgent extends BaseAcpAgent { const baseBranch = meta?.baseBranch; const environment = meta?.environment; const spokenNarration = resolveSpokenNarration(meta); + const computerUse = meta?.computerUse === true; const buildInProcessMcpServers = (): Record< string, McpSdkServerConfigWithInstance @@ -1903,8 +1904,9 @@ export class ClaudeAcpAgent extends BaseAcpAgent { taskId, taskRunId: meta?.taskRunId, baseBranch, + platform: process.platform, }, - { environment, spokenNarration }, + { environment, spokenNarration, computerUse }, ); return server ? { [LOCAL_TOOLS_MCP_NAME]: server } : {}; }; diff --git a/packages/agent/src/adapters/claude/mcp/local-tools.test.ts b/packages/agent/src/adapters/claude/mcp/local-tools.test.ts index 2525432245..1152808899 100644 --- a/packages/agent/src/adapters/claude/mcp/local-tools.test.ts +++ b/packages/agent/src/adapters/claude/mcp/local-tools.test.ts @@ -52,6 +52,71 @@ describe("createLocalToolsMcpServer", () => { expect(server).toBeUndefined(); }); + it("exposes computer tools for opted-in local macOS sessions", async () => { + const server = createLocalToolsMcpServer( + { cwd: "/repo", platform: "darwin" }, + { environment: "local", computerUse: true }, + ); + if (!server) { + throw new Error("expected the local-tools server to be registered"); + } + + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + await server.instance.connect(serverTransport); + const client = new Client({ name: "test", version: "1.0.0" }); + await client.connect(clientTransport); + + const { tools } = await client.listTools(); + expect(tools.map((tool) => tool.name)).toContain("computer_screenshot"); + + await client.close(); + }); + + it.each([ + { + platform: "linux" as const, + environment: "local" as const, + enabled: true, + }, + { + platform: "darwin" as const, + environment: "cloud" as const, + enabled: true, + }, + { + platform: "darwin" as const, + environment: "local" as const, + enabled: false, + }, + ])( + "does not expose computer tools for unsupported context %#", + async (input) => { + const server = createLocalToolsMcpServer( + { cwd: "/repo", platform: input.platform }, + { environment: input.environment, computerUse: input.enabled }, + ); + + if (!server) { + expect(server).toBeUndefined(); + return; + } + + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + await server.instance.connect(serverTransport); + const client = new Client({ name: "test", version: "1.0.0" }); + await client.connect(clientTransport); + + const { tools } = await client.listTools(); + expect(tools.map((tool) => tool.name)).not.toContain( + "computer_screenshot", + ); + + await client.close(); + }, + ); + it("exposes git_signed_commit over MCP in a cloud run with a token", async () => { const server = createLocalToolsMcpServer( { cwd: "/repo", token: "ghs_x" }, diff --git a/packages/agent/src/adapters/claude/types.ts b/packages/agent/src/adapters/claude/types.ts index 80078dfa8f..88ab5b12ad 100644 --- a/packages/agent/src/adapters/claude/types.ts +++ b/packages/agent/src/adapters/claude/types.ts @@ -200,6 +200,7 @@ export type NewSessionMeta = { * emits always (consumers gate playback), local stays silent. */ spokenNarration?: boolean; + computerUse?: boolean; jsonSchema?: Record | null; mcpToolApprovals?: McpToolApprovals; claudeCode?: { diff --git a/packages/agent/src/adapters/codex-app-server/codex-app-server-agent.ts b/packages/agent/src/adapters/codex-app-server/codex-app-server-agent.ts index 8b1ce61b89..4fd39ed49d 100644 --- a/packages/agent/src/adapters/codex-app-server/codex-app-server-agent.ts +++ b/packages/agent/src/adapters/codex-app-server/codex-app-server-agent.ts @@ -93,6 +93,7 @@ type AppServerSessionMeta = { environment?: "local" | "cloud"; channelMode?: boolean; spokenNarration?: boolean; + computerUse?: boolean; baseBranch?: string; nativeGoal?: NativeGoalState; }; @@ -540,6 +541,7 @@ export class CodexAppServerAgent extends BaseAcpAgent { environment: meta.environment, channelMode: meta.channelMode, spokenNarration: resolveSpokenNarration(meta), + computerUse: meta.computerUse, taskId: meta.taskId, taskRunId: meta.taskRunId, persistence: meta.persistence, diff --git a/packages/agent/src/adapters/codex-app-server/local-tools-mcp-server.ts b/packages/agent/src/adapters/codex-app-server/local-tools-mcp-server.ts index 7c65cfe560..12d22bcfbc 100644 --- a/packages/agent/src/adapters/codex-app-server/local-tools-mcp-server.ts +++ b/packages/agent/src/adapters/codex-app-server/local-tools-mcp-server.ts @@ -34,8 +34,10 @@ if (!ctxEnv) { let parsed: { cwd: string; taskId?: string; + taskRunId?: string; token?: string; baseBranch?: string; + platform?: NodeJS.Platform; }; try { parsed = JSON.parse(Buffer.from(ctxEnv, "base64").toString("utf-8")); @@ -51,7 +53,9 @@ const ctx: LocalToolCtx = { cwd: parsed.cwd, token: parsed.token ?? readGithubTokenFromEnv(), taskId: parsed.taskId, + taskRunId: parsed.taskRunId, baseBranch: parsed.baseBranch, + platform: parsed.platform, }; const enabledNames = (process.env.POSTHOG_LOCAL_TOOLS_ENABLED ?? "") @@ -74,4 +78,6 @@ for (const t of tools) { } const transport = new StdioServerTransport(); -await server.connect(transport); +server.connect(transport).catch((error) => { + die(`Failed to connect stdio transport: ${error}`); +}); diff --git a/packages/agent/src/adapters/codex-app-server/local-tools-mcp.test.ts b/packages/agent/src/adapters/codex-app-server/local-tools-mcp.test.ts index 0b53b4ceac..bf0ebeba3e 100644 --- a/packages/agent/src/adapters/codex-app-server/local-tools-mcp.test.ts +++ b/packages/agent/src/adapters/codex-app-server/local-tools-mcp.test.ts @@ -82,6 +82,7 @@ describe("buildLocalToolsServer", () => { expect(ctx.taskId).toBe("task-1"); expect(ctx.taskRunId).toBe("run-1"); expect(ctx.baseBranch).toBe("master"); + expect(ctx.platform).toBe(process.platform); }); it("returns a server but omits token env vars when no token is present", () => { @@ -130,4 +131,52 @@ describe("buildLocalToolsServer", () => { buildLocalToolsServer({ cwd: "/repo" }, { environment: "local" }), ).toBeNull(); }); + + it("exposes computer tools for opted-in local macOS sessions", () => { + const server = buildLocalToolsServer( + { cwd: "/repo", platform: "darwin" }, + { environment: "local", computerUse: true }, + ); + + const enabled = + server?.env.find((entry) => entry.name === "POSTHOG_LOCAL_TOOLS_ENABLED") + ?.value ?? ""; + expect(enabled.split(",")).toEqual( + expect.arrayContaining([ + "computer_screenshot", + "computer_open_application", + "computer_click", + "computer_type", + "computer_key", + ]), + ); + }); + + it.each([ + { + platform: "linux" as const, + environment: "local" as const, + enabled: true, + }, + { + platform: "darwin" as const, + environment: "cloud" as const, + enabled: true, + }, + { + platform: "darwin" as const, + environment: "local" as const, + enabled: false, + }, + ])("does not expose computer tools for unsupported context %#", (input) => { + const server = buildLocalToolsServer( + { cwd: "/repo", platform: input.platform }, + { environment: input.environment, computerUse: input.enabled }, + ); + + const enabled = + server?.env.find((entry) => entry.name === "POSTHOG_LOCAL_TOOLS_ENABLED") + ?.value ?? ""; + expect(enabled.split(",")).not.toContain("computer_screenshot"); + }); }); diff --git a/packages/agent/src/adapters/codex-app-server/local-tools-mcp.ts b/packages/agent/src/adapters/codex-app-server/local-tools-mcp.ts index b46d5102c2..afdcbcc6d1 100644 --- a/packages/agent/src/adapters/codex-app-server/local-tools-mcp.ts +++ b/packages/agent/src/adapters/codex-app-server/local-tools-mcp.ts @@ -68,7 +68,7 @@ function toMcpServerStdio( * registry; the server is only injected when at least one passes. */ export function buildLocalToolsServer( - ctx: { cwd?: string }, + ctx: { cwd?: string; platform?: NodeJS.Platform }, meta: LocalToolsMeta | undefined, ): McpServerStdio | null { const cwd = ctx.cwd; @@ -81,6 +81,7 @@ export function buildLocalToolsServer( taskId: resolveTaskId(meta), taskRunId: meta?.taskRunId, baseBranch: meta?.baseBranch, + platform: ctx.platform ?? process.platform, }; const tools = enabledLocalTools(toolCtx, meta); if (tools.length === 0) { diff --git a/packages/agent/src/adapters/local-tools/index.ts b/packages/agent/src/adapters/local-tools/index.ts index c2aab7139d..b1d1bbd5dd 100644 --- a/packages/agent/src/adapters/local-tools/index.ts +++ b/packages/agent/src/adapters/local-tools/index.ts @@ -1,5 +1,6 @@ import type { LocalTool, LocalToolCtx, LocalToolGateMeta } from "./registry"; import { cloneRepoTool } from "./tools/clone-repo"; +import { computerUseTools } from "./tools/computer-use"; import { listReposTool } from "./tools/list-repos"; import { signedCommitTool } from "./tools/signed-commit"; import { signedMergeTool } from "./tools/signed-merge"; @@ -23,6 +24,7 @@ export const LOCAL_TOOLS: LocalTool[] = [ listReposTool, cloneRepoTool, speakTool, + ...computerUseTools, ]; /** Tools whose gate passes for the given context — the set to actually expose. */ diff --git a/packages/agent/src/adapters/local-tools/registry.ts b/packages/agent/src/adapters/local-tools/registry.ts index d40c327f3b..164e5d8e3e 100644 --- a/packages/agent/src/adapters/local-tools/registry.ts +++ b/packages/agent/src/adapters/local-tools/registry.ts @@ -12,6 +12,7 @@ export const LOCAL_TOOLS_MCP_NAME = "posthog-code-tools"; /** Runtime context handed to every local tool's handler and gate. */ export interface LocalToolCtx { cwd: string; + platform?: NodeJS.Platform; /** GitHub token available to the sandbox, if any. */ token?: string; taskId?: string; @@ -30,6 +31,8 @@ export interface LocalToolGateMeta { channelMode?: boolean; /** Spoken narration is on for this session: enables the speak tool. */ spokenNarration?: boolean; + /** Desktop computer control is enabled for this local session. */ + computerUse?: boolean; } /** @@ -38,7 +41,20 @@ export interface LocalToolGateMeta { * both attach an open `_meta`). */ export interface LocalToolResult { - content: { type: "text"; text: string }[]; + content: Array< + | { + type: "text"; + text: string; + data?: never; + mimeType?: never; + } + | { + type: "image"; + text?: never; + data: string; + mimeType: string; + } + >; isError?: true; [key: string]: unknown; } diff --git a/packages/agent/src/adapters/local-tools/tools/computer-use.test.ts b/packages/agent/src/adapters/local-tools/tools/computer-use.test.ts new file mode 100644 index 0000000000..d420bf998f --- /dev/null +++ b/packages/agent/src/adapters/local-tools/tools/computer-use.test.ts @@ -0,0 +1,147 @@ +import { execFile } from "node:child_process"; +import { readFile, unlink } from "node:fs/promises"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { + computerClickTool, + computerKeyTool, + computerOpenApplicationTool, + computerScreenshotTool, + computerTypeTool, + computerUseTools, +} from "./computer-use"; + +vi.mock("node:child_process", () => ({ execFile: vi.fn() })); +vi.mock("node:fs/promises", () => ({ + readFile: vi.fn(), + unlink: vi.fn().mockResolvedValue(undefined), +})); + +const mockedExecFile = vi.mocked(execFile); +const mockedReadFile = vi.mocked(readFile); +const mockedUnlink = vi.mocked(unlink); + +const context = { cwd: "/repo", platform: "darwin" as const }; +const enabledMeta = { environment: "local" as const, computerUse: true }; + +describe("computer use tools", () => { + beforeEach(() => { + vi.clearAllMocks(); + mockedExecFile.mockImplementation(((...args: unknown[]) => { + const callback = args.at(-1); + if (typeof callback === "function") { + callback(null, "", ""); + } + return undefined; + }) as unknown as typeof execFile); + mockedReadFile.mockResolvedValue(Buffer.from("png")); + mockedUnlink.mockResolvedValue(undefined); + }); + + it("registers unique computer tool names", () => { + const names = computerUseTools.map((tool) => tool.name); + expect(new Set(names).size).toBe(names.length); + }); + + it.each([ + { + platform: "linux" as const, + environment: "local" as const, + enabled: true, + }, + { + platform: "darwin" as const, + environment: "cloud" as const, + enabled: true, + }, + { + platform: "darwin" as const, + environment: "local" as const, + enabled: false, + }, + ])("stays hidden for unsupported context %#", (input) => { + expect( + computerScreenshotTool.isEnabled( + { cwd: "/repo", platform: input.platform }, + { environment: input.environment, computerUse: input.enabled }, + ), + ).toBe(false); + }); + + it("is enabled for opted-in local macOS sessions", () => { + expect(computerScreenshotTool.isEnabled(context, enabledMeta)).toBe(true); + }); + + it("returns a screenshot image and removes the temporary file", async () => { + const result = await computerScreenshotTool.handler(context, {}); + + expect(result.content).toEqual( + expect.arrayContaining([ + expect.objectContaining({ type: "image", mimeType: "image/jpeg" }), + ]), + ); + expect(mockedExecFile).toHaveBeenCalledWith( + "/usr/bin/sips", + expect.arrayContaining(["--resampleHeightWidthMax", "1600"]), + { encoding: "utf8" }, + expect.any(Function), + ); + expect(mockedUnlink).toHaveBeenCalledTimes(2); + }); + + it("retries screenshot compression until the image fits the request budget", async () => { + mockedReadFile + .mockResolvedValueOnce(Buffer.alloc(1_000_001)) + .mockResolvedValueOnce(Buffer.from("jpeg")); + + const result = await computerScreenshotTool.handler(context, {}); + + expect(result.content).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + type: "image", + data: Buffer.from("jpeg").toString("base64"), + }), + ]), + ); + expect(mockedExecFile).toHaveBeenCalledWith( + "/usr/bin/sips", + expect.arrayContaining(["--resampleHeightWidthMax", "1200"]), + { encoding: "utf8" }, + expect.any(Function), + ); + }); + + it("opens applications without invoking a shell", async () => { + await computerOpenApplicationTool.handler(context, { + application: "Notes", + }); + + expect(mockedExecFile).toHaveBeenCalledWith( + "/usr/bin/open", + ["-a", "Notes"], + { encoding: "utf8" }, + expect.any(Function), + ); + expect(mockedExecFile).toHaveBeenCalledWith( + "/usr/bin/osascript", + expect.any(Array), + { encoding: "utf8" }, + expect.any(Function), + ); + }); + + it.each([ + [computerClickTool, { x: 100, y: 200 }], + [computerTypeTool, { text: "hello" }], + [computerKeyTool, { key: "l", modifiers: ["command"] }], + ] as const)("uses osascript for %s", async (tool, args) => { + await tool.handler(context, args); + + expect(mockedExecFile).toHaveBeenCalledWith( + "/usr/bin/osascript", + expect.any(Array), + { encoding: "utf8" }, + expect.any(Function), + ); + }); +}); diff --git a/packages/agent/src/adapters/local-tools/tools/computer-use.ts b/packages/agent/src/adapters/local-tools/tools/computer-use.ts new file mode 100644 index 0000000000..d416f65fc0 --- /dev/null +++ b/packages/agent/src/adapters/local-tools/tools/computer-use.ts @@ -0,0 +1,277 @@ +import { execFile } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { readFile, unlink } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { z } from "zod"; +import { + defineLocalTool, + type LocalTool, + type LocalToolCtx, + type LocalToolResult, +} from "../registry"; + +const modifierSchema = z.enum(["command", "control", "option", "shift"]); +const MAX_SCREENSHOT_BYTES = 1_000_000; +const screenshotCompressionAttempts = [ + { maxDimension: 1600, quality: "75" }, + { maxDimension: 1200, quality: "60" }, + { maxDimension: 900, quality: "45" }, +] as const; +const namedKeySchema = z.enum([ + "enter", + "tab", + "escape", + "delete", + "space", + "arrow_up", + "arrow_down", + "arrow_left", + "arrow_right", + "page_up", + "page_down", + "home", + "end", +]); + +const keyCodes: Record, number> = { + enter: 36, + tab: 48, + escape: 53, + delete: 51, + space: 49, + arrow_up: 126, + arrow_down: 125, + arrow_left: 123, + arrow_right: 124, + page_up: 116, + page_down: 121, + home: 115, + end: 119, +}; + +function run(file: string, args: string[]): Promise { + return new Promise((resolve, reject) => { + execFile(file, args, { encoding: "utf8" }, (error, stdout) => { + if (error) { + reject(error); + } else { + resolve(stdout.trim()); + } + }); + }); +} + +function runAppleScript(script: string, args: string[] = []): Promise { + return run("/usr/bin/osascript", ["-e", script, "--", ...args]); +} + +function computerUseEnabled( + ctx: LocalToolCtx, + meta: { environment?: "local" | "cloud"; computerUse?: boolean } | undefined, +): boolean { + return ( + (ctx.platform ?? process.platform) === "darwin" && + meta?.environment === "local" && + meta.computerUse === true + ); +} + +async function resultFrom( + action: () => Promise, +): Promise { + try { + return await action(); + } catch (error) { + return { + content: [ + { + type: "text", + text: + error instanceof Error + ? error.message + : "Computer control failed unexpectedly", + }, + ], + isError: true, + }; + } +} + +export const computerScreenshotTool = defineLocalTool({ + name: "computer_screenshot", + description: + "Capture the user's current macOS display. Use this before clicking and after each action to verify the actual result. Requires Screen Recording permission for PostHog Code.", + schema: {}, + alwaysLoad: true, + isEnabled: computerUseEnabled, + handler: async () => + resultFrom(async () => { + const id = randomUUID(); + const sourcePath = join(tmpdir(), `posthog-computer-${id}.png`); + const outputPath = join(tmpdir(), `posthog-computer-${id}.jpg`); + try { + await run("/usr/sbin/screencapture", ["-x", "-t", "png", sourcePath]); + for (const attempt of screenshotCompressionAttempts) { + await run("/usr/bin/sips", [ + "--resampleHeightWidthMax", + String(attempt.maxDimension), + "--setProperty", + "format", + "jpeg", + "--setProperty", + "formatOptions", + attempt.quality, + sourcePath, + "--out", + outputPath, + ]); + const data = await readFile(outputPath); + if (data.byteLength <= MAX_SCREENSHOT_BYTES) { + return { + content: [ + { type: "text", text: "Captured the current display." }, + { + type: "image", + data: data.toString("base64"), + mimeType: "image/jpeg", + }, + ], + }; + } + } + throw new Error("Captured screenshot exceeds the 1 MB image limit"); + } finally { + await Promise.all([ + unlink(sourcePath).catch(() => undefined), + unlink(outputPath).catch(() => undefined), + ]); + } + }), +}); + +export const computerListApplicationsTool = defineLocalTool({ + name: "computer_list_applications", + description: + "List visible applications currently running on the user's Mac. Use this before focusing or opening an application when its exact name is uncertain.", + schema: {}, + alwaysLoad: true, + isEnabled: computerUseEnabled, + handler: async () => + resultFrom(async () => { + const applications = await runAppleScript( + 'tell application "System Events" to get name of every application process whose background only is false', + ); + return { + content: [ + { + type: "text", + text: applications || "No visible applications are running.", + }, + ], + }; + }), +}); + +export const computerOpenApplicationTool = defineLocalTool({ + name: "computer_open_application", + description: + "Open or focus a macOS application by its display name, for example Safari, Slack, or Notes. Verify the result with computer_screenshot.", + schema: { + application: z.string().trim().min(1).max(120), + }, + alwaysLoad: true, + isEnabled: computerUseEnabled, + handler: async (_ctx, { application }) => + resultFrom(async () => { + await run("/usr/bin/open", ["-a", application]); + await runAppleScript( + 'on run argv\nset applicationName to item 1 of argv\ntell application "System Events" to set frontmost of process applicationName to true\nend run', + [application], + ); + return { + content: [{ type: "text", text: `Opened ${application}.` }], + }; + }), +}); + +export const computerClickTool = defineLocalTool({ + name: "computer_click", + description: + "Click an absolute screen coordinate on the user's Mac. Take a screenshot first, click the center of the intended control, then take another screenshot. Requires Accessibility permission for PostHog Code.", + schema: { + x: z.number().int().min(0).max(100_000), + y: z.number().int().min(0).max(100_000), + }, + alwaysLoad: true, + isEnabled: computerUseEnabled, + handler: async (_ctx, { x, y }) => + resultFrom(async () => { + await runAppleScript( + 'on run argv\nset clickX to item 1 of argv as integer\nset clickY to item 2 of argv as integer\ntell application "System Events" to click at {clickX, clickY}\nend run', + [String(x), String(y)], + ); + return { content: [{ type: "text", text: `Clicked ${x}, ${y}.` }] }; + }), +}); + +export const computerTypeTool = defineLocalTool({ + name: "computer_type", + description: + "Type text into the currently focused macOS control. Click the target field first and never use this for passwords, tokens, recovery codes, or other secrets. Requires Accessibility permission for PostHog Code.", + schema: { + text: z.string().min(1).max(4_000), + }, + alwaysLoad: true, + isEnabled: computerUseEnabled, + handler: async (_ctx, { text }) => + resultFrom(async () => { + await runAppleScript( + 'on run argv\ntell application "System Events" to keystroke (item 1 of argv)\nend run', + [text], + ); + return { + content: [{ type: "text", text: `Typed ${text.length} characters.` }], + }; + }), +}); + +export const computerKeyTool = defineLocalTool({ + name: "computer_key", + description: + "Press a named key or a single letter/number with optional modifiers on the user's Mac, for example command+l or enter. Requires Accessibility permission for PostHog Code.", + schema: { + key: z.union([namedKeySchema, z.string().regex(/^[a-z0-9]$/i)]), + modifiers: z.array(modifierSchema).max(4).default([]), + }, + alwaysLoad: true, + isEnabled: computerUseEnabled, + handler: async (_ctx, { key, modifiers }) => + resultFrom(async () => { + const modifierList = modifiers.map((modifier) => `${modifier} down`); + const usingClause = + modifierList.length > 0 ? ` using {${modifierList.join(", ")}}` : ""; + const script = + key in keyCodes + ? `tell application "System Events" to key code ${keyCodes[key as keyof typeof keyCodes]}${usingClause}` + : `tell application "System Events" to keystroke "${key.toLowerCase()}"${usingClause}`; + await runAppleScript(script); + return { + content: [ + { + type: "text", + text: `Pressed ${[...modifiers, key].join("+")}.`, + }, + ], + }; + }), +}); + +export const computerUseTools: LocalTool[] = [ + computerScreenshotTool, + computerListApplicationsTool, + computerOpenApplicationTool, + computerClickTool, + computerTypeTool, + computerKeyTool, +]; diff --git a/packages/core/src/sessions/sessionService.ts b/packages/core/src/sessions/sessionService.ts index c40fcc9622..9dc6727e6b 100644 --- a/packages/core/src/sessions/sessionService.ts +++ b/packages/core/src/sessions/sessionService.ts @@ -338,6 +338,7 @@ export interface SessionServiceDeps { spokenNotifications?: boolean; spokenNarrationEnabled?: boolean; browserUse?: boolean; + computerUse?: boolean; }; usageLimit: { show: (...args: any[]) => any }; readonly addDirectoryDialog: { open: boolean }; @@ -1094,6 +1095,7 @@ export class SessionService { rtkEnabledLocal, spokenNarrationEnabled, browserUse, + computerUse, } = this.d.settings; const result = await this.d.trpc.agent.reconnect.mutate({ taskId, @@ -1102,6 +1104,7 @@ export class SessionService { rtkEnabled: rtkEnabledLocal, spokenNarration: spokenNarrationEnabled === true, browserUse: browserUse === true, + computerUse: computerUse === true, apiHost: auth.apiHost, projectId: auth.projectId, logUrl, @@ -1426,6 +1429,7 @@ export class SessionService { rtkEnabledLocal, spokenNarrationEnabled, browserUse, + computerUse, } = this.d.settings; const preferredModel = model ?? this.d.DEFAULT_GATEWAY_MODEL; const result = await this.d.trpc.agent.start.mutate({ @@ -1440,6 +1444,7 @@ export class SessionService { rtkEnabled: rtkEnabledLocal, spokenNarration: spokenNarrationEnabled === true, browserUse: browserUse === true, + computerUse: computerUse === true, effort: effortLevelSchema.safeParse(reasoningLevel).success ? (reasoningLevel as EffortLevel) : undefined, diff --git a/packages/ui/src/features/sessions/sessionServiceHost.ts b/packages/ui/src/features/sessions/sessionServiceHost.ts index edb71ffa10..5d67a55443 100644 --- a/packages/ui/src/features/sessions/sessionServiceHost.ts +++ b/packages/ui/src/features/sessions/sessionServiceHost.ts @@ -138,6 +138,7 @@ function buildSessionServiceDeps(): SessionServiceDeps { import.meta.env.DEV, ), browserUse: state.browserUse, + computerUse: state.computerUse, }; }, usageLimit: { diff --git a/packages/ui/src/features/settings/sections/AdvancedSettings.tsx b/packages/ui/src/features/settings/sections/AdvancedSettings.tsx index b2ca90c572..9c27caf810 100644 --- a/packages/ui/src/features/settings/sections/AdvancedSettings.tsx +++ b/packages/ui/src/features/settings/sections/AdvancedSettings.tsx @@ -35,6 +35,8 @@ export function AdvancedSettings() { const setRtkEnabledCloud = useSettingsStore((s) => s.setRtkEnabledCloud); const browserUse = useSettingsStore((s) => s.browserUse); const setBrowserUse = useSettingsStore((s) => s.setBrowserUse); + const computerUse = useSettingsStore((s) => s.computerUse); + const setComputerUse = useSettingsStore((s) => s.setComputerUse); const hostTRPC = useHostTRPC(); const { data: rtkStatus } = useQuery(hostTRPC.agent.rtkStatus.queryOptions()); const devModeClient = useServiceOptional(DEV_MODE_CLIENT); @@ -91,11 +93,21 @@ export function AdvancedSettings() { + + + { "local", ); expect(useSettingsStore.getState().browserUse).toBe(false); + expect(useSettingsStore.getState().computerUse).toBe(false); }); }); @@ -222,6 +223,7 @@ describe("feature settingsStore cloud selections", () => { ["dismissibleUpdateBanners", false, true], ["showSidebarWorktrees", false, true], ["browserUse", false, true], + ["computerUse", false, true], ] as const)("rehydrates %s", async (field, initial, persisted) => { getItem.mockResolvedValue( JSON.stringify({ state: { [field]: persisted }, version: 0 }), diff --git a/packages/ui/src/features/settings/settingsStore.ts b/packages/ui/src/features/settings/settingsStore.ts index 1dcea3148e..d62653d9d6 100644 --- a/packages/ui/src/features/settings/settingsStore.ts +++ b/packages/ui/src/features/settings/settingsStore.ts @@ -212,6 +212,7 @@ interface SettingsStore { rtkEnabledLocal: boolean; rtkEnabledCloud: boolean; browserUse: boolean; + computerUse: boolean; setAllowBypassPermissions: (enabled: boolean) => void; setPreventSleepWhileRunning: (enabled: boolean) => void; setDebugLogsCloudRuns: (enabled: boolean) => void; @@ -219,6 +220,7 @@ interface SettingsStore { setRtkEnabledLocal: (enabled: boolean) => void; setRtkEnabledCloud: (enabled: boolean) => void; setBrowserUse: (enabled: boolean) => void; + setComputerUse: (enabled: boolean) => void; // Terminal terminalFont: TerminalFont; @@ -431,6 +433,7 @@ export const useSettingsStore = create()( rtkEnabledLocal: true, rtkEnabledCloud: true, browserUse: false, + computerUse: false, setAllowBypassPermissions: (enabled) => set({ allowBypassPermissions: enabled }), setPreventSleepWhileRunning: (enabled) => @@ -441,6 +444,7 @@ export const useSettingsStore = create()( setRtkEnabledLocal: (enabled) => set({ rtkEnabledLocal: enabled }), setRtkEnabledCloud: (enabled) => set({ rtkEnabledCloud: enabled }), setBrowserUse: (enabled) => set({ browserUse: enabled }), + setComputerUse: (enabled) => set({ computerUse: enabled }), // Terminal terminalFont: "berkeley-mono", @@ -571,6 +575,7 @@ export const useSettingsStore = create()( rtkEnabledLocal: state.rtkEnabledLocal, rtkEnabledCloud: state.rtkEnabledCloud, browserUse: state.browserUse, + computerUse: state.computerUse, // Terminal terminalFont: state.terminalFont, diff --git a/packages/workspace-server/src/services/agent/agent.test.ts b/packages/workspace-server/src/services/agent/agent.test.ts index 305586aa9c..d9eff93c85 100644 --- a/packages/workspace-server/src/services/agent/agent.test.ts +++ b/packages/workspace-server/src/services/agent/agent.test.ts @@ -470,6 +470,34 @@ describe("AgentService", () => { ); }); + it.each([{ computerUse: true }, { computerUse: false }])( + "threads computerUse $computerUse into newSession meta", + async ({ computerUse }) => { + await service.startSession({ + ...baseSessionParams, + adapter: "codex", + computerUse, + }); + + expect(mockNewSession).toHaveBeenCalledTimes(1); + expect(mockNewSession.mock.calls[0][0]._meta).toMatchObject({ + computerUse, + }); + }, + ); + + it("omits computerUse from newSession meta when unset", async () => { + await service.startSession({ + ...baseSessionParams, + adapter: "claude", + }); + + expect(mockNewSession).toHaveBeenCalledTimes(1); + expect(mockNewSession.mock.calls[0][0]._meta).not.toHaveProperty( + "computerUse", + ); + }); + it("keeps browser configuration out of adapter-specific session meta", async () => { await service.startSession({ ...baseSessionParams, diff --git a/packages/workspace-server/src/services/agent/agent.ts b/packages/workspace-server/src/services/agent/agent.ts index 014c16442b..f988f97a77 100644 --- a/packages/workspace-server/src/services/agent/agent.ts +++ b/packages/workspace-server/src/services/agent/agent.ts @@ -293,6 +293,8 @@ interface SessionConfig { spokenNarration?: boolean; /** Whether local sessions may launch the isolated browser-use tools. */ browserUse?: boolean; + /** Whether local sessions may control the macOS desktop. */ + computerUse?: boolean; } /** Pull the adapter's `agentCapabilities._meta.posthog.steering` from initialize. */ @@ -1011,6 +1013,9 @@ If a repository IS genuinely required, attach one in this priority order: ...(config.spokenNarration !== undefined && { spokenNarration: config.spokenNarration, }), + ...(config.computerUse !== undefined && { + computerUse: config.computerUse, + }), mcpToolApprovals: toolApprovals, ...(permissionMode && { permissionMode }), ...(model != null && { model }), @@ -1086,6 +1091,9 @@ If a repository IS genuinely required, attach one in this priority order: ...(config.spokenNarration !== undefined && { spokenNarration: config.spokenNarration, }), + ...(config.computerUse !== undefined && { + computerUse: config.computerUse, + }), mcpToolApprovals: toolApprovals, ...(permissionMode && { permissionMode }), ...(model != null && { model }), @@ -1115,6 +1123,9 @@ If a repository IS genuinely required, attach one in this priority order: ...(config.spokenNarration !== undefined && { spokenNarration: config.spokenNarration, }), + ...(config.computerUse !== undefined && { + computerUse: config.computerUse, + }), mcpToolApprovals: toolApprovals, ...(permissionMode && { permissionMode }), ...(model != null && { model }), @@ -2044,6 +2055,7 @@ For git operations while detached: spokenNarration: "spokenNarration" in params ? params.spokenNarration : undefined, browserUse: "browserUse" in params ? params.browserUse : undefined, + computerUse: "computerUse" in params ? params.computerUse : undefined, }; } diff --git a/packages/workspace-server/src/services/agent/schemas.ts b/packages/workspace-server/src/services/agent/schemas.ts index 5670e9419f..1713100e35 100644 --- a/packages/workspace-server/src/services/agent/schemas.ts +++ b/packages/workspace-server/src/services/agent/schemas.ts @@ -100,6 +100,8 @@ export const startSessionInput = z.object({ spokenNarration: z.boolean().optional(), /** Enables the isolated Playwright browser tools for local sessions. */ browserUse: z.boolean().optional(), + /** Enables local macOS screen, application, mouse, and keyboard tools. */ + computerUse: z.boolean().optional(), }); export type StartSessionInput = z.infer; @@ -237,6 +239,8 @@ export const reconnectSessionInput = z.object({ spokenNarration: z.boolean().optional(), /** See startSessionInput.browserUse. */ browserUse: z.boolean().optional(), + /** See startSessionInput.computerUse. */ + computerUse: z.boolean().optional(), }); export type ReconnectSessionInput = z.infer;