diff --git a/docs/COMPUTER-USE.md b/docs/COMPUTER-USE.md index 53a91e085e..dbc6c51c0a 100644 --- a/docs/COMPUTER-USE.md +++ b/docs/COMPUTER-USE.md @@ -2,37 +2,37 @@ # Computer use -PostHog Code can give local agent sessions tools to see and control the macOS desktop across supported agent adapters. +PostHog Code can give local agent sessions tools to control the Mac and cloud tasks tools to control an isolated Linux desktop in their sandbox. ## Enable it 1. Open **Settings → Advanced**. 2. Enable **Computer use**. -3. Start a new local session. -4. Grant Screen Recording and Accessibility access when macOS requests it. +3. Start a new local session or cloud task. +4. For local sessions, grant Screen Recording and Accessibility access when macOS requests it. -The setting applies when a session starts. Existing sessions are unchanged. +The setting applies when a session or cloud task starts. Existing runs are unchanged. ## Behavior - Computer use is opt-in and disabled by default. -- It is available only to local sessions on macOS. +- Local sessions control the user's Mac. Cloud tasks control a virtual Linux desktop inside their own sandbox. - Agents can capture the desktop, list visible applications, open or focus applications, click coordinates, type text, and press keys. - Claude, Codex, and future adapters receive the same PostHog-owned MCP tools rather than adapter-specific computer-control implementations. -- Cloud sessions and unsupported operating systems do not receive the tools. +- Unsupported operating systems do not receive the tools. - Tool calls use the existing MCP tool-call and approval pipeline. ## Safety - Review the screen before approving an action and verify the result after it runs. - Do not ask an agent to enter passwords, tokens, recovery codes, or other secrets. -- Disable the setting and start a new session to remove the tools. +- Disable the setting before starting a new session or cloud task to omit the tools. - Revoke access in **System Settings → Privacy & Security → Screen Recording** or **Accessibility** to prevent native control. ## Scope -The initial implementation uses macOS system utilities for screenshots, application launching, mouse input, and keyboard input. It does not provide computer control to cloud tasks because those tasks do not run on the user's computer. Remote computer use would require an explicit device connection, user-presence controls, and a separate security design. +Local sessions use macOS system utilities for screenshots, application launching, mouse input, and keyboard input. Cloud tasks start a virtual X display and lightweight window manager in the sandbox, then use Linux desktop utilities for the same actions. The cloud desktop contains only the task sandbox and cannot access the user's computer. ## Implementation -Computer actions are registered in the shared local-tools MCP registry. The Claude adapter exposes that registry through its in-process MCP server, while the Codex adapter exposes the same registry through its stdio MCP server. Session metadata gates the tools by environment, operating system, and the user's opt-in setting. +Computer actions are registered in the shared local-tools MCP registry. The Claude adapter exposes that registry through its in-process MCP server, while the Codex adapter exposes the same registry through the packaged stdio MCP server. Session metadata gates the tools by environment, operating system, and the user's opt-in setting. diff --git a/packages/agent/src/adapters/codex-app-server/local-tools-mcp.test.ts b/packages/agent/src/adapters/codex-app-server/local-tools-mcp.test.ts index bf0ebeba3e..d48af2c81f 100644 --- a/packages/agent/src/adapters/codex-app-server/local-tools-mcp.test.ts +++ b/packages/agent/src/adapters/codex-app-server/local-tools-mcp.test.ts @@ -14,6 +14,7 @@ describe("buildLocalToolsServer", () => { sandbox: process.env.IS_SANDBOX, ghToken: process.env.GH_TOKEN, githubToken: process.env.GITHUB_TOKEN, + display: process.env.DISPLAY, }; beforeEach(() => { @@ -22,12 +23,14 @@ describe("buildLocalToolsServer", () => { delete process.env.IS_SANDBOX; delete process.env.GH_TOKEN; delete process.env.GITHUB_TOKEN; + delete process.env.DISPLAY; }); afterEach(() => { restore("IS_SANDBOX", saved.sandbox); restore("GH_TOKEN", saved.ghToken); restore("GITHUB_TOKEN", saved.githubToken); + restore("DISPLAY", saved.display); }); function restore(key: string, value: string | undefined): void { @@ -152,6 +155,20 @@ describe("buildLocalToolsServer", () => { ); }); + it("exposes computer tools for opted-in cloud Linux sessions", () => { + process.env.DISPLAY = ":99"; + const server = buildLocalToolsServer( + { cwd: "/repo", platform: "linux" }, + { environment: "cloud", computerUse: true }, + ); + + const enabled = + server?.env.find((entry) => entry.name === "POSTHOG_LOCAL_TOOLS_ENABLED") + ?.value ?? ""; + expect(enabled.split(",")).toContain("computer_screenshot"); + expect(server?.env).toContainEqual({ name: "DISPLAY", value: ":99" }); + }); + it.each([ { platform: "linux" as const, diff --git a/packages/agent/src/adapters/codex-app-server/local-tools-mcp.ts b/packages/agent/src/adapters/codex-app-server/local-tools-mcp.ts index afdcbcc6d1..9d7432929d 100644 --- a/packages/agent/src/adapters/codex-app-server/local-tools-mcp.ts +++ b/packages/agent/src/adapters/codex-app-server/local-tools-mcp.ts @@ -45,6 +45,9 @@ function toMcpServerStdio( // binary, which boots the full app without this var. Inert on real node. { name: "ELECTRON_RUN_AS_NODE", value: "1" }, ]; + if (process.env.DISPLAY) { + env.push({ name: "DISPLAY", value: process.env.DISPLAY }); + } if (ctx.token) { // Token also on the child env so its own git remote ops authenticate. env.push( diff --git a/packages/agent/src/adapters/local-tools/registry.ts b/packages/agent/src/adapters/local-tools/registry.ts index 164e5d8e3e..20a011b8a5 100644 --- a/packages/agent/src/adapters/local-tools/registry.ts +++ b/packages/agent/src/adapters/local-tools/registry.ts @@ -31,7 +31,7 @@ export interface LocalToolGateMeta { channelMode?: boolean; /** Spoken narration is on for this session: enables the speak tool. */ spokenNarration?: boolean; - /** Desktop computer control is enabled for this local session. */ + /** Computer control is enabled for this session. */ computerUse?: boolean; } diff --git a/packages/agent/src/adapters/local-tools/tools/computer-use.test.ts b/packages/agent/src/adapters/local-tools/tools/computer-use.test.ts index d420bf998f..a8bbe101d3 100644 --- a/packages/agent/src/adapters/local-tools/tools/computer-use.test.ts +++ b/packages/agent/src/adapters/local-tools/tools/computer-use.test.ts @@ -1,4 +1,4 @@ -import { execFile } from "node:child_process"; +import { execFile, spawn } from "node:child_process"; import { readFile, unlink } from "node:fs/promises"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { @@ -10,18 +10,21 @@ import { computerUseTools, } from "./computer-use"; -vi.mock("node:child_process", () => ({ execFile: vi.fn() })); +vi.mock("node:child_process", () => ({ execFile: vi.fn(), spawn: vi.fn() })); vi.mock("node:fs/promises", () => ({ readFile: vi.fn(), unlink: vi.fn().mockResolvedValue(undefined), })); const mockedExecFile = vi.mocked(execFile); +const mockedSpawn = vi.mocked(spawn); const mockedReadFile = vi.mocked(readFile); const mockedUnlink = vi.mocked(unlink); const context = { cwd: "/repo", platform: "darwin" as const }; const enabledMeta = { environment: "local" as const, computerUse: true }; +const cloudContext = { cwd: "/repo", platform: "linux" as const }; +const cloudMeta = { environment: "cloud" as const, computerUse: true }; describe("computer use tools", () => { beforeEach(() => { @@ -33,6 +36,7 @@ describe("computer use tools", () => { } return undefined; }) as unknown as typeof execFile); + mockedSpawn.mockReturnValue({ unref: vi.fn() } as never); mockedReadFile.mockResolvedValue(Buffer.from("png")); mockedUnlink.mockResolvedValue(undefined); }); @@ -71,6 +75,12 @@ describe("computer use tools", () => { expect(computerScreenshotTool.isEnabled(context, enabledMeta)).toBe(true); }); + it("is enabled for opted-in cloud Linux sessions", () => { + expect(computerScreenshotTool.isEnabled(cloudContext, cloudMeta)).toBe( + true, + ); + }); + it("returns a screenshot image and removes the temporary file", async () => { const result = await computerScreenshotTool.handler(context, {}); @@ -130,6 +140,33 @@ describe("computer use tools", () => { ); }); + it("opens the cloud browser without invoking a shell", async () => { + const unref = vi.fn(); + mockedSpawn.mockReturnValueOnce({ unref } as never); + + await computerOpenApplicationTool.handler(cloudContext, { + application: "Browser", + }); + + expect(mockedSpawn).toHaveBeenCalledWith( + "/usr/bin/epiphany", + ["--new-window"], + { detached: true, stdio: "ignore" }, + ); + expect(unref).toHaveBeenCalledOnce(); + }); + + it("uses Linux desktop utilities in cloud sessions", async () => { + await computerClickTool.handler(cloudContext, { x: 100, y: 200 }); + + expect(mockedExecFile).toHaveBeenCalledWith( + "/usr/bin/xdotool", + ["mousemove", "100", "200", "click", "1"], + { encoding: "utf8" }, + expect.any(Function), + ); + }); + it.each([ [computerClickTool, { x: 100, y: 200 }], [computerTypeTool, { text: "hello" }], diff --git a/packages/agent/src/adapters/local-tools/tools/computer-use.ts b/packages/agent/src/adapters/local-tools/tools/computer-use.ts index d416f65fc0..d0062d3ae0 100644 --- a/packages/agent/src/adapters/local-tools/tools/computer-use.ts +++ b/packages/agent/src/adapters/local-tools/tools/computer-use.ts @@ -1,4 +1,4 @@ -import { execFile } from "node:child_process"; +import { execFile, spawn } from "node:child_process"; import { randomUUID } from "node:crypto"; import { readFile, unlink } from "node:fs/promises"; import { tmpdir } from "node:os"; @@ -12,12 +12,6 @@ import { } from "../registry"; const modifierSchema = z.enum(["command", "control", "option", "shift"]); -const MAX_SCREENSHOT_BYTES = 1_000_000; -const screenshotCompressionAttempts = [ - { maxDimension: 1600, quality: "75" }, - { maxDimension: 1200, quality: "60" }, - { maxDimension: 900, quality: "45" }, -] as const; const namedKeySchema = z.enum([ "enter", "tab", @@ -33,8 +27,13 @@ const namedKeySchema = z.enum([ "home", "end", ]); - -const keyCodes: Record, number> = { +const MAX_SCREENSHOT_BYTES = 1_000_000; +const screenshotCompressionAttempts = [ + { maxDimension: 1600, quality: "75" }, + { maxDimension: 1200, quality: "60" }, + { maxDimension: 900, quality: "45" }, +] as const; +const macKeyCodes: Record, number> = { enter: 36, tab: 48, escape: 53, @@ -49,16 +48,27 @@ const keyCodes: Record, number> = { home: 115, end: 119, }; +const linuxKeys: Record, string> = { + enter: "Return", + tab: "Tab", + escape: "Escape", + delete: "BackSpace", + space: "space", + arrow_up: "Up", + arrow_down: "Down", + arrow_left: "Left", + arrow_right: "Right", + page_up: "Page_Up", + page_down: "Page_Down", + home: "Home", + end: "End", +}; function run(file: string, args: string[]): Promise { return new Promise((resolve, reject) => { - execFile(file, args, { encoding: "utf8" }, (error, stdout) => { - if (error) { - reject(error); - } else { - resolve(stdout.trim()); - } - }); + execFile(file, args, { encoding: "utf8" }, (error, stdout) => + error ? reject(error) : resolve(stdout.trim()), + ); }); } @@ -66,14 +76,18 @@ function runAppleScript(script: string, args: string[] = []): Promise { return run("/usr/bin/osascript", ["-e", script, "--", ...args]); } +function platform(ctx: LocalToolCtx): NodeJS.Platform { + return ctx.platform ?? process.platform; +} + function computerUseEnabled( ctx: LocalToolCtx, meta: { environment?: "local" | "cloud"; computerUse?: boolean } | undefined, ): boolean { + if (meta?.computerUse !== true) return false; return ( - (ctx.platform ?? process.platform) === "darwin" && - meta?.environment === "local" && - meta.computerUse === true + (platform(ctx) === "darwin" && meta.environment === "local") || + (platform(ctx) === "linux" && meta.environment === "cloud") ); } @@ -98,70 +112,88 @@ async function resultFrom( } } +async function captureScreenshot(ctx: LocalToolCtx): Promise { + const id = randomUUID(); + const sourcePath = join(tmpdir(), `posthog-computer-${id}.png`); + const outputPath = join(tmpdir(), `posthog-computer-${id}.jpg`); + try { + if (platform(ctx) === "darwin") { + await run("/usr/sbin/screencapture", ["-x", "-t", "png", sourcePath]); + } else { + await run("/usr/bin/import", ["-window", "root", sourcePath]); + } + for (const attempt of screenshotCompressionAttempts) { + if (platform(ctx) === "darwin") { + await run("/usr/bin/sips", [ + "--resampleHeightWidthMax", + String(attempt.maxDimension), + "--setProperty", + "format", + "jpeg", + "--setProperty", + "formatOptions", + attempt.quality, + sourcePath, + "--out", + outputPath, + ]); + } else { + await run("/usr/bin/convert", [ + sourcePath, + "-resize", + `${attempt.maxDimension}x${attempt.maxDimension}>`, + "-quality", + attempt.quality, + outputPath, + ]); + } + const data = await readFile(outputPath); + if (data.byteLength <= MAX_SCREENSHOT_BYTES) { + return { + content: [ + { type: "text", text: "Captured the current display." }, + { + type: "image", + data: data.toString("base64"), + mimeType: "image/jpeg", + }, + ], + }; + } + } + throw new Error("Captured screenshot exceeds the 1 MB image limit"); + } finally { + await Promise.all([ + unlink(sourcePath).catch(() => undefined), + unlink(outputPath).catch(() => undefined), + ]); + } +} + export const computerScreenshotTool = defineLocalTool({ name: "computer_screenshot", description: - "Capture the user's current macOS display. Use this before clicking and after each action to verify the actual result. Requires Screen Recording permission for PostHog Code.", + "Capture the current computer display. Use this before clicking and after each action to verify the actual result.", schema: {}, alwaysLoad: true, isEnabled: computerUseEnabled, - handler: async () => - resultFrom(async () => { - const id = randomUUID(); - const sourcePath = join(tmpdir(), `posthog-computer-${id}.png`); - const outputPath = join(tmpdir(), `posthog-computer-${id}.jpg`); - try { - await run("/usr/sbin/screencapture", ["-x", "-t", "png", sourcePath]); - for (const attempt of screenshotCompressionAttempts) { - await run("/usr/bin/sips", [ - "--resampleHeightWidthMax", - String(attempt.maxDimension), - "--setProperty", - "format", - "jpeg", - "--setProperty", - "formatOptions", - attempt.quality, - sourcePath, - "--out", - outputPath, - ]); - const data = await readFile(outputPath); - if (data.byteLength <= MAX_SCREENSHOT_BYTES) { - return { - content: [ - { type: "text", text: "Captured the current display." }, - { - type: "image", - data: data.toString("base64"), - mimeType: "image/jpeg", - }, - ], - }; - } - } - throw new Error("Captured screenshot exceeds the 1 MB image limit"); - } finally { - await Promise.all([ - unlink(sourcePath).catch(() => undefined), - unlink(outputPath).catch(() => undefined), - ]); - } - }), + handler: async (ctx) => resultFrom(() => captureScreenshot(ctx)), }); export const computerListApplicationsTool = defineLocalTool({ name: "computer_list_applications", - description: - "List visible applications currently running on the user's Mac. Use this before focusing or opening an application when its exact name is uncertain.", + description: "List visible applications on the current computer.", schema: {}, alwaysLoad: true, isEnabled: computerUseEnabled, - handler: async () => + handler: async (ctx) => resultFrom(async () => { - const applications = await runAppleScript( - 'tell application "System Events" to get name of every application process whose background only is false', - ); + const applications = + platform(ctx) === "darwin" + ? await runAppleScript( + 'tell application "System Events" to get name of every application process whose background only is false', + ) + : await run("/usr/bin/wmctrl", ["-lx"]); return { content: [ { @@ -176,41 +208,63 @@ export const computerListApplicationsTool = defineLocalTool({ export const computerOpenApplicationTool = defineLocalTool({ name: "computer_open_application", description: - "Open or focus a macOS application by its display name, for example Safari, Slack, or Notes. Verify the result with computer_screenshot.", - schema: { - application: z.string().trim().min(1).max(120), - }, + "Open or focus an application by name. In cloud tasks, common names include Chrome and Terminal.", + schema: { application: z.string().trim().min(1).max(120) }, alwaysLoad: true, isEnabled: computerUseEnabled, - handler: async (_ctx, { application }) => + handler: async (ctx, { application }) => resultFrom(async () => { - await run("/usr/bin/open", ["-a", application]); - await runAppleScript( - 'on run argv\nset applicationName to item 1 of argv\ntell application "System Events" to set frontmost of process applicationName to true\nend run', - [application], - ); - return { - content: [{ type: "text", text: `Opened ${application}.` }], - }; + if (platform(ctx) === "darwin") { + await run("/usr/bin/open", ["-a", application]); + await runAppleScript( + 'on run argv\nset applicationName to item 1 of argv\ntell application "System Events" to set frontmost of process applicationName to true\nend run', + [application], + ); + } else { + const normalized = application.toLowerCase(); + const command = + normalized.includes("chrome") || normalized.includes("browser") + ? "/usr/bin/epiphany" + : normalized.includes("terminal") + ? "/usr/bin/xterm" + : "/usr/bin/gtk-launch"; + const args = + command === "/usr/bin/gtk-launch" + ? [application] + : command === "/usr/bin/epiphany" + ? ["--new-window"] + : []; + spawn(command, args, { detached: true, stdio: "ignore" }).unref(); + } + return { content: [{ type: "text", text: `Opened ${application}.` }] }; }), }); export const computerClickTool = defineLocalTool({ name: "computer_click", description: - "Click an absolute screen coordinate on the user's Mac. Take a screenshot first, click the center of the intended control, then take another screenshot. Requires Accessibility permission for PostHog Code.", + "Click an absolute screen coordinate. Take a screenshot before and after the click.", schema: { x: z.number().int().min(0).max(100_000), y: z.number().int().min(0).max(100_000), }, alwaysLoad: true, isEnabled: computerUseEnabled, - handler: async (_ctx, { x, y }) => + handler: async (ctx, { x, y }) => resultFrom(async () => { - await runAppleScript( - 'on run argv\nset clickX to item 1 of argv as integer\nset clickY to item 2 of argv as integer\ntell application "System Events" to click at {clickX, clickY}\nend run', - [String(x), String(y)], - ); + if (platform(ctx) === "darwin") + await runAppleScript( + 'on run argv\nset clickX to item 1 of argv as integer\nset clickY to item 2 of argv as integer\ntell application "System Events" to click at {clickX, clickY}\nend run', + [String(x), String(y)], + ); + else + await run("/usr/bin/xdotool", [ + "mousemove", + String(x), + String(y), + "click", + "1", + ]); return { content: [{ type: "text", text: `Clicked ${x}, ${y}.` }] }; }), }); @@ -218,18 +272,18 @@ export const computerClickTool = defineLocalTool({ export const computerTypeTool = defineLocalTool({ name: "computer_type", description: - "Type text into the currently focused macOS control. Click the target field first and never use this for passwords, tokens, recovery codes, or other secrets. Requires Accessibility permission for PostHog Code.", - schema: { - text: z.string().min(1).max(4_000), - }, + "Type text into the focused control. Never use this for passwords, tokens, recovery codes, or other secrets.", + schema: { text: z.string().min(1).max(4_000) }, alwaysLoad: true, isEnabled: computerUseEnabled, - handler: async (_ctx, { text }) => + handler: async (ctx, { text }) => resultFrom(async () => { - await runAppleScript( - 'on run argv\ntell application "System Events" to keystroke (item 1 of argv)\nend run', - [text], - ); + if (platform(ctx) === "darwin") + await runAppleScript( + 'on run argv\ntell application "System Events" to keystroke (item 1 of argv)\nend run', + [text], + ); + else await run("/usr/bin/xdotool", ["type", "--delay", "1", "--", text]); return { content: [{ type: "text", text: `Typed ${text.length} characters.` }], }; @@ -239,29 +293,45 @@ export const computerTypeTool = defineLocalTool({ export const computerKeyTool = defineLocalTool({ name: "computer_key", description: - "Press a named key or a single letter/number with optional modifiers on the user's Mac, for example command+l or enter. Requires Accessibility permission for PostHog Code.", + "Press a named key or a single letter/number with optional modifiers.", schema: { key: z.union([namedKeySchema, z.string().regex(/^[a-z0-9]$/i)]), modifiers: z.array(modifierSchema).max(4).default([]), }, alwaysLoad: true, isEnabled: computerUseEnabled, - handler: async (_ctx, { key, modifiers }) => + handler: async (ctx, { key, modifiers }) => resultFrom(async () => { - const modifierList = modifiers.map((modifier) => `${modifier} down`); - const usingClause = - modifierList.length > 0 ? ` using {${modifierList.join(", ")}}` : ""; - const script = - key in keyCodes - ? `tell application "System Events" to key code ${keyCodes[key as keyof typeof keyCodes]}${usingClause}` - : `tell application "System Events" to keystroke "${key.toLowerCase()}"${usingClause}`; - await runAppleScript(script); + if (platform(ctx) === "darwin") { + const modifierList = modifiers.map((modifier) => `${modifier} down`); + const usingClause = + modifierList.length > 0 ? ` using {${modifierList.join(", ")}}` : ""; + const script = + key in macKeyCodes + ? `tell application "System Events" to key code ${macKeyCodes[key as keyof typeof macKeyCodes]}${usingClause}` + : `tell application "System Events" to keystroke "${key.toLowerCase()}"${usingClause}`; + await runAppleScript(script); + } else { + const linuxModifiers = modifiers.map( + (modifier) => + ({ + command: "super", + control: "ctrl", + option: "alt", + shift: "shift", + })[modifier], + ); + await run("/usr/bin/xdotool", [ + "key", + [ + ...linuxModifiers, + linuxKeys[key as keyof typeof linuxKeys] ?? key.toLowerCase(), + ].join("+"), + ]); + } return { content: [ - { - type: "text", - text: `Pressed ${[...modifiers, key].join("+")}.`, - }, + { type: "text", text: `Pressed ${[...modifiers, key].join("+")}.` }, ], }; }), diff --git a/packages/agent/src/server/agent-server.ts b/packages/agent/src/server/agent-server.ts index 140863f0da..4181624bc1 100644 --- a/packages/agent/src/server/agent-server.ts +++ b/packages/agent/src/server/agent-server.ts @@ -1493,6 +1493,7 @@ export class AgentServer { taskRunId: payload.run_id, taskId: payload.task_id, environment: "cloud", + computerUse: this.config.computerUse === true, systemPrompt: sessionSystemPrompt, ...(this.config.model && { model: this.config.model }), allowedDomains: this.config.allowedDomains, diff --git a/packages/agent/src/server/bin.ts b/packages/agent/src/server/bin.ts index 6f5a85c6f6..affcd95a69 100644 --- a/packages/agent/src/server/bin.ts +++ b/packages/agent/src/server/bin.ts @@ -114,6 +114,7 @@ program "--relayMcpServers ", "Desktop-relayed MCP server names as JSON array (docs/cloud-mcp-relay.md)", ) + .option("--computerUse ", "Enable native cloud computer control") .option("--createPr ", "Whether this run may publish changes") .option( "--autoPublish ", @@ -142,6 +143,10 @@ program const env = envResult.data; const mode = options.mode === "background" ? "background" : "interactive"; + const computerUse = parseBooleanOption( + options.computerUse, + "--computerUse", + ); const createPr = parseBooleanOption(options.createPr, "--createPr"); const autoPublish = parseBooleanOption( options.autoPublish, @@ -208,6 +213,7 @@ program autoPublish, mcpServers, relayMcpServers, + computerUse, baseBranch: options.baseBranch, claudeCode, allowedDomains, diff --git a/packages/agent/src/server/types.ts b/packages/agent/src/server/types.ts index 460a455fed..77688e9820 100644 --- a/packages/agent/src/server/types.ts +++ b/packages/agent/src/server/types.ts @@ -39,6 +39,7 @@ export interface AgentServerConfig { * each name against local config at execution time. */ relayMcpServers?: string[]; + computerUse?: boolean; baseBranch?: string; claudeCode?: ClaudeCodeConfig; allowedDomains?: string[]; diff --git a/packages/api-client/src/posthog-client.test.ts b/packages/api-client/src/posthog-client.test.ts index 3e6509ac08..f8cbbd361f 100644 --- a/packages/api-client/src/posthog-client.test.ts +++ b/packages/api-client/src/posthog-client.test.ts @@ -223,6 +223,7 @@ describe("PostHogAPIClient", () => { model: "gpt-5.4", reasoningLevel: "high", initialPermissionMode: "auto", + computerUse: true, }), ).resolves.toEqual({ id: "run-123", environment: "cloud" }); @@ -238,6 +239,7 @@ describe("PostHogAPIClient", () => { model: "gpt-5.4", reasoning_effort: "high", initial_permission_mode: "auto", + computer_use: true, environment: "cloud", }), }, diff --git a/packages/api-client/src/posthog-client.ts b/packages/api-client/src/posthog-client.ts index a9c5e72318..5b60c1b33b 100644 --- a/packages/api-client/src/posthog-client.ts +++ b/packages/api-client/src/posthog-client.ts @@ -595,6 +595,7 @@ interface CloudRunOptions { autoPublish?: boolean; /** Only false is sent: opts the run out of rtk command-output compression. */ rtkEnabled?: boolean; + computerUse?: boolean; runSource?: CloudRunSource; signalReportId?: string; initialPermissionMode?: ExecutionMode; @@ -690,6 +691,9 @@ function buildCloudRunRequestBody( if (options?.rtkEnabled === false) { body.rtk_enabled = false; } + if (options?.computerUse === true) { + body.computer_use = true; + } if (options?.runSource) { body.run_source = options.runSource; } diff --git a/packages/core/src/task-detail/taskCreationApiClient.ts b/packages/core/src/task-detail/taskCreationApiClient.ts index b09c5f75d2..9ecdf63bd8 100644 --- a/packages/core/src/task-detail/taskCreationApiClient.ts +++ b/packages/core/src/task-detail/taskCreationApiClient.ts @@ -19,6 +19,7 @@ export interface CreateTaskRunClientOptions { prAuthorshipMode?: PrAuthorshipMode; autoPublish?: boolean; rtkEnabled?: boolean; + computerUse?: boolean; runSource?: CloudRunSource; signalReportId?: string; initialPermissionMode?: string; diff --git a/packages/core/src/task-detail/taskCreationHost.ts b/packages/core/src/task-detail/taskCreationHost.ts index a419ccf60a..4f72c7bcaf 100644 --- a/packages/core/src/task-detail/taskCreationHost.ts +++ b/packages/core/src/task-detail/taskCreationHost.ts @@ -76,6 +76,7 @@ export interface RecordClaudeCliImportArgs { export interface ITaskCreationHost { getAuthenticatedClient(): Promise; assertCloudUsageAvailable(): Promise; + isComputerUseEnabled(): boolean; getTaskDirectory(taskId: string, repoKey?: string): Promise; /** * Ensure a per-task scratch working directory exists for a repo-less channel diff --git a/packages/core/src/task-detail/taskCreationSaga.test.ts b/packages/core/src/task-detail/taskCreationSaga.test.ts index 6a06da8206..b96a5fd299 100644 --- a/packages/core/src/task-detail/taskCreationSaga.test.ts +++ b/packages/core/src/task-detail/taskCreationSaga.test.ts @@ -45,6 +45,7 @@ const sessionService = { disconnectFromTask: vi.fn(), rememberInitialCloudPrompt: vi.fn(), markTaskCreationInFlight: vi.fn(), + designateRelayedMcpServers: vi.fn().mockResolvedValue(undefined), } as unknown as SessionService; const createTask = (overrides: Partial = {}): Task => ({ @@ -155,6 +156,7 @@ describe("TaskCreationSaga", () => { reasoningLevel: "high", cloudAutoPublish: true, cloudRtkEnabled: false, + cloudComputerUse: true, }); expect(result.success).toBe(true); @@ -173,6 +175,7 @@ describe("TaskCreationSaga", () => { prAuthorshipMode: "user", autoPublish: true, rtkEnabled: false, + computerUse: true, runSource: "manual", signalReportId: undefined, initialPermissionMode: "auto", @@ -733,6 +736,35 @@ describe("TaskCreationSaga", () => { }, ); + it("uses a cold run when desktop-relayed MCP servers are requested", async () => { + const createdTask = createTask(); + const startedTask = createTask({ latest_run: createRun() }); + const createTaskMock = vi.fn().mockResolvedValue(createdTask); + const createTaskRunMock = vi.fn().mockResolvedValue(createRun()); + const saga = makeSaga({ + createTask: createTaskMock, + createTaskRun: createTaskRunMock, + startTaskRun: vi.fn().mockResolvedValue(startedTask), + }); + const relayedMcpServers = [{ name: "local-database" }]; + + const result = await saga.run({ + content: "Ship the fix", + repository: "posthog/posthog", + workspaceMode: "cloud", + branch: "main", + relayedMcpServers, + }); + + expect(result.success).toBe(true); + expect(mockHost.takeWarmTaskLease).not.toHaveBeenCalled(); + expect(createTaskMock.mock.calls[0][0].branch).toBeUndefined(); + expect(createTaskRunMock).toHaveBeenCalledWith( + "task-123", + expect.objectContaining({ relayedMcpServers }), + ); + }); + it("reuses a warm run built from the selected custom image", async () => { mockHost.takeWarmTaskLease.mockReturnValue({ taskId: "warm-task", diff --git a/packages/core/src/task-detail/taskCreationSaga.ts b/packages/core/src/task-detail/taskCreationSaga.ts index d74a341d35..09d338ba75 100644 --- a/packages/core/src/task-detail/taskCreationSaga.ts +++ b/packages/core/src/task-detail/taskCreationSaga.ts @@ -404,6 +404,7 @@ export class TaskCreationSaga extends Saga< prAuthorshipMode, autoPublish: input.cloudAutoPublish, rtkEnabled: input.cloudRtkEnabled, + computerUse: input.cloudComputerUse, runSource: input.cloudRunSource ?? "manual", signalReportId: input.signalReportId, homeQuickAction: input.homeQuickActionLabel, @@ -685,6 +686,14 @@ export class TaskCreationSaga extends Saga< augmented, }; + if ( + input.cloudComputerUse || + input.importedMcpServers?.length || + input.relayedMcpServers?.length + ) { + return { ...base, suppressWarmReuse: true }; + } + const lease = input.repository ? this.deps.host.takeWarmTaskLease({ repository: input.repository, diff --git a/packages/core/src/task-detail/taskService.test.ts b/packages/core/src/task-detail/taskService.test.ts index 049e775170..49051f0394 100644 --- a/packages/core/src/task-detail/taskService.test.ts +++ b/packages/core/src/task-detail/taskService.test.ts @@ -3,6 +3,7 @@ import type { RootLogger } from "@posthog/di/logger"; import { describe, expect, it, vi } from "vitest"; import type { TaskCreationEffects } from "./taskCreationEffects"; import type { ITaskCreationHost } from "./taskCreationHost"; +import { TaskCreationSaga } from "./taskCreationSaga"; import { buildWorktreeAdoptionInput } from "./taskInput"; import { TaskService } from "./taskService"; @@ -17,7 +18,7 @@ const rootLogger = { scope: () => scopedLog, } as unknown as RootLogger; -function makeService(): TaskService { +function makeService(options?: { computerUse?: boolean }): TaskService { const host = { // The API client's createTask rejects so tests can observe that an input // made it past validation (failedStep lands on task_creation, not @@ -31,6 +32,7 @@ function makeService(): TaskService { sendRunCommand: vi.fn(), updateTask: vi.fn(), })), + isComputerUseEnabled: vi.fn(() => options?.computerUse ?? false), detectRepo: vi.fn(async () => null), getFolders: vi.fn(async () => []), addFolder: vi.fn(async () => ({ id: "folder-1", path: "/repo" })), @@ -74,4 +76,29 @@ describe("TaskService.createTask validation", () => { if (result.success) throw new Error("expected task_creation failure"); expect(result.failedStep).toBe("task_creation"); }); + + it("enables computer use on every opted-in cloud task", async () => { + const run = vi.spyOn(TaskCreationSaga.prototype, "run").mockResolvedValue({ + success: false, + error: "stop after input capture", + failedStep: "task_creation", + }); + + await makeService({ computerUse: true }).createTask( + { + content: "Use my desktop", + repository: "posthog/code", + workspaceMode: "cloud", + }, + undefined, + { skipCloudUsagePreflight: true }, + ); + + expect(run).toHaveBeenCalledWith( + expect.objectContaining({ + cloudComputerUse: true, + }), + ); + run.mockRestore(); + }); }); diff --git a/packages/core/src/task-detail/taskService.ts b/packages/core/src/task-detail/taskService.ts index 78e6e0c25f..65c57be071 100644 --- a/packages/core/src/task-detail/taskService.ts +++ b/packages/core/src/task-detail/taskService.ts @@ -74,6 +74,14 @@ export class TaskService { }; } + const effectiveInput: TaskCreationInput = + input.workspaceMode === "cloud" + ? { + ...input, + cloudComputerUse: this.host.isComputerUseEnabled(), + } + : input; + const posthogClient = await this.host.getAuthenticatedClient(); if (!posthogClient) { return { @@ -85,7 +93,10 @@ export class TaskService { // Backstop for callers that bypass useTaskCreation (e.g. inbox); the helper shows the modal. // Callers that already pre-flighted pass skipCloudUsagePreflight to avoid a second fetch. - if (!options?.skipCloudUsagePreflight && input.workspaceMode === "cloud") { + if ( + !options?.skipCloudUsagePreflight && + effectiveInput.workspaceMode === "cloud" + ) { try { await this.host.assertCloudUsageAvailable(); } catch (error) { @@ -112,7 +123,7 @@ export class TaskService { onTaskReady: onTaskReady ? (output) => { this.effects.onWorkspaceCreated(output); - this.effects.onCreateSuccess(output, input); + this.effects.onCreateSuccess(output, effectiveInput); onTaskReady(output); } : undefined, @@ -120,12 +131,12 @@ export class TaskService { this.log, ); - const result = await saga.run(input); + const result = await saga.run(effectiveInput); if (result.success) { this.effects.onWorkspaceCreated(result.data); if (!onTaskReady) { - this.effects.onCreateSuccess(result.data, input); + this.effects.onCreateSuccess(result.data, effectiveInput); } } diff --git a/packages/shared/src/task-creation-domain.ts b/packages/shared/src/task-creation-domain.ts index 8b073027a9..cf94f5c000 100644 --- a/packages/shared/src/task-creation-domain.ts +++ b/packages/shared/src/task-creation-domain.ts @@ -52,6 +52,7 @@ export interface TaskCreationInput { * meaningful: it opts the run out of the server-side default (enabled). */ cloudRtkEnabled?: boolean; + cloudComputerUse?: boolean; signalReportId?: string; additionalDirectories?: string[]; /** diff --git a/packages/ui/src/features/settings/sections/AdvancedSettings.tsx b/packages/ui/src/features/settings/sections/AdvancedSettings.tsx index 9c27caf810..804884b73a 100644 --- a/packages/ui/src/features/settings/sections/AdvancedSettings.tsx +++ b/packages/ui/src/features/settings/sections/AdvancedSettings.tsx @@ -100,7 +100,7 @@ export function AdvancedSettings() {