From 54efcf0a13467e086160b5794710dd945a670c94 Mon Sep 17 00:00:00 2001 From: Richard Solomou Date: Sun, 19 Jul 2026 16:06:29 +0300 Subject: [PATCH 1/3] feat(agent): relay computer use to cloud tasks Generated-By: PostHog Code Task-Id: c424306e-3221-4201-bbce-83723deb6a13 --- docs/COMPUTER-USE.md | 18 +++--- docs/cloud-mcp-relay.md | 10 +++- packages/agent/package.json | 4 ++ .../codex-app-server/local-tools-mcp.test.ts | 16 ++++++ .../codex-app-server/local-tools-mcp.ts | 43 ++------------ .../src/adapters/local-tools/stdio-server.ts | 57 +++++++++++++++++++ packages/agent/tsup.config.ts | 1 + .../core/src/local-mcp/localMcpImport.test.ts | 21 ++++++- packages/core/src/local-mcp/localMcpImport.ts | 21 ++++++- .../core/src/task-detail/taskCreationHost.ts | 1 + .../core/src/task-detail/taskService.test.ts | 30 +++++++++- packages/core/src/task-detail/taskService.ts | 23 ++++++-- packages/shared/src/index.ts | 1 + packages/shared/src/local-mcp-domain.ts | 2 + .../settings/sections/AdvancedSettings.tsx | 2 +- .../task-detail/taskCreationHostImpl.ts | 5 ++ .../src/services/mcp-relay/mcp-relay.test.ts | 25 ++++++++ .../src/services/mcp-relay/mcp-relay.ts | 33 +++++++++-- 18 files changed, 252 insertions(+), 61 deletions(-) create mode 100644 packages/agent/src/adapters/local-tools/stdio-server.ts diff --git a/docs/COMPUTER-USE.md b/docs/COMPUTER-USE.md index 53a91e085e..d52d5b485f 100644 --- a/docs/COMPUTER-USE.md +++ b/docs/COMPUTER-USE.md @@ -2,37 +2,39 @@ # Computer use -PostHog Code can give local agent sessions tools to see and control the macOS desktop across supported agent adapters. +PostHog Code can give local agent sessions and cloud tasks tools to see and control the macOS desktop across supported agent adapters. ## Enable it 1. Open **Settings → Advanced**. 2. Enable **Computer use**. -3. Start a new local session. +3. Start a new local session or cloud task. 4. Grant Screen Recording and Accessibility access when macOS requests it. -The setting applies when a session starts. Existing sessions are unchanged. +The setting applies when a session or cloud task starts. Existing runs are unchanged. ## Behavior - Computer use is opt-in and disabled by default. -- It is available only to local sessions on macOS. +- It is available to local sessions on macOS and cloud tasks created from the desktop app. - Agents can capture the desktop, list visible applications, open or focus applications, click coordinates, type text, and press keys. - Claude, Codex, and future adapters receive the same PostHog-owned MCP tools rather than adapter-specific computer-control implementations. -- Cloud sessions and unsupported operating systems do not receive the tools. +- Cloud tasks relay computer actions back to the desktop app, which must remain open and connected for the run to keep using them. +- Unsupported operating systems do not receive the tools. - Tool calls use the existing MCP tool-call and approval pipeline. ## Safety - Review the screen before approving an action and verify the result after it runs. - Do not ask an agent to enter passwords, tokens, recovery codes, or other secrets. -- Disable the setting and start a new session to remove the tools. +- Cloud computer actions require approval on the connected desktop unless the action was allowed for the rest of that run. +- Disable the setting before starting a new session or cloud task to omit the tools. Closing the desktop app disconnects computer use from active cloud tasks. - Revoke access in **System Settings → Privacy & Security → Screen Recording** or **Accessibility** to prevent native control. ## Scope -The initial implementation uses macOS system utilities for screenshots, application launching, mouse input, and keyboard input. It does not provide computer control to cloud tasks because those tasks do not run on the user's computer. Remote computer use would require an explicit device connection, user-presence controls, and a separate security design. +The implementation uses macOS system utilities for screenshots, application launching, mouse input, and keyboard input. Cloud workers do not access the computer directly: PostHog Code designates a built-in MCP server for the cloud run, relays each request through the existing task command channel, and executes approved actions on the connected desktop. Relay designations are held in memory, so restarting the app disconnects active runs rather than silently reconnecting them. ## Implementation -Computer actions are registered in the shared local-tools MCP registry. The Claude adapter exposes that registry through its in-process MCP server, while the Codex adapter exposes the same registry through its stdio MCP server. Session metadata gates the tools by environment, operating system, and the user's opt-in setting. +Computer actions are registered in the shared local-tools MCP registry. The Claude adapter exposes that registry through its in-process MCP server, while the Codex adapter and cloud relay expose the same registry through the packaged stdio MCP server. Local session metadata and cloud run relay designations gate the tools by operating system and the user's opt-in setting. diff --git a/docs/cloud-mcp-relay.md b/docs/cloud-mcp-relay.md index ec098c067f..88885ea19b 100644 --- a/docs/cloud-mcp-relay.md +++ b/docs/cloud-mcp-relay.md @@ -1,3 +1,5 @@ + + # Design: relaying local MCP servers into cloud task runs Status: **implemented** (same PR as the import work), behind the same @@ -16,6 +18,12 @@ them, and relayed servers stop working for in-flight runs (they 503 after the liveness window) rather than surviving a handoff. Tool allowlisting for relayed servers persists for the session lifetime, not to the backend. +The built-in `posthog-code-computer-use` designation resolves to the packaged +PostHog local-tools MCP server instead of `~/.claude.json`. It is added to new +cloud runs when the desktop **Computer use** setting is enabled and exposes +only the macOS computer tools. The same relay approval policy applies, so tool +calls execute only while the creating desktop remains connected. + ## Problem Two classes of local MCP servers cannot be imported into a cloud sandbox by @@ -53,7 +61,7 @@ The MCP relay mirrors this with a new event/command pair. ## Architecture -``` +```text sandbox desktop ┌──────────────────────────────┐ ┌──────────────────────────────┐ │ Claude/Codex adapter │ │ McpRelayService │ diff --git a/packages/agent/package.json b/packages/agent/package.json index 07804467f5..06aca68ff1 100644 --- a/packages/agent/package.json +++ b/packages/agent/package.json @@ -72,6 +72,10 @@ "types": "./dist/adapters/claude/mcp/tool-metadata.d.ts", "import": "./dist/adapters/claude/mcp/tool-metadata.js" }, + "./adapters/local-tools/stdio-server": { + "types": "./dist/adapters/local-tools/stdio-server.d.ts", + "import": "./dist/adapters/local-tools/stdio-server.js" + }, "./execution-mode": { "types": "./dist/execution-mode.d.ts", "import": "./dist/execution-mode.js" diff --git a/packages/agent/src/adapters/codex-app-server/local-tools-mcp.test.ts b/packages/agent/src/adapters/codex-app-server/local-tools-mcp.test.ts index bf0ebeba3e..5c4d3f0bf6 100644 --- a/packages/agent/src/adapters/codex-app-server/local-tools-mcp.test.ts +++ b/packages/agent/src/adapters/codex-app-server/local-tools-mcp.test.ts @@ -1,5 +1,6 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { LOCAL_TOOLS_MCP_NAME } from "../local-tools"; +import { buildComputerUseStdioServer } from "../local-tools/stdio-server"; import { buildLocalToolsServer } from "./local-tools-mcp"; // The dist asset isn't on the walk-up path in unit tests, so make existsSync @@ -10,6 +11,21 @@ vi.mock("node:fs", async (importActual) => { }); describe("buildLocalToolsServer", () => { + it("builds the computer-only server for a macOS desktop relay", () => { + const server = buildComputerUseStdioServer("/repo", "darwin"); + + expect(server?.env.POSTHOG_LOCAL_TOOLS_ENABLED).toBe( + [ + "computer_screenshot", + "computer_list_applications", + "computer_open_application", + "computer_click", + "computer_type", + "computer_key", + ].join(","), + ); + }); + const saved = { sandbox: process.env.IS_SANDBOX, ghToken: process.env.GH_TOKEN, diff --git a/packages/agent/src/adapters/codex-app-server/local-tools-mcp.ts b/packages/agent/src/adapters/codex-app-server/local-tools-mcp.ts index afdcbcc6d1..da6d633201 100644 --- a/packages/agent/src/adapters/codex-app-server/local-tools-mcp.ts +++ b/packages/agent/src/adapters/codex-app-server/local-tools-mcp.ts @@ -6,15 +6,13 @@ */ import type { McpServerStdio } from "@agentclientprotocol/sdk"; -import { ghTokenEnv } from "@posthog/git/signed-commit"; import { resolveGithubToken } from "../../utils/github-token"; -import { resolveBundledMcpScript } from "../../utils/resolve-bundled-script"; import { enabledLocalTools, - LOCAL_TOOLS_MCP_NAME, type LocalToolCtx, type LocalToolGateMeta, } from "../local-tools"; +import { buildLocalToolsStdioServer } from "../local-tools/stdio-server"; import { resolveTaskId } from "../session-meta"; /** @@ -29,39 +27,6 @@ export interface LocalToolsMeta extends LocalToolGateMeta { baseBranch?: string; } -function toMcpServerStdio( - ctx: LocalToolCtx, - enabledNames: string[], -): McpServerStdio { - const scriptPath = resolveBundledMcpScript( - "adapters/codex-app-server/local-tools-mcp-server.js", - ); - const ctxBase64 = Buffer.from(JSON.stringify(ctx)).toString("base64"); - const env = [ - { name: "POSTHOG_LOCAL_TOOLS_CTX", value: ctxBase64 }, - { name: "POSTHOG_LOCAL_TOOLS_ENABLED", value: enabledNames.join(",") }, - // Codex spawns this command with ELECTRON_RUN_AS_NODE removed from its own - // env (spawn.ts). In packaged desktop installs process.execPath is the app - // binary, which boots the full app without this var. Inert on real node. - { name: "ELECTRON_RUN_AS_NODE", value: "1" }, - ]; - if (ctx.token) { - // Token also on the child env so its own git remote ops authenticate. - env.push( - ...Object.entries(ghTokenEnv(ctx.token)).map(([name, value]) => ({ - name, - value, - })), - ); - } - return { - name: LOCAL_TOOLS_MCP_NAME, - command: process.execPath, - args: [scriptPath], - env, - }; -} - /** * Returns the local-tools stdio server config to inject, or null when no tool's * gate passes (e.g. local/desktop run with no GH token). Tools self-gate via the @@ -87,8 +52,12 @@ export function buildLocalToolsServer( if (tools.length === 0) { return null; } - return toMcpServerStdio( + const server = buildLocalToolsStdioServer( toolCtx, tools.map((t) => t.name), ); + return { + ...server, + env: Object.entries(server.env).map(([name, value]) => ({ name, value })), + }; } diff --git a/packages/agent/src/adapters/local-tools/stdio-server.ts b/packages/agent/src/adapters/local-tools/stdio-server.ts new file mode 100644 index 0000000000..b2caaa2f1f --- /dev/null +++ b/packages/agent/src/adapters/local-tools/stdio-server.ts @@ -0,0 +1,57 @@ +import { ghTokenEnv } from "@posthog/git/signed-commit"; +import { resolveBundledMcpScript } from "../../utils/resolve-bundled-script"; +import { + LOCAL_TOOLS_MCP_NAME, + type LocalToolCtx, + type LocalToolGateMeta, +} from "./registry"; +import { computerUseTools } from "./tools/computer-use"; + +export interface LocalToolsStdioServer { + name: string; + command: string; + args: string[]; + env: Record; +} + +export function buildLocalToolsStdioServer( + ctx: LocalToolCtx, + enabledNames: string[], +): LocalToolsStdioServer { + const scriptPath = resolveBundledMcpScript( + "adapters/codex-app-server/local-tools-mcp-server.js", + ); + const env: Record = { + POSTHOG_LOCAL_TOOLS_CTX: Buffer.from(JSON.stringify(ctx)).toString( + "base64", + ), + POSTHOG_LOCAL_TOOLS_ENABLED: enabledNames.join(","), + ELECTRON_RUN_AS_NODE: "1", + }; + if (ctx.token) { + Object.assign(env, ghTokenEnv(ctx.token)); + } + return { + name: LOCAL_TOOLS_MCP_NAME, + command: process.execPath, + args: [scriptPath], + env, + }; +} + +export function buildComputerUseStdioServer( + cwd: string, + platform: NodeJS.Platform = process.platform, +): LocalToolsStdioServer | null { + const ctx: LocalToolCtx = { cwd, platform }; + const meta: LocalToolGateMeta = { + environment: "local", + computerUse: true, + }; + const tools = computerUseTools.filter((tool) => tool.isEnabled(ctx, meta)); + if (tools.length === 0) return null; + return buildLocalToolsStdioServer( + ctx, + tools.map((tool) => tool.name), + ); +} diff --git a/packages/agent/tsup.config.ts b/packages/agent/tsup.config.ts index 52c3bd2fbf..ad7cf1011c 100644 --- a/packages/agent/tsup.config.ts +++ b/packages/agent/tsup.config.ts @@ -124,6 +124,7 @@ export default defineConfig([ "src/adapters/codex-app-server/models.ts", "src/adapters/codex-app-server/local-tools-mcp-server.ts", "src/adapters/claude/mcp/tool-metadata.ts", + "src/adapters/local-tools/stdio-server.ts", "src/adapters/reasoning-effort.ts", "src/execution-mode.ts", "src/server/schemas.ts", diff --git a/packages/core/src/local-mcp/localMcpImport.test.ts b/packages/core/src/local-mcp/localMcpImport.test.ts index c792c5118e..58f929e19f 100644 --- a/packages/core/src/local-mcp/localMcpImport.test.ts +++ b/packages/core/src/local-mcp/localMcpImport.test.ts @@ -1,6 +1,10 @@ -import type { LocalMcpServerDescriptor } from "@posthog/shared"; +import { + CLOUD_COMPUTER_USE_MCP_NAME, + type LocalMcpServerDescriptor, +} from "@posthog/shared"; import { describe, expect, it } from "vitest"; import { + addCloudComputerUseRelay, classifyLocalMcpServer, isPrivateHostname, type LocalMcpCloudClassification, @@ -199,6 +203,21 @@ describe("LocalMcpImportService", () => { }); describe("partitionLocalMcpServersForRun", () => { + it("puts built-in computer use first in the relay designation", () => { + const result = addCloudComputerUseRelay([], true); + + expect(result).toEqual([{ name: CLOUD_COMPUTER_USE_MCP_NAME }]); + }); + + it("does not duplicate the built-in computer use relay name", () => { + const result = addCloudComputerUseRelay( + [{ name: CLOUD_COMPUTER_USE_MCP_NAME }], + true, + ); + + expect(result).toEqual([{ name: CLOUD_COMPUTER_USE_MCP_NAME }]); + }); + const importable = (name: string): LocalMcpCloudClassification => ({ name, availability: "importable", diff --git a/packages/core/src/local-mcp/localMcpImport.ts b/packages/core/src/local-mcp/localMcpImport.ts index bafab32ec2..1344a8a431 100644 --- a/packages/core/src/local-mcp/localMcpImport.ts +++ b/packages/core/src/local-mcp/localMcpImport.ts @@ -4,7 +4,11 @@ import type { CloudMcpServerRelayDesignation, LocalMcpServerDescriptor, } from "@posthog/shared"; -import { isPrivateIpv4Octets, isPrivateIpv6Literal } from "@posthog/shared"; +import { + CLOUD_COMPUTER_USE_MCP_NAME, + isPrivateIpv4Octets, + isPrivateIpv6Literal, +} from "@posthog/shared"; import { inject, injectable } from "inversify"; import { LOCAL_MCP_WORKSPACE_CLIENT } from "./identifiers"; @@ -151,6 +155,17 @@ export interface LocalMcpServersForRun { relayed: CloudMcpServerRelayDesignation[]; } +export function addCloudComputerUseRelay( + servers: CloudMcpServerRelayDesignation[], + enabled: boolean, +): CloudMcpServerRelayDesignation[] { + if (!enabled) return servers; + return [ + { name: CLOUD_COMPUTER_USE_MCP_NAME }, + ...servers.filter((server) => server.name !== CLOUD_COMPUTER_USE_MCP_NAME), + ].slice(0, MAX_RELAYED_MCP_SERVERS); +} + /** * Split classified local servers into the run-creation payload's imported and * relayed lists for the run's adapter. @@ -170,7 +185,7 @@ export function partitionLocalMcpServersForRun( const imported = relayImportable ? [] : servers.flatMap((server) => (server.remote ? [server.remote] : [])); - const relayed = servers + const relayedServers = servers .filter( (server) => server.availability === "requires_desktop" || @@ -185,8 +200,8 @@ export function partitionLocalMcpServersForRun( ? -1 : 1, ) - .slice(0, MAX_RELAYED_MCP_SERVERS) .map((server) => ({ name: server.name })); + const relayed = relayedServers.slice(0, MAX_RELAYED_MCP_SERVERS); return { imported, relayed }; } diff --git a/packages/core/src/task-detail/taskCreationHost.ts b/packages/core/src/task-detail/taskCreationHost.ts index a419ccf60a..4f72c7bcaf 100644 --- a/packages/core/src/task-detail/taskCreationHost.ts +++ b/packages/core/src/task-detail/taskCreationHost.ts @@ -76,6 +76,7 @@ export interface RecordClaudeCliImportArgs { export interface ITaskCreationHost { getAuthenticatedClient(): Promise; assertCloudUsageAvailable(): Promise; + isComputerUseEnabled(): boolean; getTaskDirectory(taskId: string, repoKey?: string): Promise; /** * Ensure a per-task scratch working directory exists for a repo-less channel diff --git a/packages/core/src/task-detail/taskService.test.ts b/packages/core/src/task-detail/taskService.test.ts index 049e775170..197da5757e 100644 --- a/packages/core/src/task-detail/taskService.test.ts +++ b/packages/core/src/task-detail/taskService.test.ts @@ -1,8 +1,10 @@ import type { SessionService } from "@posthog/core/sessions/sessionService"; import type { RootLogger } from "@posthog/di/logger"; +import { CLOUD_COMPUTER_USE_MCP_NAME } from "@posthog/shared"; import { describe, expect, it, vi } from "vitest"; import type { TaskCreationEffects } from "./taskCreationEffects"; import type { ITaskCreationHost } from "./taskCreationHost"; +import { TaskCreationSaga } from "./taskCreationSaga"; import { buildWorktreeAdoptionInput } from "./taskInput"; import { TaskService } from "./taskService"; @@ -17,7 +19,7 @@ const rootLogger = { scope: () => scopedLog, } as unknown as RootLogger; -function makeService(): TaskService { +function makeService(options?: { computerUse?: boolean }): TaskService { const host = { // The API client's createTask rejects so tests can observe that an input // made it past validation (failedStep lands on task_creation, not @@ -31,6 +33,7 @@ function makeService(): TaskService { sendRunCommand: vi.fn(), updateTask: vi.fn(), })), + isComputerUseEnabled: vi.fn(() => options?.computerUse ?? false), detectRepo: vi.fn(async () => null), getFolders: vi.fn(async () => []), addFolder: vi.fn(async () => ({ id: "folder-1", path: "/repo" })), @@ -74,4 +77,29 @@ describe("TaskService.createTask validation", () => { if (result.success) throw new Error("expected task_creation failure"); expect(result.failedStep).toBe("task_creation"); }); + + it("adds the computer-use relay to every cloud task", async () => { + const run = vi.spyOn(TaskCreationSaga.prototype, "run").mockResolvedValue({ + success: false, + error: "stop after input capture", + failedStep: "task_creation", + }); + + await makeService({ computerUse: true }).createTask( + { + content: "Use my desktop", + repository: "posthog/code", + workspaceMode: "cloud", + }, + undefined, + { skipCloudUsagePreflight: true }, + ); + + expect(run).toHaveBeenCalledWith( + expect.objectContaining({ + relayedMcpServers: [{ name: CLOUD_COMPUTER_USE_MCP_NAME }], + }), + ); + run.mockRestore(); + }); }); diff --git a/packages/core/src/task-detail/taskService.ts b/packages/core/src/task-detail/taskService.ts index 78e6e0c25f..2ad5eee44c 100644 --- a/packages/core/src/task-detail/taskService.ts +++ b/packages/core/src/task-detail/taskService.ts @@ -11,6 +11,7 @@ import type { } from "@posthog/shared"; import type { Task } from "@posthog/shared/domain-types"; import { inject, injectable } from "inversify"; +import { addCloudComputerUseRelay } from "../local-mcp/localMcpImport"; import { TASK_CREATION_EFFECTS, TASK_CREATION_HOST } from "./identifiers"; import type { TaskCreationEffects } from "./taskCreationEffects"; import type { ITaskCreationHost } from "./taskCreationHost"; @@ -74,6 +75,17 @@ export class TaskService { }; } + const effectiveInput: TaskCreationInput = + input.workspaceMode === "cloud" + ? { + ...input, + relayedMcpServers: addCloudComputerUseRelay( + input.relayedMcpServers ?? [], + this.host.isComputerUseEnabled(), + ), + } + : input; + const posthogClient = await this.host.getAuthenticatedClient(); if (!posthogClient) { return { @@ -85,7 +97,10 @@ export class TaskService { // Backstop for callers that bypass useTaskCreation (e.g. inbox); the helper shows the modal. // Callers that already pre-flighted pass skipCloudUsagePreflight to avoid a second fetch. - if (!options?.skipCloudUsagePreflight && input.workspaceMode === "cloud") { + if ( + !options?.skipCloudUsagePreflight && + effectiveInput.workspaceMode === "cloud" + ) { try { await this.host.assertCloudUsageAvailable(); } catch (error) { @@ -112,7 +127,7 @@ export class TaskService { onTaskReady: onTaskReady ? (output) => { this.effects.onWorkspaceCreated(output); - this.effects.onCreateSuccess(output, input); + this.effects.onCreateSuccess(output, effectiveInput); onTaskReady(output); } : undefined, @@ -120,12 +135,12 @@ export class TaskService { this.log, ); - const result = await saga.run(input); + const result = await saga.run(effectiveInput); if (result.success) { this.effects.onWorkspaceCreated(result.data); if (!onTaskReady) { - this.effects.onCreateSuccess(result.data, input); + this.effects.onCreateSuccess(result.data, effectiveInput); } } diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index cffe7a2b5d..eb354c8843 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -134,6 +134,7 @@ export type { LocalMcpServerScope, LocalMcpTransport, } from "./local-mcp-domain"; +export { CLOUD_COMPUTER_USE_MCP_NAME } from "./local-mcp-domain"; export { formatMention, type MentionSegment, diff --git a/packages/shared/src/local-mcp-domain.ts b/packages/shared/src/local-mcp-domain.ts index c779a7c95e..09c2b58a15 100644 --- a/packages/shared/src/local-mcp-domain.ts +++ b/packages/shared/src/local-mcp-domain.ts @@ -3,6 +3,8 @@ // reads the config from disk; @posthog/core classifies each server by whether // it can be imported into a cloud sandbox. +export const CLOUD_COMPUTER_USE_MCP_NAME = "posthog-code-computer-use"; + /** Where a local MCP server definition came from in ~/.claude.json. */ export type LocalMcpServerScope = "user" | "project"; diff --git a/packages/ui/src/features/settings/sections/AdvancedSettings.tsx b/packages/ui/src/features/settings/sections/AdvancedSettings.tsx index 9c27caf810..be0a154b0c 100644 --- a/packages/ui/src/features/settings/sections/AdvancedSettings.tsx +++ b/packages/ui/src/features/settings/sections/AdvancedSettings.tsx @@ -100,7 +100,7 @@ export function AdvancedSettings() { { } describe("McpRelayServiceImpl", () => { + it("connects the built-in computer-use server without local config", async () => { + const service = new BuiltInMcpRelayService(); + const execution = service.execute("run-1", CLOUD_COMPUTER_USE_MCP_NAME, { + jsonrpc: "2.0", + id: 1, + method: "tools/list", + }); + await flush(); + + service.transports[0].respond({ jsonrpc: "2.0", id: 1, result: {} }); + + await expect(execution).resolves.toEqual({ + payload: { jsonrpc: "2.0", id: 1, result: {} }, + }); + }); + it("rejects an unknown server name without connecting", async () => { const service = makeService("known"); diff --git a/packages/workspace-server/src/services/mcp-relay/mcp-relay.ts b/packages/workspace-server/src/services/mcp-relay/mcp-relay.ts index 3544da31e1..5ccf65c799 100644 --- a/packages/workspace-server/src/services/mcp-relay/mcp-relay.ts +++ b/packages/workspace-server/src/services/mcp-relay/mcp-relay.ts @@ -1,3 +1,4 @@ +import { homedir } from "node:os"; import { SSEClientTransport } from "@modelcontextprotocol/sdk/client/sse.js"; import { getDefaultEnvironment, @@ -11,11 +12,13 @@ import { loadUserClaudeJsonMcpServerEntries, parseClaudeJsonTransport, } from "@posthog/agent/adapters/claude/session/mcp-config"; +import { buildComputerUseStdioServer } from "@posthog/agent/adapters/local-tools/stdio-server"; import { ROOT_LOGGER, type RootLogger, type ScopedLogger, } from "@posthog/di/logger"; +import { CLOUD_COMPUTER_USE_MCP_NAME } from "@posthog/shared"; import { inject, injectable, preDestroy } from "inversify"; import type { McpRelayExecution, McpRelayService } from "./identifiers"; @@ -75,8 +78,13 @@ export class McpRelayServiceImpl implements McpRelayService { let connectionPromise = byServer?.get(server); if (!connectionPromise) { - const entry = this.loadServerEntries().find((e) => e.name === server); - if (!entry) { + const transport = this.createBuiltInTransport(server); + const entry = transport + ? undefined + : this.loadServerEntries().find( + (candidate) => candidate.name === server, + ); + if (!transport && !entry) { this.log.warn("Relay request for unknown local MCP server", { runId, server, @@ -88,7 +96,12 @@ export class McpRelayServiceImpl implements McpRelayService { }, }; } - connectionPromise = this.openConnection(runId, server, entry); + let resolvedTransport = transport; + if (!resolvedTransport) { + if (!entry) throw new Error(`Missing MCP server entry for ${server}`); + resolvedTransport = this.createTransport(entry); + } + connectionPromise = this.openConnection(runId, server, resolvedTransport); if (!byServer) { byServer = new Map(); this.connections.set(runId, byServer); @@ -178,12 +191,22 @@ export class McpRelayServiceImpl implements McpRelayService { ); } + protected createBuiltInTransport(server: string): Transport | null { + if (server !== CLOUD_COMPUTER_USE_MCP_NAME) return null; + const config = buildComputerUseStdioServer(homedir()); + if (!config) return null; + return new StdioClientTransport({ + command: config.command, + args: config.args, + env: { ...getDefaultEnvironment(), ...config.env }, + }); + } + private async openConnection( runId: string, server: string, - entry: ClaudeJsonMcpServerEntry, + transport: Transport, ): Promise { - const transport = this.createTransport(entry); const connection: RelayConnection = { transport, nextLocalId: 1, From 7ebf612f62273c2eb4a695f23d134a26d2e83dab Mon Sep 17 00:00:00 2001 From: Richard Solomou Date: Sun, 19 Jul 2026 16:37:03 +0300 Subject: [PATCH 2/3] fix(agent): preserve cloud MCP relays Generated-By: PostHog Code Task-Id: c6d78416-85c8-4304-bd69-e309562468c4 --- docs/cloud-mcp-relay.md | 4 ++- .../codex-app-server/local-tools-mcp.test.ts | 12 +++++-- .../src/task-detail/taskCreationSaga.test.ts | 31 +++++++++++++++++++ .../core/src/task-detail/taskCreationSaga.ts | 4 +++ 4 files changed, 47 insertions(+), 4 deletions(-) diff --git a/docs/cloud-mcp-relay.md b/docs/cloud-mcp-relay.md index 88885ea19b..a34aa73c51 100644 --- a/docs/cloud-mcp-relay.md +++ b/docs/cloud-mcp-relay.md @@ -22,7 +22,9 @@ The built-in `posthog-code-computer-use` designation resolves to the packaged PostHog local-tools MCP server instead of `~/.claude.json`. It is added to new cloud runs when the desktop **Computer use** setting is enabled and exposes only the macOS computer tools. The same relay approval policy applies, so tool -calls execute only while the creating desktop remains connected. +calls execute only while the creating desktop remains connected. Runs with +imported or relayed MCP servers skip pre-warmed sandbox reuse because the +server list is not known when the warm run starts. ## Problem diff --git a/packages/agent/src/adapters/codex-app-server/local-tools-mcp.test.ts b/packages/agent/src/adapters/codex-app-server/local-tools-mcp.test.ts index 5c4d3f0bf6..f9d6c866d2 100644 --- a/packages/agent/src/adapters/codex-app-server/local-tools-mcp.test.ts +++ b/packages/agent/src/adapters/codex-app-server/local-tools-mcp.test.ts @@ -3,11 +3,17 @@ import { LOCAL_TOOLS_MCP_NAME } from "../local-tools"; import { buildComputerUseStdioServer } from "../local-tools/stdio-server"; import { buildLocalToolsServer } from "./local-tools-mcp"; -// The dist asset isn't on the walk-up path in unit tests, so make existsSync -// succeed; nothing spawns the script — we only inspect the path. +// Isolate bundled-script resolution and the sandbox env file; the test only +// inspects the generated stdio config. vi.mock("node:fs", async (importActual) => { const actual = await importActual(); - return { ...actual, existsSync: vi.fn().mockReturnValue(true) }; + return { + ...actual, + existsSync: vi.fn().mockReturnValue(true), + readFileSync: vi.fn(() => { + throw new Error("sandbox env file unavailable"); + }), + }; }); describe("buildLocalToolsServer", () => { diff --git a/packages/core/src/task-detail/taskCreationSaga.test.ts b/packages/core/src/task-detail/taskCreationSaga.test.ts index 6a06da8206..48cfcbe4d2 100644 --- a/packages/core/src/task-detail/taskCreationSaga.test.ts +++ b/packages/core/src/task-detail/taskCreationSaga.test.ts @@ -1,4 +1,5 @@ import type { SessionService } from "@posthog/core/sessions/sessionService"; +import { CLOUD_COMPUTER_USE_MCP_NAME } from "@posthog/shared"; import type { Task, TaskRun } from "@posthog/shared/domain-types"; import { beforeEach, describe, expect, it, vi } from "vitest"; import type { @@ -45,6 +46,7 @@ const sessionService = { disconnectFromTask: vi.fn(), rememberInitialCloudPrompt: vi.fn(), markTaskCreationInFlight: vi.fn(), + designateRelayedMcpServers: vi.fn().mockResolvedValue(undefined), } as unknown as SessionService; const createTask = (overrides: Partial = {}): Task => ({ @@ -733,6 +735,35 @@ describe("TaskCreationSaga", () => { }, ); + it("uses a cold run when desktop-relayed MCP servers are requested", async () => { + const createdTask = createTask(); + const startedTask = createTask({ latest_run: createRun() }); + const createTaskMock = vi.fn().mockResolvedValue(createdTask); + const createTaskRunMock = vi.fn().mockResolvedValue(createRun()); + const saga = makeSaga({ + createTask: createTaskMock, + createTaskRun: createTaskRunMock, + startTaskRun: vi.fn().mockResolvedValue(startedTask), + }); + const relayedMcpServers = [{ name: CLOUD_COMPUTER_USE_MCP_NAME }]; + + const result = await saga.run({ + content: "Ship the fix", + repository: "posthog/posthog", + workspaceMode: "cloud", + branch: "main", + relayedMcpServers, + }); + + expect(result.success).toBe(true); + expect(mockHost.takeWarmTaskLease).not.toHaveBeenCalled(); + expect(createTaskMock.mock.calls[0][0].branch).toBeUndefined(); + expect(createTaskRunMock).toHaveBeenCalledWith( + "task-123", + expect.objectContaining({ relayedMcpServers }), + ); + }); + it("reuses a warm run built from the selected custom image", async () => { mockHost.takeWarmTaskLease.mockReturnValue({ taskId: "warm-task", diff --git a/packages/core/src/task-detail/taskCreationSaga.ts b/packages/core/src/task-detail/taskCreationSaga.ts index d74a341d35..0ba36ca7d9 100644 --- a/packages/core/src/task-detail/taskCreationSaga.ts +++ b/packages/core/src/task-detail/taskCreationSaga.ts @@ -685,6 +685,10 @@ export class TaskCreationSaga extends Saga< augmented, }; + if (input.importedMcpServers?.length || input.relayedMcpServers?.length) { + return { ...base, suppressWarmReuse: true }; + } + const lease = input.repository ? this.deps.host.takeWarmTaskLease({ repository: input.repository, From 98441cb45c88608ff448408fb908d5b8e69f7a03 Mon Sep 17 00:00:00 2001 From: Richard Solomou Date: Sun, 19 Jul 2026 17:14:35 +0300 Subject: [PATCH 3/3] feat(agent): give cloud tasks isolated computer use Generated-By: PostHog Code Task-Id: c6d78416-85c8-4304-bd69-e309562468c4 --- docs/COMPUTER-USE.md | 14 +- docs/cloud-mcp-relay.md | 12 +- packages/agent/package.json | 4 - .../codex-app-server/local-tools-mcp.test.ts | 45 ++- .../codex-app-server/local-tools-mcp.ts | 46 ++- .../src/adapters/local-tools/registry.ts | 2 +- .../src/adapters/local-tools/stdio-server.ts | 57 ---- .../local-tools/tools/computer-use.test.ts | 41 ++- .../local-tools/tools/computer-use.ts | 292 +++++++++++------- packages/agent/src/server/agent-server.ts | 1 + packages/agent/src/server/bin.ts | 6 + packages/agent/src/server/types.ts | 1 + packages/agent/tsup.config.ts | 1 - .../api-client/src/posthog-client.test.ts | 2 + packages/api-client/src/posthog-client.ts | 4 + .../core/src/local-mcp/localMcpImport.test.ts | 21 +- packages/core/src/local-mcp/localMcpImport.ts | 21 +- .../src/task-detail/taskCreationApiClient.ts | 1 + .../src/task-detail/taskCreationSaga.test.ts | 5 +- .../core/src/task-detail/taskCreationSaga.ts | 7 +- .../core/src/task-detail/taskService.test.ts | 5 +- packages/core/src/task-detail/taskService.ts | 6 +- packages/shared/src/index.ts | 1 - packages/shared/src/local-mcp-domain.ts | 2 - packages/shared/src/task-creation-domain.ts | 1 + .../settings/sections/AdvancedSettings.tsx | 2 +- .../src/services/mcp-relay/mcp-relay.test.ts | 25 -- .../src/services/mcp-relay/mcp-relay.ts | 33 +- 28 files changed, 326 insertions(+), 332 deletions(-) delete mode 100644 packages/agent/src/adapters/local-tools/stdio-server.ts diff --git a/docs/COMPUTER-USE.md b/docs/COMPUTER-USE.md index d52d5b485f..dbc6c51c0a 100644 --- a/docs/COMPUTER-USE.md +++ b/docs/COMPUTER-USE.md @@ -2,24 +2,23 @@ # Computer use -PostHog Code can give local agent sessions and cloud tasks tools to see and control the macOS desktop across supported agent adapters. +PostHog Code can give local agent sessions tools to control the Mac and cloud tasks tools to control an isolated Linux desktop in their sandbox. ## Enable it 1. Open **Settings → Advanced**. 2. Enable **Computer use**. 3. Start a new local session or cloud task. -4. Grant Screen Recording and Accessibility access when macOS requests it. +4. For local sessions, grant Screen Recording and Accessibility access when macOS requests it. The setting applies when a session or cloud task starts. Existing runs are unchanged. ## Behavior - Computer use is opt-in and disabled by default. -- It is available to local sessions on macOS and cloud tasks created from the desktop app. +- Local sessions control the user's Mac. Cloud tasks control a virtual Linux desktop inside their own sandbox. - Agents can capture the desktop, list visible applications, open or focus applications, click coordinates, type text, and press keys. - Claude, Codex, and future adapters receive the same PostHog-owned MCP tools rather than adapter-specific computer-control implementations. -- Cloud tasks relay computer actions back to the desktop app, which must remain open and connected for the run to keep using them. - Unsupported operating systems do not receive the tools. - Tool calls use the existing MCP tool-call and approval pipeline. @@ -27,14 +26,13 @@ The setting applies when a session or cloud task starts. Existing runs are uncha - Review the screen before approving an action and verify the result after it runs. - Do not ask an agent to enter passwords, tokens, recovery codes, or other secrets. -- Cloud computer actions require approval on the connected desktop unless the action was allowed for the rest of that run. -- Disable the setting before starting a new session or cloud task to omit the tools. Closing the desktop app disconnects computer use from active cloud tasks. +- Disable the setting before starting a new session or cloud task to omit the tools. - Revoke access in **System Settings → Privacy & Security → Screen Recording** or **Accessibility** to prevent native control. ## Scope -The implementation uses macOS system utilities for screenshots, application launching, mouse input, and keyboard input. Cloud workers do not access the computer directly: PostHog Code designates a built-in MCP server for the cloud run, relays each request through the existing task command channel, and executes approved actions on the connected desktop. Relay designations are held in memory, so restarting the app disconnects active runs rather than silently reconnecting them. +Local sessions use macOS system utilities for screenshots, application launching, mouse input, and keyboard input. Cloud tasks start a virtual X display and lightweight window manager in the sandbox, then use Linux desktop utilities for the same actions. The cloud desktop contains only the task sandbox and cannot access the user's computer. ## Implementation -Computer actions are registered in the shared local-tools MCP registry. The Claude adapter exposes that registry through its in-process MCP server, while the Codex adapter and cloud relay expose the same registry through the packaged stdio MCP server. Local session metadata and cloud run relay designations gate the tools by operating system and the user's opt-in setting. +Computer actions are registered in the shared local-tools MCP registry. The Claude adapter exposes that registry through its in-process MCP server, while the Codex adapter exposes the same registry through the packaged stdio MCP server. Session metadata gates the tools by environment, operating system, and the user's opt-in setting. diff --git a/docs/cloud-mcp-relay.md b/docs/cloud-mcp-relay.md index a34aa73c51..ec098c067f 100644 --- a/docs/cloud-mcp-relay.md +++ b/docs/cloud-mcp-relay.md @@ -1,5 +1,3 @@ - - # Design: relaying local MCP servers into cloud task runs Status: **implemented** (same PR as the import work), behind the same @@ -18,14 +16,6 @@ them, and relayed servers stop working for in-flight runs (they 503 after the liveness window) rather than surviving a handoff. Tool allowlisting for relayed servers persists for the session lifetime, not to the backend. -The built-in `posthog-code-computer-use` designation resolves to the packaged -PostHog local-tools MCP server instead of `~/.claude.json`. It is added to new -cloud runs when the desktop **Computer use** setting is enabled and exposes -only the macOS computer tools. The same relay approval policy applies, so tool -calls execute only while the creating desktop remains connected. Runs with -imported or relayed MCP servers skip pre-warmed sandbox reuse because the -server list is not known when the warm run starts. - ## Problem Two classes of local MCP servers cannot be imported into a cloud sandbox by @@ -63,7 +53,7 @@ The MCP relay mirrors this with a new event/command pair. ## Architecture -```text +``` sandbox desktop ┌──────────────────────────────┐ ┌──────────────────────────────┐ │ Claude/Codex adapter │ │ McpRelayService │ diff --git a/packages/agent/package.json b/packages/agent/package.json index 06aca68ff1..07804467f5 100644 --- a/packages/agent/package.json +++ b/packages/agent/package.json @@ -72,10 +72,6 @@ "types": "./dist/adapters/claude/mcp/tool-metadata.d.ts", "import": "./dist/adapters/claude/mcp/tool-metadata.js" }, - "./adapters/local-tools/stdio-server": { - "types": "./dist/adapters/local-tools/stdio-server.d.ts", - "import": "./dist/adapters/local-tools/stdio-server.js" - }, "./execution-mode": { "types": "./dist/execution-mode.d.ts", "import": "./dist/execution-mode.js" diff --git a/packages/agent/src/adapters/codex-app-server/local-tools-mcp.test.ts b/packages/agent/src/adapters/codex-app-server/local-tools-mcp.test.ts index f9d6c866d2..d48af2c81f 100644 --- a/packages/agent/src/adapters/codex-app-server/local-tools-mcp.test.ts +++ b/packages/agent/src/adapters/codex-app-server/local-tools-mcp.test.ts @@ -1,41 +1,20 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { LOCAL_TOOLS_MCP_NAME } from "../local-tools"; -import { buildComputerUseStdioServer } from "../local-tools/stdio-server"; import { buildLocalToolsServer } from "./local-tools-mcp"; -// Isolate bundled-script resolution and the sandbox env file; the test only -// inspects the generated stdio config. +// The dist asset isn't on the walk-up path in unit tests, so make existsSync +// succeed; nothing spawns the script — we only inspect the path. vi.mock("node:fs", async (importActual) => { const actual = await importActual(); - return { - ...actual, - existsSync: vi.fn().mockReturnValue(true), - readFileSync: vi.fn(() => { - throw new Error("sandbox env file unavailable"); - }), - }; + return { ...actual, existsSync: vi.fn().mockReturnValue(true) }; }); describe("buildLocalToolsServer", () => { - it("builds the computer-only server for a macOS desktop relay", () => { - const server = buildComputerUseStdioServer("/repo", "darwin"); - - expect(server?.env.POSTHOG_LOCAL_TOOLS_ENABLED).toBe( - [ - "computer_screenshot", - "computer_list_applications", - "computer_open_application", - "computer_click", - "computer_type", - "computer_key", - ].join(","), - ); - }); - const saved = { sandbox: process.env.IS_SANDBOX, ghToken: process.env.GH_TOKEN, githubToken: process.env.GITHUB_TOKEN, + display: process.env.DISPLAY, }; beforeEach(() => { @@ -44,12 +23,14 @@ describe("buildLocalToolsServer", () => { delete process.env.IS_SANDBOX; delete process.env.GH_TOKEN; delete process.env.GITHUB_TOKEN; + delete process.env.DISPLAY; }); afterEach(() => { restore("IS_SANDBOX", saved.sandbox); restore("GH_TOKEN", saved.ghToken); restore("GITHUB_TOKEN", saved.githubToken); + restore("DISPLAY", saved.display); }); function restore(key: string, value: string | undefined): void { @@ -174,6 +155,20 @@ describe("buildLocalToolsServer", () => { ); }); + it("exposes computer tools for opted-in cloud Linux sessions", () => { + process.env.DISPLAY = ":99"; + const server = buildLocalToolsServer( + { cwd: "/repo", platform: "linux" }, + { environment: "cloud", computerUse: true }, + ); + + const enabled = + server?.env.find((entry) => entry.name === "POSTHOG_LOCAL_TOOLS_ENABLED") + ?.value ?? ""; + expect(enabled.split(",")).toContain("computer_screenshot"); + expect(server?.env).toContainEqual({ name: "DISPLAY", value: ":99" }); + }); + it.each([ { platform: "linux" as const, diff --git a/packages/agent/src/adapters/codex-app-server/local-tools-mcp.ts b/packages/agent/src/adapters/codex-app-server/local-tools-mcp.ts index da6d633201..9d7432929d 100644 --- a/packages/agent/src/adapters/codex-app-server/local-tools-mcp.ts +++ b/packages/agent/src/adapters/codex-app-server/local-tools-mcp.ts @@ -6,13 +6,15 @@ */ import type { McpServerStdio } from "@agentclientprotocol/sdk"; +import { ghTokenEnv } from "@posthog/git/signed-commit"; import { resolveGithubToken } from "../../utils/github-token"; +import { resolveBundledMcpScript } from "../../utils/resolve-bundled-script"; import { enabledLocalTools, + LOCAL_TOOLS_MCP_NAME, type LocalToolCtx, type LocalToolGateMeta, } from "../local-tools"; -import { buildLocalToolsStdioServer } from "../local-tools/stdio-server"; import { resolveTaskId } from "../session-meta"; /** @@ -27,6 +29,42 @@ export interface LocalToolsMeta extends LocalToolGateMeta { baseBranch?: string; } +function toMcpServerStdio( + ctx: LocalToolCtx, + enabledNames: string[], +): McpServerStdio { + const scriptPath = resolveBundledMcpScript( + "adapters/codex-app-server/local-tools-mcp-server.js", + ); + const ctxBase64 = Buffer.from(JSON.stringify(ctx)).toString("base64"); + const env = [ + { name: "POSTHOG_LOCAL_TOOLS_CTX", value: ctxBase64 }, + { name: "POSTHOG_LOCAL_TOOLS_ENABLED", value: enabledNames.join(",") }, + // Codex spawns this command with ELECTRON_RUN_AS_NODE removed from its own + // env (spawn.ts). In packaged desktop installs process.execPath is the app + // binary, which boots the full app without this var. Inert on real node. + { name: "ELECTRON_RUN_AS_NODE", value: "1" }, + ]; + if (process.env.DISPLAY) { + env.push({ name: "DISPLAY", value: process.env.DISPLAY }); + } + if (ctx.token) { + // Token also on the child env so its own git remote ops authenticate. + env.push( + ...Object.entries(ghTokenEnv(ctx.token)).map(([name, value]) => ({ + name, + value, + })), + ); + } + return { + name: LOCAL_TOOLS_MCP_NAME, + command: process.execPath, + args: [scriptPath], + env, + }; +} + /** * Returns the local-tools stdio server config to inject, or null when no tool's * gate passes (e.g. local/desktop run with no GH token). Tools self-gate via the @@ -52,12 +90,8 @@ export function buildLocalToolsServer( if (tools.length === 0) { return null; } - const server = buildLocalToolsStdioServer( + return toMcpServerStdio( toolCtx, tools.map((t) => t.name), ); - return { - ...server, - env: Object.entries(server.env).map(([name, value]) => ({ name, value })), - }; } diff --git a/packages/agent/src/adapters/local-tools/registry.ts b/packages/agent/src/adapters/local-tools/registry.ts index 164e5d8e3e..20a011b8a5 100644 --- a/packages/agent/src/adapters/local-tools/registry.ts +++ b/packages/agent/src/adapters/local-tools/registry.ts @@ -31,7 +31,7 @@ export interface LocalToolGateMeta { channelMode?: boolean; /** Spoken narration is on for this session: enables the speak tool. */ spokenNarration?: boolean; - /** Desktop computer control is enabled for this local session. */ + /** Computer control is enabled for this session. */ computerUse?: boolean; } diff --git a/packages/agent/src/adapters/local-tools/stdio-server.ts b/packages/agent/src/adapters/local-tools/stdio-server.ts deleted file mode 100644 index b2caaa2f1f..0000000000 --- a/packages/agent/src/adapters/local-tools/stdio-server.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { ghTokenEnv } from "@posthog/git/signed-commit"; -import { resolveBundledMcpScript } from "../../utils/resolve-bundled-script"; -import { - LOCAL_TOOLS_MCP_NAME, - type LocalToolCtx, - type LocalToolGateMeta, -} from "./registry"; -import { computerUseTools } from "./tools/computer-use"; - -export interface LocalToolsStdioServer { - name: string; - command: string; - args: string[]; - env: Record; -} - -export function buildLocalToolsStdioServer( - ctx: LocalToolCtx, - enabledNames: string[], -): LocalToolsStdioServer { - const scriptPath = resolveBundledMcpScript( - "adapters/codex-app-server/local-tools-mcp-server.js", - ); - const env: Record = { - POSTHOG_LOCAL_TOOLS_CTX: Buffer.from(JSON.stringify(ctx)).toString( - "base64", - ), - POSTHOG_LOCAL_TOOLS_ENABLED: enabledNames.join(","), - ELECTRON_RUN_AS_NODE: "1", - }; - if (ctx.token) { - Object.assign(env, ghTokenEnv(ctx.token)); - } - return { - name: LOCAL_TOOLS_MCP_NAME, - command: process.execPath, - args: [scriptPath], - env, - }; -} - -export function buildComputerUseStdioServer( - cwd: string, - platform: NodeJS.Platform = process.platform, -): LocalToolsStdioServer | null { - const ctx: LocalToolCtx = { cwd, platform }; - const meta: LocalToolGateMeta = { - environment: "local", - computerUse: true, - }; - const tools = computerUseTools.filter((tool) => tool.isEnabled(ctx, meta)); - if (tools.length === 0) return null; - return buildLocalToolsStdioServer( - ctx, - tools.map((tool) => tool.name), - ); -} diff --git a/packages/agent/src/adapters/local-tools/tools/computer-use.test.ts b/packages/agent/src/adapters/local-tools/tools/computer-use.test.ts index d420bf998f..a8bbe101d3 100644 --- a/packages/agent/src/adapters/local-tools/tools/computer-use.test.ts +++ b/packages/agent/src/adapters/local-tools/tools/computer-use.test.ts @@ -1,4 +1,4 @@ -import { execFile } from "node:child_process"; +import { execFile, spawn } from "node:child_process"; import { readFile, unlink } from "node:fs/promises"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { @@ -10,18 +10,21 @@ import { computerUseTools, } from "./computer-use"; -vi.mock("node:child_process", () => ({ execFile: vi.fn() })); +vi.mock("node:child_process", () => ({ execFile: vi.fn(), spawn: vi.fn() })); vi.mock("node:fs/promises", () => ({ readFile: vi.fn(), unlink: vi.fn().mockResolvedValue(undefined), })); const mockedExecFile = vi.mocked(execFile); +const mockedSpawn = vi.mocked(spawn); const mockedReadFile = vi.mocked(readFile); const mockedUnlink = vi.mocked(unlink); const context = { cwd: "/repo", platform: "darwin" as const }; const enabledMeta = { environment: "local" as const, computerUse: true }; +const cloudContext = { cwd: "/repo", platform: "linux" as const }; +const cloudMeta = { environment: "cloud" as const, computerUse: true }; describe("computer use tools", () => { beforeEach(() => { @@ -33,6 +36,7 @@ describe("computer use tools", () => { } return undefined; }) as unknown as typeof execFile); + mockedSpawn.mockReturnValue({ unref: vi.fn() } as never); mockedReadFile.mockResolvedValue(Buffer.from("png")); mockedUnlink.mockResolvedValue(undefined); }); @@ -71,6 +75,12 @@ describe("computer use tools", () => { expect(computerScreenshotTool.isEnabled(context, enabledMeta)).toBe(true); }); + it("is enabled for opted-in cloud Linux sessions", () => { + expect(computerScreenshotTool.isEnabled(cloudContext, cloudMeta)).toBe( + true, + ); + }); + it("returns a screenshot image and removes the temporary file", async () => { const result = await computerScreenshotTool.handler(context, {}); @@ -130,6 +140,33 @@ describe("computer use tools", () => { ); }); + it("opens the cloud browser without invoking a shell", async () => { + const unref = vi.fn(); + mockedSpawn.mockReturnValueOnce({ unref } as never); + + await computerOpenApplicationTool.handler(cloudContext, { + application: "Browser", + }); + + expect(mockedSpawn).toHaveBeenCalledWith( + "/usr/bin/epiphany", + ["--new-window"], + { detached: true, stdio: "ignore" }, + ); + expect(unref).toHaveBeenCalledOnce(); + }); + + it("uses Linux desktop utilities in cloud sessions", async () => { + await computerClickTool.handler(cloudContext, { x: 100, y: 200 }); + + expect(mockedExecFile).toHaveBeenCalledWith( + "/usr/bin/xdotool", + ["mousemove", "100", "200", "click", "1"], + { encoding: "utf8" }, + expect.any(Function), + ); + }); + it.each([ [computerClickTool, { x: 100, y: 200 }], [computerTypeTool, { text: "hello" }], diff --git a/packages/agent/src/adapters/local-tools/tools/computer-use.ts b/packages/agent/src/adapters/local-tools/tools/computer-use.ts index d416f65fc0..d0062d3ae0 100644 --- a/packages/agent/src/adapters/local-tools/tools/computer-use.ts +++ b/packages/agent/src/adapters/local-tools/tools/computer-use.ts @@ -1,4 +1,4 @@ -import { execFile } from "node:child_process"; +import { execFile, spawn } from "node:child_process"; import { randomUUID } from "node:crypto"; import { readFile, unlink } from "node:fs/promises"; import { tmpdir } from "node:os"; @@ -12,12 +12,6 @@ import { } from "../registry"; const modifierSchema = z.enum(["command", "control", "option", "shift"]); -const MAX_SCREENSHOT_BYTES = 1_000_000; -const screenshotCompressionAttempts = [ - { maxDimension: 1600, quality: "75" }, - { maxDimension: 1200, quality: "60" }, - { maxDimension: 900, quality: "45" }, -] as const; const namedKeySchema = z.enum([ "enter", "tab", @@ -33,8 +27,13 @@ const namedKeySchema = z.enum([ "home", "end", ]); - -const keyCodes: Record, number> = { +const MAX_SCREENSHOT_BYTES = 1_000_000; +const screenshotCompressionAttempts = [ + { maxDimension: 1600, quality: "75" }, + { maxDimension: 1200, quality: "60" }, + { maxDimension: 900, quality: "45" }, +] as const; +const macKeyCodes: Record, number> = { enter: 36, tab: 48, escape: 53, @@ -49,16 +48,27 @@ const keyCodes: Record, number> = { home: 115, end: 119, }; +const linuxKeys: Record, string> = { + enter: "Return", + tab: "Tab", + escape: "Escape", + delete: "BackSpace", + space: "space", + arrow_up: "Up", + arrow_down: "Down", + arrow_left: "Left", + arrow_right: "Right", + page_up: "Page_Up", + page_down: "Page_Down", + home: "Home", + end: "End", +}; function run(file: string, args: string[]): Promise { return new Promise((resolve, reject) => { - execFile(file, args, { encoding: "utf8" }, (error, stdout) => { - if (error) { - reject(error); - } else { - resolve(stdout.trim()); - } - }); + execFile(file, args, { encoding: "utf8" }, (error, stdout) => + error ? reject(error) : resolve(stdout.trim()), + ); }); } @@ -66,14 +76,18 @@ function runAppleScript(script: string, args: string[] = []): Promise { return run("/usr/bin/osascript", ["-e", script, "--", ...args]); } +function platform(ctx: LocalToolCtx): NodeJS.Platform { + return ctx.platform ?? process.platform; +} + function computerUseEnabled( ctx: LocalToolCtx, meta: { environment?: "local" | "cloud"; computerUse?: boolean } | undefined, ): boolean { + if (meta?.computerUse !== true) return false; return ( - (ctx.platform ?? process.platform) === "darwin" && - meta?.environment === "local" && - meta.computerUse === true + (platform(ctx) === "darwin" && meta.environment === "local") || + (platform(ctx) === "linux" && meta.environment === "cloud") ); } @@ -98,70 +112,88 @@ async function resultFrom( } } +async function captureScreenshot(ctx: LocalToolCtx): Promise { + const id = randomUUID(); + const sourcePath = join(tmpdir(), `posthog-computer-${id}.png`); + const outputPath = join(tmpdir(), `posthog-computer-${id}.jpg`); + try { + if (platform(ctx) === "darwin") { + await run("/usr/sbin/screencapture", ["-x", "-t", "png", sourcePath]); + } else { + await run("/usr/bin/import", ["-window", "root", sourcePath]); + } + for (const attempt of screenshotCompressionAttempts) { + if (platform(ctx) === "darwin") { + await run("/usr/bin/sips", [ + "--resampleHeightWidthMax", + String(attempt.maxDimension), + "--setProperty", + "format", + "jpeg", + "--setProperty", + "formatOptions", + attempt.quality, + sourcePath, + "--out", + outputPath, + ]); + } else { + await run("/usr/bin/convert", [ + sourcePath, + "-resize", + `${attempt.maxDimension}x${attempt.maxDimension}>`, + "-quality", + attempt.quality, + outputPath, + ]); + } + const data = await readFile(outputPath); + if (data.byteLength <= MAX_SCREENSHOT_BYTES) { + return { + content: [ + { type: "text", text: "Captured the current display." }, + { + type: "image", + data: data.toString("base64"), + mimeType: "image/jpeg", + }, + ], + }; + } + } + throw new Error("Captured screenshot exceeds the 1 MB image limit"); + } finally { + await Promise.all([ + unlink(sourcePath).catch(() => undefined), + unlink(outputPath).catch(() => undefined), + ]); + } +} + export const computerScreenshotTool = defineLocalTool({ name: "computer_screenshot", description: - "Capture the user's current macOS display. Use this before clicking and after each action to verify the actual result. Requires Screen Recording permission for PostHog Code.", + "Capture the current computer display. Use this before clicking and after each action to verify the actual result.", schema: {}, alwaysLoad: true, isEnabled: computerUseEnabled, - handler: async () => - resultFrom(async () => { - const id = randomUUID(); - const sourcePath = join(tmpdir(), `posthog-computer-${id}.png`); - const outputPath = join(tmpdir(), `posthog-computer-${id}.jpg`); - try { - await run("/usr/sbin/screencapture", ["-x", "-t", "png", sourcePath]); - for (const attempt of screenshotCompressionAttempts) { - await run("/usr/bin/sips", [ - "--resampleHeightWidthMax", - String(attempt.maxDimension), - "--setProperty", - "format", - "jpeg", - "--setProperty", - "formatOptions", - attempt.quality, - sourcePath, - "--out", - outputPath, - ]); - const data = await readFile(outputPath); - if (data.byteLength <= MAX_SCREENSHOT_BYTES) { - return { - content: [ - { type: "text", text: "Captured the current display." }, - { - type: "image", - data: data.toString("base64"), - mimeType: "image/jpeg", - }, - ], - }; - } - } - throw new Error("Captured screenshot exceeds the 1 MB image limit"); - } finally { - await Promise.all([ - unlink(sourcePath).catch(() => undefined), - unlink(outputPath).catch(() => undefined), - ]); - } - }), + handler: async (ctx) => resultFrom(() => captureScreenshot(ctx)), }); export const computerListApplicationsTool = defineLocalTool({ name: "computer_list_applications", - description: - "List visible applications currently running on the user's Mac. Use this before focusing or opening an application when its exact name is uncertain.", + description: "List visible applications on the current computer.", schema: {}, alwaysLoad: true, isEnabled: computerUseEnabled, - handler: async () => + handler: async (ctx) => resultFrom(async () => { - const applications = await runAppleScript( - 'tell application "System Events" to get name of every application process whose background only is false', - ); + const applications = + platform(ctx) === "darwin" + ? await runAppleScript( + 'tell application "System Events" to get name of every application process whose background only is false', + ) + : await run("/usr/bin/wmctrl", ["-lx"]); return { content: [ { @@ -176,41 +208,63 @@ export const computerListApplicationsTool = defineLocalTool({ export const computerOpenApplicationTool = defineLocalTool({ name: "computer_open_application", description: - "Open or focus a macOS application by its display name, for example Safari, Slack, or Notes. Verify the result with computer_screenshot.", - schema: { - application: z.string().trim().min(1).max(120), - }, + "Open or focus an application by name. In cloud tasks, common names include Chrome and Terminal.", + schema: { application: z.string().trim().min(1).max(120) }, alwaysLoad: true, isEnabled: computerUseEnabled, - handler: async (_ctx, { application }) => + handler: async (ctx, { application }) => resultFrom(async () => { - await run("/usr/bin/open", ["-a", application]); - await runAppleScript( - 'on run argv\nset applicationName to item 1 of argv\ntell application "System Events" to set frontmost of process applicationName to true\nend run', - [application], - ); - return { - content: [{ type: "text", text: `Opened ${application}.` }], - }; + if (platform(ctx) === "darwin") { + await run("/usr/bin/open", ["-a", application]); + await runAppleScript( + 'on run argv\nset applicationName to item 1 of argv\ntell application "System Events" to set frontmost of process applicationName to true\nend run', + [application], + ); + } else { + const normalized = application.toLowerCase(); + const command = + normalized.includes("chrome") || normalized.includes("browser") + ? "/usr/bin/epiphany" + : normalized.includes("terminal") + ? "/usr/bin/xterm" + : "/usr/bin/gtk-launch"; + const args = + command === "/usr/bin/gtk-launch" + ? [application] + : command === "/usr/bin/epiphany" + ? ["--new-window"] + : []; + spawn(command, args, { detached: true, stdio: "ignore" }).unref(); + } + return { content: [{ type: "text", text: `Opened ${application}.` }] }; }), }); export const computerClickTool = defineLocalTool({ name: "computer_click", description: - "Click an absolute screen coordinate on the user's Mac. Take a screenshot first, click the center of the intended control, then take another screenshot. Requires Accessibility permission for PostHog Code.", + "Click an absolute screen coordinate. Take a screenshot before and after the click.", schema: { x: z.number().int().min(0).max(100_000), y: z.number().int().min(0).max(100_000), }, alwaysLoad: true, isEnabled: computerUseEnabled, - handler: async (_ctx, { x, y }) => + handler: async (ctx, { x, y }) => resultFrom(async () => { - await runAppleScript( - 'on run argv\nset clickX to item 1 of argv as integer\nset clickY to item 2 of argv as integer\ntell application "System Events" to click at {clickX, clickY}\nend run', - [String(x), String(y)], - ); + if (platform(ctx) === "darwin") + await runAppleScript( + 'on run argv\nset clickX to item 1 of argv as integer\nset clickY to item 2 of argv as integer\ntell application "System Events" to click at {clickX, clickY}\nend run', + [String(x), String(y)], + ); + else + await run("/usr/bin/xdotool", [ + "mousemove", + String(x), + String(y), + "click", + "1", + ]); return { content: [{ type: "text", text: `Clicked ${x}, ${y}.` }] }; }), }); @@ -218,18 +272,18 @@ export const computerClickTool = defineLocalTool({ export const computerTypeTool = defineLocalTool({ name: "computer_type", description: - "Type text into the currently focused macOS control. Click the target field first and never use this for passwords, tokens, recovery codes, or other secrets. Requires Accessibility permission for PostHog Code.", - schema: { - text: z.string().min(1).max(4_000), - }, + "Type text into the focused control. Never use this for passwords, tokens, recovery codes, or other secrets.", + schema: { text: z.string().min(1).max(4_000) }, alwaysLoad: true, isEnabled: computerUseEnabled, - handler: async (_ctx, { text }) => + handler: async (ctx, { text }) => resultFrom(async () => { - await runAppleScript( - 'on run argv\ntell application "System Events" to keystroke (item 1 of argv)\nend run', - [text], - ); + if (platform(ctx) === "darwin") + await runAppleScript( + 'on run argv\ntell application "System Events" to keystroke (item 1 of argv)\nend run', + [text], + ); + else await run("/usr/bin/xdotool", ["type", "--delay", "1", "--", text]); return { content: [{ type: "text", text: `Typed ${text.length} characters.` }], }; @@ -239,29 +293,45 @@ export const computerTypeTool = defineLocalTool({ export const computerKeyTool = defineLocalTool({ name: "computer_key", description: - "Press a named key or a single letter/number with optional modifiers on the user's Mac, for example command+l or enter. Requires Accessibility permission for PostHog Code.", + "Press a named key or a single letter/number with optional modifiers.", schema: { key: z.union([namedKeySchema, z.string().regex(/^[a-z0-9]$/i)]), modifiers: z.array(modifierSchema).max(4).default([]), }, alwaysLoad: true, isEnabled: computerUseEnabled, - handler: async (_ctx, { key, modifiers }) => + handler: async (ctx, { key, modifiers }) => resultFrom(async () => { - const modifierList = modifiers.map((modifier) => `${modifier} down`); - const usingClause = - modifierList.length > 0 ? ` using {${modifierList.join(", ")}}` : ""; - const script = - key in keyCodes - ? `tell application "System Events" to key code ${keyCodes[key as keyof typeof keyCodes]}${usingClause}` - : `tell application "System Events" to keystroke "${key.toLowerCase()}"${usingClause}`; - await runAppleScript(script); + if (platform(ctx) === "darwin") { + const modifierList = modifiers.map((modifier) => `${modifier} down`); + const usingClause = + modifierList.length > 0 ? ` using {${modifierList.join(", ")}}` : ""; + const script = + key in macKeyCodes + ? `tell application "System Events" to key code ${macKeyCodes[key as keyof typeof macKeyCodes]}${usingClause}` + : `tell application "System Events" to keystroke "${key.toLowerCase()}"${usingClause}`; + await runAppleScript(script); + } else { + const linuxModifiers = modifiers.map( + (modifier) => + ({ + command: "super", + control: "ctrl", + option: "alt", + shift: "shift", + })[modifier], + ); + await run("/usr/bin/xdotool", [ + "key", + [ + ...linuxModifiers, + linuxKeys[key as keyof typeof linuxKeys] ?? key.toLowerCase(), + ].join("+"), + ]); + } return { content: [ - { - type: "text", - text: `Pressed ${[...modifiers, key].join("+")}.`, - }, + { type: "text", text: `Pressed ${[...modifiers, key].join("+")}.` }, ], }; }), diff --git a/packages/agent/src/server/agent-server.ts b/packages/agent/src/server/agent-server.ts index 140863f0da..4181624bc1 100644 --- a/packages/agent/src/server/agent-server.ts +++ b/packages/agent/src/server/agent-server.ts @@ -1493,6 +1493,7 @@ export class AgentServer { taskRunId: payload.run_id, taskId: payload.task_id, environment: "cloud", + computerUse: this.config.computerUse === true, systemPrompt: sessionSystemPrompt, ...(this.config.model && { model: this.config.model }), allowedDomains: this.config.allowedDomains, diff --git a/packages/agent/src/server/bin.ts b/packages/agent/src/server/bin.ts index 6f5a85c6f6..affcd95a69 100644 --- a/packages/agent/src/server/bin.ts +++ b/packages/agent/src/server/bin.ts @@ -114,6 +114,7 @@ program "--relayMcpServers ", "Desktop-relayed MCP server names as JSON array (docs/cloud-mcp-relay.md)", ) + .option("--computerUse ", "Enable native cloud computer control") .option("--createPr ", "Whether this run may publish changes") .option( "--autoPublish ", @@ -142,6 +143,10 @@ program const env = envResult.data; const mode = options.mode === "background" ? "background" : "interactive"; + const computerUse = parseBooleanOption( + options.computerUse, + "--computerUse", + ); const createPr = parseBooleanOption(options.createPr, "--createPr"); const autoPublish = parseBooleanOption( options.autoPublish, @@ -208,6 +213,7 @@ program autoPublish, mcpServers, relayMcpServers, + computerUse, baseBranch: options.baseBranch, claudeCode, allowedDomains, diff --git a/packages/agent/src/server/types.ts b/packages/agent/src/server/types.ts index 460a455fed..77688e9820 100644 --- a/packages/agent/src/server/types.ts +++ b/packages/agent/src/server/types.ts @@ -39,6 +39,7 @@ export interface AgentServerConfig { * each name against local config at execution time. */ relayMcpServers?: string[]; + computerUse?: boolean; baseBranch?: string; claudeCode?: ClaudeCodeConfig; allowedDomains?: string[]; diff --git a/packages/agent/tsup.config.ts b/packages/agent/tsup.config.ts index ad7cf1011c..52c3bd2fbf 100644 --- a/packages/agent/tsup.config.ts +++ b/packages/agent/tsup.config.ts @@ -124,7 +124,6 @@ export default defineConfig([ "src/adapters/codex-app-server/models.ts", "src/adapters/codex-app-server/local-tools-mcp-server.ts", "src/adapters/claude/mcp/tool-metadata.ts", - "src/adapters/local-tools/stdio-server.ts", "src/adapters/reasoning-effort.ts", "src/execution-mode.ts", "src/server/schemas.ts", diff --git a/packages/api-client/src/posthog-client.test.ts b/packages/api-client/src/posthog-client.test.ts index 3e6509ac08..f8cbbd361f 100644 --- a/packages/api-client/src/posthog-client.test.ts +++ b/packages/api-client/src/posthog-client.test.ts @@ -223,6 +223,7 @@ describe("PostHogAPIClient", () => { model: "gpt-5.4", reasoningLevel: "high", initialPermissionMode: "auto", + computerUse: true, }), ).resolves.toEqual({ id: "run-123", environment: "cloud" }); @@ -238,6 +239,7 @@ describe("PostHogAPIClient", () => { model: "gpt-5.4", reasoning_effort: "high", initial_permission_mode: "auto", + computer_use: true, environment: "cloud", }), }, diff --git a/packages/api-client/src/posthog-client.ts b/packages/api-client/src/posthog-client.ts index a9c5e72318..5b60c1b33b 100644 --- a/packages/api-client/src/posthog-client.ts +++ b/packages/api-client/src/posthog-client.ts @@ -595,6 +595,7 @@ interface CloudRunOptions { autoPublish?: boolean; /** Only false is sent: opts the run out of rtk command-output compression. */ rtkEnabled?: boolean; + computerUse?: boolean; runSource?: CloudRunSource; signalReportId?: string; initialPermissionMode?: ExecutionMode; @@ -690,6 +691,9 @@ function buildCloudRunRequestBody( if (options?.rtkEnabled === false) { body.rtk_enabled = false; } + if (options?.computerUse === true) { + body.computer_use = true; + } if (options?.runSource) { body.run_source = options.runSource; } diff --git a/packages/core/src/local-mcp/localMcpImport.test.ts b/packages/core/src/local-mcp/localMcpImport.test.ts index 58f929e19f..c792c5118e 100644 --- a/packages/core/src/local-mcp/localMcpImport.test.ts +++ b/packages/core/src/local-mcp/localMcpImport.test.ts @@ -1,10 +1,6 @@ -import { - CLOUD_COMPUTER_USE_MCP_NAME, - type LocalMcpServerDescriptor, -} from "@posthog/shared"; +import type { LocalMcpServerDescriptor } from "@posthog/shared"; import { describe, expect, it } from "vitest"; import { - addCloudComputerUseRelay, classifyLocalMcpServer, isPrivateHostname, type LocalMcpCloudClassification, @@ -203,21 +199,6 @@ describe("LocalMcpImportService", () => { }); describe("partitionLocalMcpServersForRun", () => { - it("puts built-in computer use first in the relay designation", () => { - const result = addCloudComputerUseRelay([], true); - - expect(result).toEqual([{ name: CLOUD_COMPUTER_USE_MCP_NAME }]); - }); - - it("does not duplicate the built-in computer use relay name", () => { - const result = addCloudComputerUseRelay( - [{ name: CLOUD_COMPUTER_USE_MCP_NAME }], - true, - ); - - expect(result).toEqual([{ name: CLOUD_COMPUTER_USE_MCP_NAME }]); - }); - const importable = (name: string): LocalMcpCloudClassification => ({ name, availability: "importable", diff --git a/packages/core/src/local-mcp/localMcpImport.ts b/packages/core/src/local-mcp/localMcpImport.ts index 1344a8a431..bafab32ec2 100644 --- a/packages/core/src/local-mcp/localMcpImport.ts +++ b/packages/core/src/local-mcp/localMcpImport.ts @@ -4,11 +4,7 @@ import type { CloudMcpServerRelayDesignation, LocalMcpServerDescriptor, } from "@posthog/shared"; -import { - CLOUD_COMPUTER_USE_MCP_NAME, - isPrivateIpv4Octets, - isPrivateIpv6Literal, -} from "@posthog/shared"; +import { isPrivateIpv4Octets, isPrivateIpv6Literal } from "@posthog/shared"; import { inject, injectable } from "inversify"; import { LOCAL_MCP_WORKSPACE_CLIENT } from "./identifiers"; @@ -155,17 +151,6 @@ export interface LocalMcpServersForRun { relayed: CloudMcpServerRelayDesignation[]; } -export function addCloudComputerUseRelay( - servers: CloudMcpServerRelayDesignation[], - enabled: boolean, -): CloudMcpServerRelayDesignation[] { - if (!enabled) return servers; - return [ - { name: CLOUD_COMPUTER_USE_MCP_NAME }, - ...servers.filter((server) => server.name !== CLOUD_COMPUTER_USE_MCP_NAME), - ].slice(0, MAX_RELAYED_MCP_SERVERS); -} - /** * Split classified local servers into the run-creation payload's imported and * relayed lists for the run's adapter. @@ -185,7 +170,7 @@ export function partitionLocalMcpServersForRun( const imported = relayImportable ? [] : servers.flatMap((server) => (server.remote ? [server.remote] : [])); - const relayedServers = servers + const relayed = servers .filter( (server) => server.availability === "requires_desktop" || @@ -200,8 +185,8 @@ export function partitionLocalMcpServersForRun( ? -1 : 1, ) + .slice(0, MAX_RELAYED_MCP_SERVERS) .map((server) => ({ name: server.name })); - const relayed = relayedServers.slice(0, MAX_RELAYED_MCP_SERVERS); return { imported, relayed }; } diff --git a/packages/core/src/task-detail/taskCreationApiClient.ts b/packages/core/src/task-detail/taskCreationApiClient.ts index b09c5f75d2..9ecdf63bd8 100644 --- a/packages/core/src/task-detail/taskCreationApiClient.ts +++ b/packages/core/src/task-detail/taskCreationApiClient.ts @@ -19,6 +19,7 @@ export interface CreateTaskRunClientOptions { prAuthorshipMode?: PrAuthorshipMode; autoPublish?: boolean; rtkEnabled?: boolean; + computerUse?: boolean; runSource?: CloudRunSource; signalReportId?: string; initialPermissionMode?: string; diff --git a/packages/core/src/task-detail/taskCreationSaga.test.ts b/packages/core/src/task-detail/taskCreationSaga.test.ts index 48cfcbe4d2..b96a5fd299 100644 --- a/packages/core/src/task-detail/taskCreationSaga.test.ts +++ b/packages/core/src/task-detail/taskCreationSaga.test.ts @@ -1,5 +1,4 @@ import type { SessionService } from "@posthog/core/sessions/sessionService"; -import { CLOUD_COMPUTER_USE_MCP_NAME } from "@posthog/shared"; import type { Task, TaskRun } from "@posthog/shared/domain-types"; import { beforeEach, describe, expect, it, vi } from "vitest"; import type { @@ -157,6 +156,7 @@ describe("TaskCreationSaga", () => { reasoningLevel: "high", cloudAutoPublish: true, cloudRtkEnabled: false, + cloudComputerUse: true, }); expect(result.success).toBe(true); @@ -175,6 +175,7 @@ describe("TaskCreationSaga", () => { prAuthorshipMode: "user", autoPublish: true, rtkEnabled: false, + computerUse: true, runSource: "manual", signalReportId: undefined, initialPermissionMode: "auto", @@ -745,7 +746,7 @@ describe("TaskCreationSaga", () => { createTaskRun: createTaskRunMock, startTaskRun: vi.fn().mockResolvedValue(startedTask), }); - const relayedMcpServers = [{ name: CLOUD_COMPUTER_USE_MCP_NAME }]; + const relayedMcpServers = [{ name: "local-database" }]; const result = await saga.run({ content: "Ship the fix", diff --git a/packages/core/src/task-detail/taskCreationSaga.ts b/packages/core/src/task-detail/taskCreationSaga.ts index 0ba36ca7d9..09d338ba75 100644 --- a/packages/core/src/task-detail/taskCreationSaga.ts +++ b/packages/core/src/task-detail/taskCreationSaga.ts @@ -404,6 +404,7 @@ export class TaskCreationSaga extends Saga< prAuthorshipMode, autoPublish: input.cloudAutoPublish, rtkEnabled: input.cloudRtkEnabled, + computerUse: input.cloudComputerUse, runSource: input.cloudRunSource ?? "manual", signalReportId: input.signalReportId, homeQuickAction: input.homeQuickActionLabel, @@ -685,7 +686,11 @@ export class TaskCreationSaga extends Saga< augmented, }; - if (input.importedMcpServers?.length || input.relayedMcpServers?.length) { + if ( + input.cloudComputerUse || + input.importedMcpServers?.length || + input.relayedMcpServers?.length + ) { return { ...base, suppressWarmReuse: true }; } diff --git a/packages/core/src/task-detail/taskService.test.ts b/packages/core/src/task-detail/taskService.test.ts index 197da5757e..49051f0394 100644 --- a/packages/core/src/task-detail/taskService.test.ts +++ b/packages/core/src/task-detail/taskService.test.ts @@ -1,6 +1,5 @@ import type { SessionService } from "@posthog/core/sessions/sessionService"; import type { RootLogger } from "@posthog/di/logger"; -import { CLOUD_COMPUTER_USE_MCP_NAME } from "@posthog/shared"; import { describe, expect, it, vi } from "vitest"; import type { TaskCreationEffects } from "./taskCreationEffects"; import type { ITaskCreationHost } from "./taskCreationHost"; @@ -78,7 +77,7 @@ describe("TaskService.createTask validation", () => { expect(result.failedStep).toBe("task_creation"); }); - it("adds the computer-use relay to every cloud task", async () => { + it("enables computer use on every opted-in cloud task", async () => { const run = vi.spyOn(TaskCreationSaga.prototype, "run").mockResolvedValue({ success: false, error: "stop after input capture", @@ -97,7 +96,7 @@ describe("TaskService.createTask validation", () => { expect(run).toHaveBeenCalledWith( expect.objectContaining({ - relayedMcpServers: [{ name: CLOUD_COMPUTER_USE_MCP_NAME }], + cloudComputerUse: true, }), ); run.mockRestore(); diff --git a/packages/core/src/task-detail/taskService.ts b/packages/core/src/task-detail/taskService.ts index 2ad5eee44c..65c57be071 100644 --- a/packages/core/src/task-detail/taskService.ts +++ b/packages/core/src/task-detail/taskService.ts @@ -11,7 +11,6 @@ import type { } from "@posthog/shared"; import type { Task } from "@posthog/shared/domain-types"; import { inject, injectable } from "inversify"; -import { addCloudComputerUseRelay } from "../local-mcp/localMcpImport"; import { TASK_CREATION_EFFECTS, TASK_CREATION_HOST } from "./identifiers"; import type { TaskCreationEffects } from "./taskCreationEffects"; import type { ITaskCreationHost } from "./taskCreationHost"; @@ -79,10 +78,7 @@ export class TaskService { input.workspaceMode === "cloud" ? { ...input, - relayedMcpServers: addCloudComputerUseRelay( - input.relayedMcpServers ?? [], - this.host.isComputerUseEnabled(), - ), + cloudComputerUse: this.host.isComputerUseEnabled(), } : input; diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index eb354c8843..cffe7a2b5d 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -134,7 +134,6 @@ export type { LocalMcpServerScope, LocalMcpTransport, } from "./local-mcp-domain"; -export { CLOUD_COMPUTER_USE_MCP_NAME } from "./local-mcp-domain"; export { formatMention, type MentionSegment, diff --git a/packages/shared/src/local-mcp-domain.ts b/packages/shared/src/local-mcp-domain.ts index 09c2b58a15..c779a7c95e 100644 --- a/packages/shared/src/local-mcp-domain.ts +++ b/packages/shared/src/local-mcp-domain.ts @@ -3,8 +3,6 @@ // reads the config from disk; @posthog/core classifies each server by whether // it can be imported into a cloud sandbox. -export const CLOUD_COMPUTER_USE_MCP_NAME = "posthog-code-computer-use"; - /** Where a local MCP server definition came from in ~/.claude.json. */ export type LocalMcpServerScope = "user" | "project"; diff --git a/packages/shared/src/task-creation-domain.ts b/packages/shared/src/task-creation-domain.ts index 8b073027a9..cf94f5c000 100644 --- a/packages/shared/src/task-creation-domain.ts +++ b/packages/shared/src/task-creation-domain.ts @@ -52,6 +52,7 @@ export interface TaskCreationInput { * meaningful: it opts the run out of the server-side default (enabled). */ cloudRtkEnabled?: boolean; + cloudComputerUse?: boolean; signalReportId?: string; additionalDirectories?: string[]; /** diff --git a/packages/ui/src/features/settings/sections/AdvancedSettings.tsx b/packages/ui/src/features/settings/sections/AdvancedSettings.tsx index be0a154b0c..804884b73a 100644 --- a/packages/ui/src/features/settings/sections/AdvancedSettings.tsx +++ b/packages/ui/src/features/settings/sections/AdvancedSettings.tsx @@ -100,7 +100,7 @@ export function AdvancedSettings() { { } describe("McpRelayServiceImpl", () => { - it("connects the built-in computer-use server without local config", async () => { - const service = new BuiltInMcpRelayService(); - const execution = service.execute("run-1", CLOUD_COMPUTER_USE_MCP_NAME, { - jsonrpc: "2.0", - id: 1, - method: "tools/list", - }); - await flush(); - - service.transports[0].respond({ jsonrpc: "2.0", id: 1, result: {} }); - - await expect(execution).resolves.toEqual({ - payload: { jsonrpc: "2.0", id: 1, result: {} }, - }); - }); - it("rejects an unknown server name without connecting", async () => { const service = makeService("known"); diff --git a/packages/workspace-server/src/services/mcp-relay/mcp-relay.ts b/packages/workspace-server/src/services/mcp-relay/mcp-relay.ts index 5ccf65c799..3544da31e1 100644 --- a/packages/workspace-server/src/services/mcp-relay/mcp-relay.ts +++ b/packages/workspace-server/src/services/mcp-relay/mcp-relay.ts @@ -1,4 +1,3 @@ -import { homedir } from "node:os"; import { SSEClientTransport } from "@modelcontextprotocol/sdk/client/sse.js"; import { getDefaultEnvironment, @@ -12,13 +11,11 @@ import { loadUserClaudeJsonMcpServerEntries, parseClaudeJsonTransport, } from "@posthog/agent/adapters/claude/session/mcp-config"; -import { buildComputerUseStdioServer } from "@posthog/agent/adapters/local-tools/stdio-server"; import { ROOT_LOGGER, type RootLogger, type ScopedLogger, } from "@posthog/di/logger"; -import { CLOUD_COMPUTER_USE_MCP_NAME } from "@posthog/shared"; import { inject, injectable, preDestroy } from "inversify"; import type { McpRelayExecution, McpRelayService } from "./identifiers"; @@ -78,13 +75,8 @@ export class McpRelayServiceImpl implements McpRelayService { let connectionPromise = byServer?.get(server); if (!connectionPromise) { - const transport = this.createBuiltInTransport(server); - const entry = transport - ? undefined - : this.loadServerEntries().find( - (candidate) => candidate.name === server, - ); - if (!transport && !entry) { + const entry = this.loadServerEntries().find((e) => e.name === server); + if (!entry) { this.log.warn("Relay request for unknown local MCP server", { runId, server, @@ -96,12 +88,7 @@ export class McpRelayServiceImpl implements McpRelayService { }, }; } - let resolvedTransport = transport; - if (!resolvedTransport) { - if (!entry) throw new Error(`Missing MCP server entry for ${server}`); - resolvedTransport = this.createTransport(entry); - } - connectionPromise = this.openConnection(runId, server, resolvedTransport); + connectionPromise = this.openConnection(runId, server, entry); if (!byServer) { byServer = new Map(); this.connections.set(runId, byServer); @@ -191,22 +178,12 @@ export class McpRelayServiceImpl implements McpRelayService { ); } - protected createBuiltInTransport(server: string): Transport | null { - if (server !== CLOUD_COMPUTER_USE_MCP_NAME) return null; - const config = buildComputerUseStdioServer(homedir()); - if (!config) return null; - return new StdioClientTransport({ - command: config.command, - args: config.args, - env: { ...getDefaultEnvironment(), ...config.env }, - }); - } - private async openConnection( runId: string, server: string, - transport: Transport, + entry: ClaudeJsonMcpServerEntry, ): Promise { + const transport = this.createTransport(entry); const connection: RelayConnection = { transport, nextLocalId: 1,