feat(agent): add bounded local computer use

apps/code/electron.vite.config.ts

@@ -134,6 +134,10 @@ export default defineConfig(({ mode }) => {
         },
         rollupOptions: {
           input: {
+            "adapters/codex-app-server/local-tools-mcp-server": path.resolve(
+              __dirname,
+              "../../packages/agent/src/adapters/codex-app-server/local-tools-mcp-server.ts",
+            ),
             bootstrap: path.resolve(__dirname, "src/main/bootstrap.ts"),
             "workspace-server": require.resolve(
               "@posthog/workspace-server/serve",

apps/code/package.json

@@ -15,7 +15,7 @@
     "start": "electron-vite dev --watch",
     "start:debug": "electron-vite dev --watch --inspect=5858",
     "compile": "electron-vite build",
-    "package": "electron-vite build && electron-builder build --dir --config electron-builder.ts",
+    "package": "electron-vite build && node ../../packages/agent/build/verify-local-tools-mcp-server.mjs .vite/build/adapters/codex-app-server/local-tools-mcp-server.js && electron-builder build --dir --config electron-builder.ts",
     "package:dev": "FORCE_DEV_MODE=1 SKIP_NOTARIZE=1 electron-vite build && electron-builder build --dir --config electron-builder.ts",
     "make": "electron-vite build && electron-builder build --config electron-builder.ts",
     "make:linux": "bash scripts/build-linux-docker.sh",

docs/BROWSER-USE.md

@@ -1,21 +1,21 @@
 <!-- markdownlint-disable MD013 -->
 
-# Browser use
+# Browser automation
 
 PostHog Code can give local agent sessions browser automation tools through an isolated Playwright MCP server.
 
 ## Enable it
 
 1. Install Google Chrome.
 2. Open **Settings → Advanced**.
-3. Enable **Browser use**.
+3. Enable **Browser automation**.
 4. Start a new local session.
 
 The setting applies when a session starts. Existing sessions are unchanged.
 
 ## Behavior
 
-- Browser use is opt-in and disabled by default.
+- Browser automation is opt-in and disabled by default.
 - It is available only to local sessions.
 - Each session launches an isolated Chrome profile, so it does not inherit cookies or logins from the user's normal browser profile.
 - Tool calls and screenshots use the existing MCP tool-call pipeline.

docs/COMPUTER-USE.md

@@ -0,0 +1,38 @@
+<!-- markdownlint-disable MD013 -->
+
+# Computer use
+
+PostHog Code can give local agent sessions tools to see and control the macOS desktop across supported agent adapters.
+
+## Enable it
+
+1. Open **Settings → Advanced**.
+2. Enable **Computer use**.
+3. Start a new local session.
+4. Grant Screen Recording and Accessibility access when macOS requests it.
+
+The setting applies when a session starts. Existing sessions are unchanged.
+
+## Behavior
+
+- Computer use is opt-in and disabled by default.
+- It is available only to local sessions on macOS.
+- Agents can capture the desktop, list visible applications, open or focus applications, click coordinates, type text, and press keys.
+- Claude, Codex, and future adapters receive the same PostHog-owned MCP tools rather than adapter-specific computer-control implementations.
+- Cloud sessions and unsupported operating systems do not receive the tools.
+- Tool calls use the existing MCP tool-call and approval pipeline.
+
+## Safety
+
+- Review the screen before approving an action and verify the result after it runs.
+- Do not ask an agent to enter passwords, tokens, recovery codes, or other secrets.
+- Disable the setting and start a new session to remove the tools.
+- Revoke access in **System Settings → Privacy & Security → Screen Recording** or **Accessibility** to prevent native control.
+
+## Scope
+
+The initial implementation uses macOS system utilities for screenshots, application launching, mouse input, and keyboard input. It does not provide computer control to cloud tasks because those tasks do not run on the user's computer. Remote computer use would require an explicit device connection, user-presence controls, and a separate security design.
+
+## Implementation
+
+Computer actions are registered in the shared local-tools MCP registry. The Claude adapter exposes that registry through its in-process MCP server, while the Codex adapter exposes the same registry through its stdio MCP server. Session metadata gates the tools by environment, operating system, and the user's opt-in setting.

packages/agent/build/verify-local-tools-mcp-server.mjs

@@ -8,10 +8,12 @@ import { spawn } from "node:child_process";
 import { resolve } from "node:path";
 import { fileURLToPath } from "node:url";
 
-const script = resolve(
-  fileURLToPath(new URL(".", import.meta.url)),
-  "../dist/adapters/codex-app-server/local-tools-mcp-server.js",
-);
+const script = process.argv[2]
+  ? resolve(process.argv[2])
+  : resolve(
+      fileURLToPath(new URL(".", import.meta.url)),
+      "../dist/adapters/codex-app-server/local-tools-mcp-server.js",
+    );
 
 const ctx = Buffer.from(
   JSON.stringify({ cwd: "/tmp", taskId: "smoke", token: "smoke-token" }),

packages/agent/src/adapters/claude/claude-agent.ts

@@ -1892,6 +1892,7 @@ export class ClaudeAcpAgent extends BaseAcpAgent {
     const baseBranch = meta?.baseBranch;
     const environment = meta?.environment;
     const spokenNarration = resolveSpokenNarration(meta);
+    const computerUse = meta?.computerUse === true;
     const buildInProcessMcpServers = (): Record<
       string,
       McpSdkServerConfigWithInstance
@@ -1903,8 +1904,9 @@ export class ClaudeAcpAgent extends BaseAcpAgent {
           taskId,
           taskRunId: meta?.taskRunId,
           baseBranch,
+          platform: process.platform,
         },
-        { environment, spokenNarration },
+        { environment, spokenNarration, computerUse },
       );
       return server ? { [LOCAL_TOOLS_MCP_NAME]: server } : {};
     };

packages/agent/src/adapters/claude/mcp/local-tools.test.ts

@@ -52,6 +52,71 @@ describe("createLocalToolsMcpServer", () => {
     expect(server).toBeUndefined();
   });
 
+  it("exposes computer tools for opted-in local macOS sessions", async () => {
+    const server = createLocalToolsMcpServer(
+      { cwd: "/repo", platform: "darwin" },
+      { environment: "local", computerUse: true },
+    );
+    if (!server) {
+      throw new Error("expected the local-tools server to be registered");
+    }
+
+    const [clientTransport, serverTransport] =
+      InMemoryTransport.createLinkedPair();
+    await server.instance.connect(serverTransport);
+    const client = new Client({ name: "test", version: "1.0.0" });
+    await client.connect(clientTransport);
+
+    const { tools } = await client.listTools();
+    expect(tools.map((tool) => tool.name)).toContain("computer_screenshot");
+
+    await client.close();
+  });
+
+  it.each([
+    {
+      platform: "linux" as const,
+      environment: "local" as const,
+      enabled: true,
+    },
+    {
+      platform: "darwin" as const,
+      environment: "cloud" as const,
+      enabled: true,
+    },
+    {
+      platform: "darwin" as const,
+      environment: "local" as const,
+      enabled: false,
+    },
+  ])(
+    "does not expose computer tools for unsupported context %#",
+    async (input) => {
+      const server = createLocalToolsMcpServer(
+        { cwd: "/repo", platform: input.platform },
+        { environment: input.environment, computerUse: input.enabled },
+      );
+
+      if (!server) {
+        expect(server).toBeUndefined();
+        return;
+      }
+
+      const [clientTransport, serverTransport] =
+        InMemoryTransport.createLinkedPair();
+      await server.instance.connect(serverTransport);
+      const client = new Client({ name: "test", version: "1.0.0" });
+      await client.connect(clientTransport);
+
+      const { tools } = await client.listTools();
+      expect(tools.map((tool) => tool.name)).not.toContain(
+        "computer_screenshot",
+      );
+
+      await client.close();
+    },
+  );
+
   it("exposes git_signed_commit over MCP in a cloud run with a token", async () => {
     const server = createLocalToolsMcpServer(
       { cwd: "/repo", token: "ghs_x" },

packages/agent/src/adapters/claude/types.ts

@@ -200,6 +200,7 @@ export type NewSessionMeta = {
    * emits always (consumers gate playback), local stays silent.
    */
   spokenNarration?: boolean;
+  computerUse?: boolean;
   jsonSchema?: Record<string, unknown> | null;
   mcpToolApprovals?: McpToolApprovals;
   claudeCode?: {

packages/agent/src/adapters/codex-app-server/codex-app-server-agent.ts

@@ -93,6 +93,7 @@ type AppServerSessionMeta = {
   environment?: "local" | "cloud";
   channelMode?: boolean;
   spokenNarration?: boolean;
+  computerUse?: boolean;
   baseBranch?: string;
   nativeGoal?: NativeGoalState;
 };
@@ -540,6 +541,7 @@ export class CodexAppServerAgent extends BaseAcpAgent {
       environment: meta.environment,
       channelMode: meta.channelMode,
       spokenNarration: resolveSpokenNarration(meta),
+      computerUse: meta.computerUse,
       taskId: meta.taskId,
       taskRunId: meta.taskRunId,
       persistence: meta.persistence,

packages/agent/src/adapters/codex-app-server/local-tools-mcp-server.ts

@@ -34,8 +34,10 @@ if (!ctxEnv) {
 let parsed: {
   cwd: string;
   taskId?: string;
+  taskRunId?: string;
   token?: string;
   baseBranch?: string;
+  platform?: NodeJS.Platform;
 };
 try {
   parsed = JSON.parse(Buffer.from(ctxEnv, "base64").toString("utf-8"));
@@ -51,7 +53,9 @@ const ctx: LocalToolCtx = {
   cwd: parsed.cwd,
   token: parsed.token ?? readGithubTokenFromEnv(),
   taskId: parsed.taskId,
+  taskRunId: parsed.taskRunId,
   baseBranch: parsed.baseBranch,
+  platform: parsed.platform,
 };
 
 const enabledNames = (process.env.POSTHOG_LOCAL_TOOLS_ENABLED ?? "")
@@ -74,4 +78,6 @@ for (const t of tools) {
 }
 
 const transport = new StdioServerTransport();
-await server.connect(transport);
+server.connect(transport).catch((error) => {
+  die(`Failed to connect stdio transport: ${error}`);
+});

packages/agent/src/adapters/codex-app-server/local-tools-mcp.test.ts

@@ -82,6 +82,7 @@ describe("buildLocalToolsServer", () => {
     expect(ctx.taskId).toBe("task-1");
     expect(ctx.taskRunId).toBe("run-1");
     expect(ctx.baseBranch).toBe("master");
+    expect(ctx.platform).toBe(process.platform);
   });
 
   it("returns a server but omits token env vars when no token is present", () => {
@@ -130,4 +131,52 @@ describe("buildLocalToolsServer", () => {
       buildLocalToolsServer({ cwd: "/repo" }, { environment: "local" }),
     ).toBeNull();
   });
+
+  it("exposes computer tools for opted-in local macOS sessions", () => {
+    const server = buildLocalToolsServer(
+      { cwd: "/repo", platform: "darwin" },
+      { environment: "local", computerUse: true },
+    );
+
+    const enabled =
+      server?.env.find((entry) => entry.name === "POSTHOG_LOCAL_TOOLS_ENABLED")
+        ?.value ?? "";
+    expect(enabled.split(",")).toEqual(
+      expect.arrayContaining([
+        "computer_screenshot",
+        "computer_open_application",
+        "computer_click",
+        "computer_type",
+        "computer_key",
+      ]),
+    );
+  });
+
+  it.each([
+    {
+      platform: "linux" as const,
+      environment: "local" as const,
+      enabled: true,
+    },
+    {
+      platform: "darwin" as const,
+      environment: "cloud" as const,
+      enabled: true,
+    },
+    {
+      platform: "darwin" as const,
+      environment: "local" as const,
+      enabled: false,
+    },
+  ])("does not expose computer tools for unsupported context %#", (input) => {
+    const server = buildLocalToolsServer(
+      { cwd: "/repo", platform: input.platform },
+      { environment: input.environment, computerUse: input.enabled },
+    );
+
+    const enabled =
+      server?.env.find((entry) => entry.name === "POSTHOG_LOCAL_TOOLS_ENABLED")
+        ?.value ?? "";
+    expect(enabled.split(",")).not.toContain("computer_screenshot");
+  });
 });

packages/agent/src/adapters/codex-app-server/local-tools-mcp.ts

@@ -68,7 +68,7 @@ function toMcpServerStdio(
  * registry; the server is only injected when at least one passes.
  */
 export function buildLocalToolsServer(
-  ctx: { cwd?: string },
+  ctx: { cwd?: string; platform?: NodeJS.Platform },
   meta: LocalToolsMeta | undefined,
 ): McpServerStdio | null {
   const cwd = ctx.cwd;
@@ -81,6 +81,7 @@ export function buildLocalToolsServer(
     taskId: resolveTaskId(meta),
     taskRunId: meta?.taskRunId,
     baseBranch: meta?.baseBranch,
+    platform: ctx.platform ?? process.platform,
   };
   const tools = enabledLocalTools(toolCtx, meta);
   if (tools.length === 0) {

packages/agent/src/adapters/local-tools/index.ts

@@ -1,5 +1,6 @@
 import type { LocalTool, LocalToolCtx, LocalToolGateMeta } from "./registry";
 import { cloneRepoTool } from "./tools/clone-repo";
+import { computerUseTools } from "./tools/computer-use";
 import { listReposTool } from "./tools/list-repos";
 import { signedCommitTool } from "./tools/signed-commit";
 import { signedMergeTool } from "./tools/signed-merge";
@@ -23,6 +24,7 @@ export const LOCAL_TOOLS: LocalTool[] = [
   listReposTool,
   cloneRepoTool,
   speakTool,
+  ...computerUseTools,
 ];
 
 /** Tools whose gate passes for the given context — the set to actually expose. */

packages/agent/src/adapters/local-tools/registry.ts

@@ -12,6 +12,7 @@ export const LOCAL_TOOLS_MCP_NAME = "posthog-code-tools";
 /** Runtime context handed to every local tool's handler and gate. */
 export interface LocalToolCtx {
   cwd: string;
+  platform?: NodeJS.Platform;
   /** GitHub token available to the sandbox, if any. */
   token?: string;
   taskId?: string;
@@ -30,6 +31,8 @@ export interface LocalToolGateMeta {
   channelMode?: boolean;
   /** Spoken narration is on for this session: enables the speak tool. */
   spokenNarration?: boolean;
+  /** Desktop computer control is enabled for this local session. */
+  computerUse?: boolean;
 }
 
 /**
@@ -38,7 +41,20 @@ export interface LocalToolGateMeta {
  * both attach an open `_meta`).
  */
 export interface LocalToolResult {
-  content: { type: "text"; text: string }[];
+  content: Array<
+    | {
+        type: "text";
+        text: string;
+        data?: never;
+        mimeType?: never;
+      }
+    | {
+        type: "image";
+        text?: never;
+        data: string;
+        mimeType: string;
+      }
+  >;
   isError?: true;
   [key: string]: unknown;
 }