feat(computer-use): ship munim-computer-use instead of in-tree desktop MCP copies

.mcp.json

@@ -1,7 +1,8 @@
 {
   "mcpServers": {
     "mt-desktop": {
-      "command": "native/t3-desktop-mcp/.build/release/t3-desktop-mcp"
+      "command": "node",
+      "args": ["scripts/run-desktop-mcp.ts"]
     }
   }
 }

README.md

@@ -85,7 +85,7 @@ This table lists only the differences. Once T3 Code ships a feature MT Code had
 
 | Feature                                                                                                                                                                                                       | MT Code |                 T3 Code                 |
 | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :-----: | :-------------------------------------: |
-| **Computer Use** — agents click, type, screenshot, zoom, hover, and drive browser tabs on your desktop; also published as [munimtechnologies/computer-use](https://github.com/munimtechnologies/computer-use) |   ✅    |                   ❌                    |
+| **Computer Use** — agents click, type, screenshot, zoom, hover, and drive browser tabs on your desktop; powered by open-source [munim-computer-use](https://github.com/munimtechnologies/munim-computer-use) |   ✅    |                   ❌                    |
 | **Computer History** — opt-in activity timeline (not screenshots) that agents can reference                                                                                                                   |   ✅    |                   ❌                    |
 | **Agent-chosen computers** — `computer_list` / `computer_send` start a task on another connected machine (this Mac, SSH, T3 Connect, or a paired backend) without changing **Run on**                         |   ✅    |                   ❌                    |
 | **Resume on restart** — running threads and agents pick up after the app closes                                                                                                                               |   ✅    |                   ❌                    |

apps/desktop/src/app/DesktopApp.ts

@@ -15,6 +15,7 @@ import * as ElectronProtocol from "../electron/ElectronProtocol.ts";
 import * as ElectronSafeStorage from "../electron/ElectronSafeStorage.ts";
 import { installDesktopIpcHandlers } from "../ipc/DesktopIpcHandlers.ts";
 import * as ComputerHistoryManager from "../computerHistory/ComputerHistoryManager.ts";
+import { ensureChromeNativeHostRegistered } from "../computerUse/nativeHost.ts";
 import * as DesktopAppActivation from "./DesktopAppActivation.ts";
 import * as DesktopAppIdentity from "./DesktopAppIdentity.ts";
 import * as DesktopClerk from "./DesktopClerk.ts";
@@ -260,6 +261,11 @@ const bootstrap = Effect.gen(function* () {
           Effect.logWarning("Computer History daemon bootstrap skipped", { cause }),
       }),
     );
+    // Browser control needs MT Code's Chrome native host registered; do it now
+    // (in the background) so it works without a visit to Settings.
+    if (decoded.desktopControl.enabled && decoded.desktopControl.browserControlEnabled) {
+      void ensureChromeNativeHostRegistered();
+    }
   }).pipe(
     Effect.catch((cause) =>
       Effect.logWarning("Computer History daemon bootstrap skipped", { cause }),

apps/desktop/src/computerHistory/ComputerHistoryManager.ts

@@ -28,6 +28,7 @@ import * as Effect from "effect/Effect";
 import * as Layer from "effect/Layer";
 import * as Schema from "effect/Schema";
 
+import { mtcodeDesktopProfileEnv } from "@t3tools/shared/munimComputerUse";
 import { resolveDesktopMcpBinaryPathSync } from "./resolveBinary.ts";
 
 type DaemonState = {
@@ -273,12 +274,14 @@ export const make = Effect.gen(function* () {
 
     const binary = resolveDesktopMcpBinaryPathSync();
     if (!binary) {
-      await writeUnavailableStatus(root, "t3-desktop-mcp binary not found");
+      await writeUnavailableStatus(root, "munim-computer-use binary not found");
       return;
     }
 
     const generation = state.generation + 1;
     const child = spawn(binary, ["computer-history", "--root", root], {
+      // Same MT identity the MCP server runs under (see desktopMcpLaunch.ts).
+      env: { ...process.env, ...mtcodeDesktopProfileEnv() },
       stdio: ["ignore", "ignore", "pipe"],
       detached: false,
     });

apps/desktop/src/computerHistory/resolveBinary.ts

@@ -1,47 +1,116 @@
 // @effect-diagnostics nodeBuiltinImport:off
-import * as NodeFs from "node:fs";
+import * as NodeFS from "node:fs";
+import * as NodeOS from "node:os";
 import * as NodePath from "node:path";
-import { fileURLToPath } from "node:url";
+import * as NodeURL from "node:url";
 
-const DESKTOP_MCP = process.platform === "win32" ? "t3-desktop-mcp.exe" : "t3-desktop-mcp";
+import {
+  desktopMcpPathOverride,
+  MUNIM_COMPUTER_USE_EXTENSION_DIR,
+  MUNIM_COMPUTER_USE_RESOURCE_DIR,
+  munimComputerUseAssetKey,
+  munimComputerUseCacheDir,
+  munimComputerUseCheckoutBinaries,
+  munimComputerUseExecutableName,
+  parseMunimComputerUseManifest,
+  type MunimComputerUseAssetKey,
+  type MunimComputerUsePlatform,
+} from "@t3tools/shared/munimComputerUse";
+
+const here = NodePath.dirname(NodeURL.fileURLToPath(import.meta.url));
+
+function hostPlatform(): MunimComputerUsePlatform | undefined {
+  // oxlint-disable-next-line t3code/no-global-process-runtime -- Resolved once at startup, outside any Effect runtime.
+  const platform = process.platform;
+  return platform === "darwin" || platform === "win32" || platform === "linux"
+    ? platform
+    : undefined;
+}
+
+/** A local munim-computer-use checkout (dev): `$MUNIM_COMPUTER_USE_CHECKOUT` or `~/computer-use`. */
+function checkoutRoot(): string {
+  return (
+    process.env.MUNIM_COMPUTER_USE_CHECKOUT?.trim() ||
+    NodePath.join(NodeOS.homedir(), "computer-use")
+  );
+}
+
+/** Cache dir of the pinned release the desktop build fetched, when run from a checkout. */
+function fetchedCacheDir(key: MunimComputerUseAssetKey): string | undefined {
+  const candidates = [
+    NodePath.resolve(here, "../../../../native/munim-computer-use.json"),
+    NodePath.resolve(here, "../../../native/munim-computer-use.json"),
+  ];
+  for (const manifestPath of candidates) {
+    try {
+      const { version } = parseMunimComputerUseManifest(NodeFS.readFileSync(manifestPath, "utf8"));
+      return munimComputerUseCacheDir({
+        environment: process.env,
+        homeDir: NodeOS.homedir(),
+        version,
+        key,
+        join: NodePath.join,
+      });
+    } catch {
+      // Not a checkout, or no manifest: nothing was fetched.
+    }
+  }
+  return undefined;
+}
+
+/** `…/Resources/munim-computer-use` in a packaged app. */
+function packagedDir(): string | undefined {
+  return process.resourcesPath
+    ? NodePath.join(process.resourcesPath, MUNIM_COMPUTER_USE_RESOURCE_DIR)
+    : undefined;
+}
 
 /**
- * Locate the bundled/dev desktop MCP binary for Computer History daemon spawn.
- * Mirrors server resolution but stays sync for Electron main.
+ * Locate the munim-computer-use binary for Electron main (Computer History,
+ * Chrome native-host registration). Same order as the server's resolver:
+ * `MTCODE_DESKTOP_MCP_PATH` (then the deprecated `T3CODE_DESKTOP_MCP_PATH`),
+ * the packaged copy, a local checkout build, the fetched release.
  */
 export function resolveDesktopMcpBinaryPathSync(): string | undefined {
-  const override = process.env.T3CODE_DESKTOP_MCP_PATH;
-  if (override && NodeFs.existsSync(override)) return override;
-
-  const here = NodePath.dirname(fileURLToPath(import.meta.url));
-  const candidates =
-    process.platform === "darwin"
-      ? [
-          NodePath.resolve(
-            here,
-            "../../../../native/t3-desktop-mcp/.build/apple/Products/Release",
-            DESKTOP_MCP,
-          ),
-          NodePath.resolve(here, "../../../../native/t3-desktop-mcp/.build/release", DESKTOP_MCP),
-          NodePath.resolve(
-            here,
-            "../../../native/t3-desktop-mcp/.build/apple/Products/Release",
-            DESKTOP_MCP,
-          ),
-          NodePath.resolve(process.resourcesPath ?? "", "t3-desktop-mcp", DESKTOP_MCP),
-        ]
-      : [
-          NodePath.resolve(
-            here,
-            "../../../../native/t3-desktop-mcp-rs/target/release",
-            DESKTOP_MCP,
-          ),
-          NodePath.resolve(here, "../../../native/t3-desktop-mcp-rs/target/release", DESKTOP_MCP),
-          NodePath.resolve(process.resourcesPath ?? "", "t3-desktop-mcp", DESKTOP_MCP),
-        ];
+  const platform = hostPlatform();
+  if (!platform) return undefined;
+  const executable = munimComputerUseExecutableName(platform);
+
+  const override = desktopMcpPathOverride(process.env);
+  const packaged = packagedDir();
+  const fetched = fetchedCacheDir(
+    // oxlint-disable-next-line t3code/no-global-process-runtime -- Resolved once at startup, outside any Effect runtime.
+    munimComputerUseAssetKey(platform, process.arch === "arm64" ? "arm64" : "x64"),
+  );
+  const candidates = [
+    ...(override ? [override] : []),
+    ...(packaged ? [NodePath.join(packaged, executable)] : []),
+    ...munimComputerUseCheckoutBinaries(platform).map((parts) =>
+      NodePath.join(checkoutRoot(), ...parts),
+    ),
+    ...(fetched ? [NodePath.join(fetched, executable)] : []),
+  ];
 
   for (const candidate of candidates) {
-    if (NodeFs.existsSync(candidate)) return candidate;
+    if (NodeFS.existsSync(candidate)) return candidate;
+  }
+  return undefined;
+}
+
+/**
+ * The unpacked Chrome extension the user loads in chrome://extensions: the
+ * copy packaged beside the binary, else the fetched release, else a checkout.
+ */
+export function resolveChromeExtensionDirSync(): string | undefined {
+  const packaged = packagedDir();
+  const fetched = fetchedCacheDir("chrome-extension");
+  const candidates = [
+    ...(packaged ? [NodePath.join(packaged, MUNIM_COMPUTER_USE_EXTENSION_DIR)] : []),
+    ...(fetched ? [fetched] : []),
+    NodePath.join(checkoutRoot(), MUNIM_COMPUTER_USE_EXTENSION_DIR),
+  ];
+  for (const candidate of candidates) {
+    if (NodeFS.existsSync(NodePath.join(candidate, "manifest.json"))) return candidate;
   }
   return undefined;
 }

apps/desktop/src/computerUse/nativeHost.ts

@@ -0,0 +1,43 @@
+// @effect-diagnostics nodeBuiltinImport:off - one-shot child process from Electron main.
+import * as NodeChildProcess from "node:child_process";
+
+import { mtcodeDesktopProfileEnv } from "@t3tools/shared/munimComputerUse";
+
+import { resolveDesktopMcpBinaryPathSync } from "../computerHistory/resolveBinary.ts";
+
+let registration: Promise<boolean> | undefined;
+
+/**
+ * Register MT Code's Chrome native-messaging host (`com.munim.mtcode.desktop`)
+ * so the extension reaches the desktop-control server MT Code runs.
+ *
+ * munim-computer-use does the work (`install-native-host`): under the MT
+ * profile it writes a wrapper that relays into MT Code's own bridge socket, and
+ * the host manifest for every Chrome/Chromium profile directory (the registry
+ * on Windows). The binary rewrites nothing that is already current, so this is
+ * cheap to repeat; it runs at most once per app launch and resolves to whether
+ * a browser was registered.
+ */
+export function ensureChromeNativeHostRegistered(): Promise<boolean> {
+  registration ??= new Promise((resolve) => {
+    const binary = resolveDesktopMcpBinaryPathSync();
+    if (!binary) {
+      resolve(false);
+      return;
+    }
+    NodeChildProcess.execFile(
+      binary,
+      ["install-native-host", "--binary", binary],
+      { env: { ...process.env, ...mtcodeDesktopProfileEnv() }, timeout: 15_000 },
+      (error, _stdout, stderr) => {
+        if (error) {
+          process.stderr.write(
+            `[computer-use] native host registration failed: ${stderr || error.message}\n`,
+          );
+        }
+        resolve(!error);
+      },
+    );
+  });
+  return registration;
+}

apps/desktop/src/computerUse/permissions.test.ts

@@ -18,6 +18,11 @@ vi.mock("electron", () => ({
   },
 }));
 
+// Registration runs the real munim-computer-use binary, which writes into the
+// machine's Chrome profiles; a unit test must never do that.
+const registerNativeHost = vi.hoisted(() => vi.fn(async () => false));
+vi.mock("./nativeHost.ts", () => ({ ensureChromeNativeHostRegistered: registerNativeHost }));
+
 import * as Electron from "electron";
 
 import { openComputerUsePrivacySettings, readComputerUsePermissions } from "./permissions.ts";
@@ -35,6 +40,8 @@ describe("computerUse permissions", () => {
       );
       assert.equal(state.permissions[0]?.status, "denied");
       assert.equal(state.permissions[1]?.status, "denied");
+      // Reading the state is where the extension gets set up, so it registers the host.
+      assert.ok(registerNativeHost.mock.calls.length > 0);
     } finally {
       Object.defineProperty(process, "platform", { value: previous });
     }

apps/desktop/src/computerUse/permissions.ts

@@ -9,9 +9,15 @@ import type {
   DesktopComputerUsePermissionsState,
   DesktopComputerUsePrivacyPane,
 } from "@t3tools/contracts";
+import {
+  MTCODE_CHROME_EXTENSION_ID as CHROME_EXTENSION_ID,
+  MTCODE_CHROME_NATIVE_HOST,
+  MTCODE_DESKTOP_PROFILE,
+} from "@t3tools/shared/munimComputerUse";
 import * as Electron from "electron";
 
-const CHROME_EXTENSION_ID = "kgdolgnijopbghhomnblabjkmjhnoage";
+import { resolveChromeExtensionDirSync } from "../computerHistory/resolveBinary.ts";
+import { ensureChromeNativeHostRegistered } from "./nativeHost.ts";
 
 function platformTag(): DesktopComputerUsePermissionsState["platform"] {
   switch (process.platform) {
@@ -142,14 +148,11 @@ function chromeExtensionInstalledInPreferences(preferencesPath: string): boolean
 }
 
 /**
- * Host manifest names, newest first. Installs made before the rename carry
- * only the old one, and the installer writes both, so either counts as
- * registered.
+ * MT Code writes this manifest itself (see nativeHost.ts). Older manifests from
+ * the pre-munim-computer-use installer point at a relay for a bridge MT Code no
+ * longer binds, so they do not count.
  */
-const NATIVE_HOST_MANIFEST_NAMES = [
-  "com.munim.mtcode.desktop.json",
-  "com.t3tools.t3code.desktop.json",
-] as const;
+const NATIVE_HOST_MANIFEST_NAMES = [`${MTCODE_CHROME_NATIVE_HOST}.json`] as const;
 
 function anyHostManifest(directory: string): boolean {
   return NATIVE_HOST_MANIFEST_NAMES.some((name) => {
@@ -169,7 +172,7 @@ function nativeHostRegistered(root: string): boolean {
 function nativeHostRegisteredWindows(): boolean {
   const local = process.env.LOCALAPPDATA;
   if (!local) return false;
-  return anyHostManifest(NodePath.join(local, "t3-desktop-mcp"));
+  return anyHostManifest(NodePath.join(local, MTCODE_DESKTOP_PROFILE.name));
 }
 
 function resolveChromeExtensionStatus(): {
@@ -211,9 +214,12 @@ function resolveChromeExtensionStatus(): {
     };
   }
   if (hostRegistered) {
+    const extensionDir = resolveChromeExtensionDirSync();
     return {
       status: "missing",
-      detail: "Native host registered — load the unpacked extension in chrome://extensions",
+      detail: extensionDir
+        ? `Native host registered — load the unpacked extension from ${extensionDir} in chrome://extensions`
+        : "Native host registered — load the unpacked extension in chrome://extensions",
     };
   }
   return {
@@ -223,6 +229,9 @@ function resolveChromeExtensionStatus(): {
 }
 
 export function readComputerUsePermissions(): DesktopComputerUsePermissionsState {
+  // Opening Computer Use settings is where the extension gets set up, so make
+  // sure the host it needs is registered (a no-op once done this launch).
+  void ensureChromeNativeHostRegistered();
   const platform = platformTag();
   const chromeExtension = resolveChromeExtensionStatus();