diff --git a/packages/cli/src/lib/driver/commands/mouse.ts b/packages/cli/src/lib/driver/commands/mouse.ts index 62268b172..ee03fa854 100644 --- a/packages/cli/src/lib/driver/commands/mouse.ts +++ b/packages/cli/src/lib/driver/commands/mouse.ts @@ -1,5 +1,7 @@ import { z } from "zod"; +import { updateCursorOverlayPosition } from "../cursor-overlay.js"; +import type { DriverPage, DriverSessionManager } from "../session-manager.js"; import type { DriverCommandHandlers } from "./types.js"; const ButtonSchema = z.enum(["left", "right", "middle"]).optional(); @@ -17,6 +19,7 @@ export const mouseHandlers: DriverCommandHandlers = { .parse(params); assertXPathUnavailable(returnXPath); const page = await manager.activePage(); + await positionCursorOverlay(manager, page, x, y); await page.click(x, y, { ...(button === undefined ? {} : { button }), ...(clickCount === undefined ? {} : { clickCount }), @@ -34,6 +37,7 @@ export const mouseHandlers: DriverCommandHandlers = { .parse(params); assertXPathUnavailable(returnXPath); const page = await manager.activePage(); + await positionCursorOverlay(manager, page, x, y); await page.hover(x, y); return { hovered: true }; }, @@ -50,6 +54,7 @@ export const mouseHandlers: DriverCommandHandlers = { .parse(params); assertXPathUnavailable(returnXPath); const page = await manager.activePage(); + await positionCursorOverlay(manager, page, x, y); await page.scroll(x, y, deltaX, deltaY); return { scrolled: true }; }, @@ -69,15 +74,33 @@ export const mouseHandlers: DriverCommandHandlers = { .parse(params); assertXPathUnavailable(returnXPath); const page = await manager.activePage(); + await positionCursorOverlay(manager, page, fromX, fromY); await page.dragAndDrop(fromX, fromY, toX, toY, { ...(button === undefined ? {} : { button }), ...(delay === undefined ? {} : { delay }), ...(steps === undefined ? {} : { steps }), }); + await positionCursorOverlay(manager, page, toX, toY); return { dragged: true }; }, }; +async function positionCursorOverlay( + manager: DriverSessionManager, + page: DriverPage, + x: number, + y: number, +): Promise { + if (!manager.isCursorOverlayEnabled(page)) return; + // The overlay is visual-only. A navigation can destroy its execution + // context, but that must not prevent or invalidate the real mouse action. + try { + await page.evaluate(updateCursorOverlayPosition, { x, y }); + } catch { + // Best-effort parity with V3's cursor updates. + } +} + function assertXPathUnavailable(returnXPath: boolean | undefined): void { if (returnXPath) { throw new Error("Coordinate XPath lookup is not exposed by Stagehand V4"); diff --git a/packages/cli/src/lib/driver/commands/runtime.ts b/packages/cli/src/lib/driver/commands/runtime.ts index 9f06c0fdd..ce2d79b9c 100644 --- a/packages/cli/src/lib/driver/commands/runtime.ts +++ b/packages/cli/src/lib/driver/commands/runtime.ts @@ -2,8 +2,8 @@ import { promises as fs } from "node:fs"; import { z } from "zod"; +import { CURSOR_OVERLAY_SCRIPT } from "../cursor-overlay.js"; import type { DriverCommandHandlers } from "./types.js"; -import { unavailableCursorOverlay } from "./unavailable.js"; export const runtimeHandlers: DriverCommandHandlers = { async screenshot(manager, params) { @@ -89,7 +89,13 @@ export const runtimeHandlers: DriverCommandHandlers = { return { waited: true }; }, - cursor: unavailableCursorOverlay, + async cursor(manager) { + const page = await manager.activePage(); + await page.addInitScript(CURSOR_OVERLAY_SCRIPT); + await page.evaluate(CURSOR_OVERLAY_SCRIPT); + manager.markCursorOverlayEnabled(page); + return { cursor: "enabled" }; + }, }; function parseTimeoutMs(value: string | undefined): number { diff --git a/packages/cli/src/lib/driver/commands/unavailable.ts b/packages/cli/src/lib/driver/commands/unavailable.ts deleted file mode 100644 index 6f1f3d86e..000000000 --- a/packages/cli/src/lib/driver/commands/unavailable.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { DriverError } from "../errors.js"; -import type { DriverCommandHandler } from "./types.js"; - -export const unavailableCursorOverlay: DriverCommandHandler = async () => { - throw new DriverError( - "The visible cursor overlay has not been restored in this Stagehand V4 stack layer.", - { code: "cursor_overlay_unavailable" }, - ); -}; diff --git a/packages/cli/src/lib/driver/cursor-overlay.ts b/packages/cli/src/lib/driver/cursor-overlay.ts new file mode 100644 index 000000000..6c3858eb5 --- /dev/null +++ b/packages/cli/src/lib/driver/cursor-overlay.ts @@ -0,0 +1,80 @@ +export const CURSOR_OVERLAY_SCRIPT = `(() => { + if (globalThis !== globalThis.top) return; + + const cursorId = "__browse_cursor_overlay__"; + const ensureCursor = () => { + const existing = document.getElementById(cursorId); + if (existing instanceof HTMLDivElement) return existing; + + const root = document.documentElement || document.body; + if (!root) return null; + + const cursor = document.createElement("div"); + cursor.id = cursorId; + cursor.setAttribute("aria-hidden", "true"); + Object.assign(cursor.style, { + contain: "layout style paint", + height: "24px", + left: "0px", + mixBlendMode: "normal", + pointerEvents: "none", + position: "fixed", + top: "0px", + userSelect: "none", + width: "16px", + willChange: "left,top", + zIndex: "2147483647", + }); + cursor.innerHTML = + ''; + root.appendChild(cursor); + return cursor; + }; + + const moveCursor = (x, y) => { + const cursor = ensureCursor(); + if (!cursor) return; + cursor.style.left = Math.max(0, x) + "px"; + cursor.style.top = Math.max(0, y) + "px"; + }; + + const installCursor = () => { + if (ensureCursor()) return; + if (globalThis.__browseCursorOverlayDomReadyListenerInstalled__) return; + + document.addEventListener( + "DOMContentLoaded", + () => { + globalThis.__browseCursorOverlayDomReadyListenerInstalled__ = false; + ensureCursor(); + }, + { once: true }, + ); + globalThis.__browseCursorOverlayDomReadyListenerInstalled__ = true; + }; + + globalThis.__browseMoveCursorOverlay__ = moveCursor; + installCursor(); + if (!globalThis.__browseCursorOverlayListenerInstalled__) { + document.addEventListener( + "mousemove", + (event) => { + moveCursor(event.clientX, event.clientY); + }, + { capture: true }, + ); + globalThis.__browseCursorOverlayListenerInstalled__ = true; + } +})()`; + +export function updateCursorOverlayPosition(position: { + x: number; + y: number; +}): void { + const moveCursor = ( + globalThis as typeof globalThis & { + __browseMoveCursorOverlay__?: (x: number, y: number) => void; + } + ).__browseMoveCursorOverlay__; + moveCursor?.(position.x, position.y); +} diff --git a/packages/cli/src/lib/driver/session-manager.ts b/packages/cli/src/lib/driver/session-manager.ts index 7f30ae395..4e9381b70 100644 --- a/packages/cli/src/lib/driver/session-manager.ts +++ b/packages/cli/src/lib/driver/session-manager.ts @@ -80,6 +80,7 @@ export class DriverSessionManager { private browserbaseIdentityValue: BrowserbaseIdentity = {}; private consecutiveInitFailures = 0; private context: DriverContext | null = null; + private cursorOverlayPageIds = new Set(); private lastForwardedEnvSignature: string | null = null; private pendingEnv: ForwardedEnv | undefined; private initFailure: InitFailure | null = null; @@ -207,6 +208,7 @@ export class DriverSessionManager { this.stagehand = null; this.browser = null; this.context = null; + this.cursorOverlayPageIds.clear(); this.browserbaseIdentityValue = {}; this.initFailure = null; this.consecutiveInitFailures = 0; @@ -223,6 +225,14 @@ export class DriverSessionManager { return resolveCachedSelector(selector, this.refMaps); } + markCursorOverlayEnabled(page: DriverPage): void { + this.cursorOverlayPageIds.add(page.pageId); + } + + isCursorOverlayEnabled(page: DriverPage): boolean { + return this.cursorOverlayPageIds.has(page.pageId); + } + setRefMaps(refMaps: RefMaps): void { this.refMaps = refMaps; } diff --git a/packages/cli/tests/cursor-overlay.test.ts b/packages/cli/tests/cursor-overlay.test.ts new file mode 100644 index 000000000..6ea8b1188 --- /dev/null +++ b/packages/cli/tests/cursor-overlay.test.ts @@ -0,0 +1,123 @@ +import { createContext, runInContext } from "node:vm"; + +import { describe, expect, it, vi } from "vitest"; + +import { CURSOR_OVERLAY_SCRIPT } from "../src/lib/driver/cursor-overlay.js"; + +describe("cursor overlay", () => { + it("installs after DOMContentLoaded when the document root is not ready", () => { + const harness = createCursorHarness({ ready: false }); + + harness.install(); + + expect(harness.elements.size).toBe(0); + expect(harness.listeners.has("DOMContentLoaded")).toBe(true); + + harness.makeDocumentReady(); + harness.listeners.get("DOMContentLoaded")!(); + + expect(harness.cursor()).toBeInstanceOf(FakeDiv); + }); + + it("installs a click-through cursor once in an already-ready document", () => { + const harness = createCursorHarness(); + + harness.install(); + + expect(harness.cursor()?.style).toMatchObject({ + left: "0px", + pointerEvents: "none", + position: "fixed", + top: "0px", + zIndex: "2147483647", + }); + expect(harness.listeners.has("DOMContentLoaded")).toBe(false); + expect(harness.listeners.has("mousemove")).toBe(true); + + harness.install(); + + expect(harness.document.createElement).toHaveBeenCalledOnce(); + expect(harness.document.addEventListener).toHaveBeenCalledOnce(); + expect(harness.elements.size).toBe(1); + }); + + it("moves and clamps the cursor from top-document mouse events", () => { + const harness = createCursorHarness(); + harness.install(); + + const mousemove = harness.listeners.get("mousemove")!; + mousemove({ clientX: -25, clientY: 80 }); + expect(harness.cursor()?.style).toMatchObject({ + left: "0px", + top: "80px", + }); + + mousemove({ clientX: 140, clientY: -10 }); + expect(harness.cursor()?.style).toMatchObject({ + left: "140px", + top: "0px", + }); + }); + + it("does not install inside a child frame", () => { + const harness = createCursorHarness({ topFrame: false }); + + harness.install(); + + expect(harness.elements.size).toBe(0); + expect(harness.document.createElement).not.toHaveBeenCalled(); + expect(harness.document.addEventListener).not.toHaveBeenCalled(); + }); +}); + +class FakeDiv { + id = ""; + innerHTML = ""; + style: Record = {}; + + setAttribute(): void {} +} + +type CursorEvent = { clientX: number; clientY: number }; +type CursorListener = (event?: CursorEvent) => void; + +function createCursorHarness( + options: { ready?: boolean; topFrame?: boolean } = {}, +) { + const elements = new Map(); + const listeners = new Map(); + const root = { + appendChild(element: FakeDiv) { + elements.set(element.id, element); + }, + }; + let documentElement: typeof root | null = + options.ready === false ? null : root; + const document = { + addEventListener: vi.fn((name: string, listener: CursorListener) => { + listeners.set(name, listener); + }), + body: null, + createElement: vi.fn(() => new FakeDiv()), + get documentElement() { + return documentElement; + }, + getElementById: vi.fn((id: string) => elements.get(id) ?? null), + }; + const context = createContext({ document, HTMLDivElement: FakeDiv }); + runInContext( + `globalThis.top = ${options.topFrame === false ? "{}" : "globalThis"}`, + context, + ); + + return { + cursor: () => elements.get("__browse_cursor_overlay__"), + document, + elements, + install: () => runInContext(CURSOR_OVERLAY_SCRIPT, context), + listeners, + makeDocumentReady: () => { + documentElement = root; + }, + }; +} diff --git a/packages/cli/tests/driver-commands.test.ts b/packages/cli/tests/driver-commands.test.ts index 58b9d6e55..1b11e711e 100644 --- a/packages/cli/tests/driver-commands.test.ts +++ b/packages/cli/tests/driver-commands.test.ts @@ -326,6 +326,7 @@ describe("driver commands", () => { }; const manager = { activePage: vi.fn(async () => page), + isCursorOverlayEnabled: vi.fn(() => false), } as unknown as Parameters< NonNullable<(typeof mouseHandlers)["mouse.click"]> >[0]; @@ -385,6 +386,7 @@ describe("driver commands", () => { }; const manager = { activePage: vi.fn(async () => page), + isCursorOverlayEnabled: vi.fn(() => false), } as unknown as Parameters< NonNullable<(typeof mouseHandlers)["mouse.click"]> >[0]; @@ -410,6 +412,97 @@ describe("driver commands", () => { expect(page.dragAndDrop).toHaveBeenCalledWith(70, 80, 90, 100, {}); }); + it("keeps a successful drag successful when navigation races the final cursor update", async () => { + const page = { + dragAndDrop: vi.fn(), + evaluate: vi + .fn() + .mockResolvedValueOnce(undefined) + .mockRejectedValueOnce(new Error("Execution context was destroyed")), + }; + const manager = { + activePage: vi.fn(async () => page), + isCursorOverlayEnabled: vi.fn(() => true), + } as unknown as Parameters< + NonNullable<(typeof mouseHandlers)["mouse.drag"]> + >[0]; + + await expect( + mouseHandlers["mouse.drag"]!(manager, { + fromX: 10, + fromY: 20, + toX: 30, + toY: 40, + }), + ).resolves.toEqual({ dragged: true }); + + expect(page.dragAndDrop).toHaveBeenCalledWith(10, 20, 30, 40, {}); + expect(page.evaluate).toHaveBeenCalledTimes(2); + }); + + it("keeps cursor rendering failures from blocking coordinate mouse actions", async () => { + const page = { + click: vi.fn(), + dragAndDrop: vi.fn(), + evaluate: vi.fn().mockRejectedValue(new Error("Execution context lost")), + hover: vi.fn(), + scroll: vi.fn(), + }; + const manager = { + activePage: vi.fn(async () => page), + isCursorOverlayEnabled: vi.fn(() => true), + } as unknown as Parameters< + NonNullable<(typeof mouseHandlers)["mouse.click"]> + >[0]; + + await expect( + mouseHandlers["mouse.click"]!(manager, { x: 10, y: 20 }), + ).resolves.toEqual({ clicked: true }); + await expect( + mouseHandlers["mouse.hover"]!(manager, { x: 30, y: 40 }), + ).resolves.toEqual({ hovered: true }); + await expect( + mouseHandlers["mouse.scroll"]!(manager, { + deltaX: 5, + deltaY: 500, + x: 50, + y: 60, + }), + ).resolves.toEqual({ scrolled: true }); + await expect( + mouseHandlers["mouse.drag"]!(manager, { + fromX: 70, + fromY: 80, + toX: 90, + toY: 100, + }), + ).resolves.toEqual({ dragged: true }); + + expect(page.click).toHaveBeenCalledOnce(); + expect(page.hover).toHaveBeenCalledOnce(); + expect(page.scroll).toHaveBeenCalledOnce(); + expect(page.dragAndDrop).toHaveBeenCalledOnce(); + expect(page.evaluate).toHaveBeenCalledTimes(5); + }); + + it("continues to report failures from the real mouse action", async () => { + const actionError = new Error("Mouse input failed"); + const page = { + click: vi.fn().mockRejectedValue(actionError), + evaluate: vi.fn().mockRejectedValue(new Error("Execution context lost")), + }; + const manager = { + activePage: vi.fn(async () => page), + isCursorOverlayEnabled: vi.fn(() => true), + } as unknown as Parameters< + NonNullable<(typeof mouseHandlers)["mouse.click"]> + >[0]; + + await expect( + mouseHandlers["mouse.click"]!(manager, { x: 10, y: 20 }), + ).rejects.toBe(actionError); + }); + it("fails explicitly for the V4 coordinate XPath capability", async () => { const manager = {} as Parameters< NonNullable<(typeof mouseHandlers)["mouse.click"]> @@ -447,10 +540,60 @@ describe("driver commands", () => { expect(network.enable).toHaveBeenCalledWith(page); }); - it("keeps cursor as an explicit capability gap", async () => { + it("installs the CLI-owned cursor overlay", async () => { + const page = { + addInitScript: vi.fn(), + evaluate: vi.fn(), + pageId: "page-1", + }; + const manager = { + activePage: vi.fn(async () => page), + markCursorOverlayEnabled: vi.fn(), + } as unknown as Parameters< + NonNullable<(typeof runtimeHandlers)["cursor"]> + >[0]; + + await expect(runtimeHandlers.cursor!(manager, {})).resolves.toEqual({ + cursor: "enabled", + }); + expect(page.addInitScript).toHaveBeenCalledOnce(); + expect(page.evaluate).toHaveBeenCalledOnce(); + expect(page.addInitScript).toHaveBeenCalledWith( + page.evaluate.mock.calls[0]?.[0], + ); + expect(manager.markCursorOverlayEnabled).toHaveBeenCalledWith(page); + const cursorInstaller = page.evaluate.mock.calls[0]?.[0]; + expect(cursorInstaller).toEqual(expect.any(String)); + expect(cursorInstaller).toContain("__browse_cursor_overlay__"); + expect(cursorInstaller).toContain("globalThis !== globalThis.top"); + expect(cursorInstaller).toContain('"DOMContentLoaded"'); + expect(cursorInstaller).toContain('"mousemove"'); + }); + + it("moves an enabled overlay from coordinate input before iframe-targeted actions", async () => { + const page = { + evaluate: vi.fn(), + hover: vi.fn(), + pageId: "page-1", + }; + const manager = { + activePage: vi.fn(async () => page), + isCursorOverlayEnabled: vi.fn(() => true), + } as unknown as Parameters< + NonNullable<(typeof mouseHandlers)["mouse.hover"]> + >[0]; + await expect( - runtimeHandlers.cursor!({} as never, {}), - ).rejects.toMatchObject({ code: "cursor_overlay_unavailable" }); + mouseHandlers["mouse.hover"]!(manager, { x: 30, y: 40 }), + ).resolves.toEqual({ hovered: true }); + + expect(page.evaluate).toHaveBeenCalledWith(expect.any(Function), { + x: 30, + y: 40, + }); + expect(page.evaluate.mock.invocationCallOrder[0]).toBeLessThan( + page.hover.mock.invocationCallOrder[0] ?? Number.POSITIVE_INFINITY, + ); }); it("selects a remaining tab after closing the active tab", async () => {