Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions packages/cli/src/lib/driver/commands/mouse.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import { z } from "zod";

import { updateCursorOverlayPosition } from "../cursor-overlay.js";
import type { DriverPage, DriverSessionManager } from "../session-manager.js";
import type { DriverCommandHandlers } from "./types.js";

const ButtonSchema = z.enum(["left", "right", "middle"]).optional();
Expand All @@ -17,6 +19,7 @@ export const mouseHandlers: DriverCommandHandlers = {
.parse(params);
assertXPathUnavailable(returnXPath);
const page = await manager.activePage();
await positionCursorOverlay(manager, page, x, y);
await page.click(x, y, {
...(button === undefined ? {} : { button }),
...(clickCount === undefined ? {} : { clickCount }),
Expand All @@ -34,6 +37,7 @@ export const mouseHandlers: DriverCommandHandlers = {
.parse(params);
assertXPathUnavailable(returnXPath);
const page = await manager.activePage();
await positionCursorOverlay(manager, page, x, y);
await page.hover(x, y);
return { hovered: true };
},
Expand All @@ -50,6 +54,7 @@ export const mouseHandlers: DriverCommandHandlers = {
.parse(params);
assertXPathUnavailable(returnXPath);
const page = await manager.activePage();
await positionCursorOverlay(manager, page, x, y);
await page.scroll(x, y, deltaX, deltaY);
return { scrolled: true };
},
Expand All @@ -69,15 +74,30 @@ export const mouseHandlers: DriverCommandHandlers = {
.parse(params);
assertXPathUnavailable(returnXPath);
const page = await manager.activePage();
await positionCursorOverlay(manager, page, fromX, fromY);
await page.dragAndDrop(fromX, fromY, toX, toY, {
...(button === undefined ? {} : { button }),
...(delay === undefined ? {} : { delay }),
...(steps === undefined ? {} : { steps }),
});
// A successful drag may navigate and destroy the old execution context.
// The final marker position is visual-only, so do not turn that race into a
// reported drag failure.
await positionCursorOverlay(manager, page, toX, toY).catch(() => undefined);
return { dragged: true };
},
};

async function positionCursorOverlay(
manager: DriverSessionManager,
page: DriverPage,
x: number,
y: number,
): Promise<void> {
if (!manager.isCursorOverlayEnabled(page)) return;
await page.evaluate(updateCursorOverlayPosition, { x, y });
}

function assertXPathUnavailable(returnXPath: boolean | undefined): void {
if (returnXPath) {
throw new Error("Coordinate XPath lookup is not exposed by Stagehand V4");
Expand Down
10 changes: 8 additions & 2 deletions packages/cli/src/lib/driver/commands/runtime.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@ import { promises as fs } from "node:fs";

import { z } from "zod";

import { CURSOR_OVERLAY_SCRIPT } from "../cursor-overlay.js";
import type { DriverCommandHandlers } from "./types.js";
import { unavailableCursorOverlay } from "./unavailable.js";

export const runtimeHandlers: DriverCommandHandlers = {
async screenshot(manager, params) {
Expand Down Expand Up @@ -89,7 +89,13 @@ export const runtimeHandlers: DriverCommandHandlers = {
return { waited: true };
},

cursor: unavailableCursorOverlay,
async cursor(manager) {
const page = await manager.activePage();
await page.addInitScript(CURSOR_OVERLAY_SCRIPT);
Comment thread
shrey150 marked this conversation as resolved.
await page.evaluate(CURSOR_OVERLAY_SCRIPT);
Comment thread
cubic-dev-ai[bot] marked this conversation as resolved.
manager.markCursorOverlayEnabled(page);
return { enabled: true };
},
};

function parseTimeoutMs(value: string | undefined): number {
Expand Down
9 changes: 0 additions & 9 deletions packages/cli/src/lib/driver/commands/unavailable.ts

This file was deleted.

65 changes: 65 additions & 0 deletions packages/cli/src/lib/driver/cursor-overlay.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
export const CURSOR_OVERLAY_SCRIPT = `(() => {
if (globalThis !== globalThis.top) return;

const cursorId = "__browse_cursor_overlay__";
const ensureCursor = () => {
const existing = document.getElementById(cursorId);
if (existing instanceof HTMLDivElement) return existing;

const root = document.documentElement || document.body;
if (!root) return null;

const cursor = document.createElement("div");
cursor.id = cursorId;
cursor.setAttribute("aria-hidden", "true");
Object.assign(cursor.style, {
contain: "layout style paint",
height: "24px",
left: "0px",
mixBlendMode: "normal",
pointerEvents: "none",
position: "fixed",
top: "0px",
userSelect: "none",
width: "16px",
willChange: "left,top",
zIndex: "2147483647",
});
cursor.innerHTML =
'<svg xmlns="http://www.w3.org/2000/svg" width="16" height="24" viewBox="0 0 16 24"><path d="M1 0 L1 22 L6 14 L15 14 Z" fill="black" stroke="white" stroke-width="0.7"/></svg>';
root.appendChild(cursor);
return cursor;
};

const moveCursor = (x, y) => {
const cursor = ensureCursor();
if (!cursor) return;
cursor.style.left = Math.max(0, x) + "px";
cursor.style.top = Math.max(0, y) + "px";
};

globalThis.__browseMoveCursorOverlay__ = moveCursor;
ensureCursor();
if (!globalThis.__browseCursorOverlayListenerInstalled__) {
document.addEventListener(
"mousemove",
Comment thread
shrey150 marked this conversation as resolved.
(event) => {
moveCursor(event.clientX, event.clientY);
},
{ capture: true },
);
globalThis.__browseCursorOverlayListenerInstalled__ = true;
}
})()`;

export function updateCursorOverlayPosition(position: {
x: number;
y: number;
}): void {
const moveCursor = (
globalThis as typeof globalThis & {
__browseMoveCursorOverlay__?: (x: number, y: number) => void;
}
).__browseMoveCursorOverlay__;
moveCursor?.(position.x, position.y);
}
10 changes: 10 additions & 0 deletions packages/cli/src/lib/driver/session-manager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ export class DriverSessionManager {
private browserbaseIdentityValue: BrowserbaseIdentity = {};
private consecutiveInitFailures = 0;
private context: DriverContext | null = null;
private cursorOverlayPageIds = new Set<string>();
private lastForwardedEnvSignature: string | null = null;
private pendingEnv: ForwardedEnv | undefined;
private initFailure: InitFailure | null = null;
Expand Down Expand Up @@ -207,6 +208,7 @@ export class DriverSessionManager {
this.stagehand = null;
this.browser = null;
this.context = null;
this.cursorOverlayPageIds.clear();
this.browserbaseIdentityValue = {};
this.initFailure = null;
this.consecutiveInitFailures = 0;
Expand All @@ -223,6 +225,14 @@ export class DriverSessionManager {
return resolveCachedSelector(selector, this.refMaps);
}

markCursorOverlayEnabled(page: DriverPage): void {
this.cursorOverlayPageIds.add(page.pageId);
}

isCursorOverlayEnabled(page: DriverPage): boolean {
return this.cursorOverlayPageIds.has(page.pageId);
}

setRefMaps(refMaps: RefMaps): void {
this.refMaps = refMaps;
}
Expand Down
85 changes: 82 additions & 3 deletions packages/cli/tests/driver-commands.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -326,6 +326,7 @@ describe("driver commands", () => {
};
const manager = {
activePage: vi.fn(async () => page),
isCursorOverlayEnabled: vi.fn(() => false),
} as unknown as Parameters<
NonNullable<(typeof mouseHandlers)["mouse.click"]>
>[0];
Expand Down Expand Up @@ -385,6 +386,7 @@ describe("driver commands", () => {
};
const manager = {
activePage: vi.fn(async () => page),
isCursorOverlayEnabled: vi.fn(() => false),
} as unknown as Parameters<
NonNullable<(typeof mouseHandlers)["mouse.click"]>
>[0];
Expand All @@ -410,6 +412,34 @@ describe("driver commands", () => {
expect(page.dragAndDrop).toHaveBeenCalledWith(70, 80, 90, 100, {});
});

it("keeps a successful drag successful when navigation races the final cursor update", async () => {
const page = {
dragAndDrop: vi.fn(),
evaluate: vi
.fn()
.mockResolvedValueOnce(undefined)
.mockRejectedValueOnce(new Error("Execution context was destroyed")),
};
const manager = {
activePage: vi.fn(async () => page),
isCursorOverlayEnabled: vi.fn(() => true),
} as unknown as Parameters<
NonNullable<(typeof mouseHandlers)["mouse.drag"]>
>[0];

await expect(
mouseHandlers["mouse.drag"]!(manager, {
fromX: 10,
fromY: 20,
toX: 30,
toY: 40,
}),
).resolves.toEqual({ dragged: true });

expect(page.dragAndDrop).toHaveBeenCalledWith(10, 20, 30, 40, {});
expect(page.evaluate).toHaveBeenCalledTimes(2);
});

it("fails explicitly for the V4 coordinate XPath capability", async () => {
const manager = {} as Parameters<
NonNullable<(typeof mouseHandlers)["mouse.click"]>
Expand Down Expand Up @@ -447,10 +477,59 @@ describe("driver commands", () => {
expect(network.enable).toHaveBeenCalledWith(page);
});

it("keeps cursor as an explicit capability gap", async () => {
it("installs the CLI-owned cursor overlay", async () => {
const page = {
addInitScript: vi.fn(),
evaluate: vi.fn(),
pageId: "page-1",
};
const manager = {
activePage: vi.fn(async () => page),
markCursorOverlayEnabled: vi.fn(),
} as unknown as Parameters<
NonNullable<(typeof runtimeHandlers)["cursor"]>
>[0];

await expect(runtimeHandlers.cursor!(manager, {})).resolves.toEqual({
enabled: true,
});
expect(page.addInitScript).toHaveBeenCalledOnce();
expect(page.evaluate).toHaveBeenCalledOnce();
expect(page.addInitScript).toHaveBeenCalledWith(
page.evaluate.mock.calls[0]?.[0],
);
expect(manager.markCursorOverlayEnabled).toHaveBeenCalledWith(page);
const cursorInstaller = page.evaluate.mock.calls[0]?.[0];
expect(cursorInstaller).toEqual(expect.any(String));
expect(cursorInstaller).toContain("__browse_cursor_overlay__");
expect(cursorInstaller).toContain("globalThis !== globalThis.top");
expect(cursorInstaller).toContain('"mousemove"');
});

it("moves an enabled overlay from coordinate input before iframe-targeted actions", async () => {
const page = {
evaluate: vi.fn(),
hover: vi.fn(),
pageId: "page-1",
};
const manager = {
activePage: vi.fn(async () => page),
isCursorOverlayEnabled: vi.fn(() => true),
} as unknown as Parameters<
NonNullable<(typeof mouseHandlers)["mouse.hover"]>
>[0];

await expect(
runtimeHandlers.cursor!({} as never, {}),
).rejects.toMatchObject({ code: "cursor_overlay_unavailable" });
mouseHandlers["mouse.hover"]!(manager, { x: 30, y: 40 }),
).resolves.toEqual({ hovered: true });

expect(page.evaluate).toHaveBeenCalledWith(expect.any(Function), {
x: 30,
y: 40,
});
expect(page.evaluate.mock.invocationCallOrder[0]).toBeLessThan(
page.hover.mock.invocationCallOrder[0] ?? Number.POSITIVE_INFINITY,
);
});

it("selects a remaining tab after closing the active tab", async () => {
Expand Down
Loading