Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 29 additions & 3 deletions apps/extension/src/entrypoints/content.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,13 @@ import {
} from "@/lib/help-bridge";
import {
isOverlayAgentOverlayResetMessage,
isOverlayScreenshotHideMessage,
OVERLAY_AUTOMATION_BYPASS,
OVERLAY_MSG_WHO_AM_I,
OVERLAY_SCREENSHOT_HIDE,
type OverlayAgentOverlayResetMessage,
type OverlayAutomationBypassMessage,
type OverlayScreenshotHideMessage,
type OverlayWhoAmIResponse,
} from "@/lib/overlay-bridge";
import { sendInterrupt } from "@/lib/overlay-interrupt-client";
Expand Down Expand Up @@ -49,6 +52,9 @@ export default defineContentScript({
let overlayHost: HTMLElement | null = null;
let hostLossReported = false;
let remountInProgress = false;
// Set while a screenshot capture asks us to suppress the control overlay
// so its breathing border / "stop" pill stay out of the image.
let screenshotHidden = false;

const ui = await createShadowRootUi(ctx, {
name: "browser-skill-overlay",
Expand Down Expand Up @@ -87,7 +93,10 @@ export default defineContentScript({
requests: overlayState.borrowRequests,
}),
React.createElement(ControlOverlay, {
visible: overlayState.controlVisible && overlayState.activeHelp === null,
visible:
overlayState.controlVisible &&
overlayState.activeHelp === null &&
!screenshotHidden,
interrupting: overlayState.interrupting,
automationBypass: overlayState.automationBypassCount > 0,
onInterrupt: handleInterrupt,
Expand Down Expand Up @@ -136,10 +145,27 @@ export default defineContentScript({
| HelpRequestMessage
| HelpCancelMessage
| OverlayAgentOverlayResetMessage
| OverlayAutomationBypassMessage,
| OverlayAutomationBypassMessage
| OverlayScreenshotHideMessage,
_sender: chrome.runtime.MessageSender,
sendResponse: (response: BorrowResponseMessage | HelpResponseMessage) => void,
sendResponse: (
response: BorrowResponseMessage | HelpResponseMessage | OverlayScreenshotHideMessage,
) => void,
) => {
if (isOverlayScreenshotHideMessage(message)) {
screenshotHidden = message.hidden;
renderOverlay();
if (!message.hidden) return false;
// Ack only after the overlay removal has painted so the caller
// captures a frame that no longer contains the control chrome.
requestAnimationFrame(() =>
requestAnimationFrame(() =>
sendResponse({ type: OVERLAY_SCREENSHOT_HIDE, hidden: true }),
),
);
return true;
}

if (
message &&
typeof message === "object" &&
Expand Down
26 changes: 26 additions & 0 deletions apps/extension/src/lib/__tests__/overlay-bridge.test.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
import { describe, expect, it } from "vitest";
import {
isOverlayAgentOverlayResetMessage,
isOverlayScreenshotHideMessage,
OVERLAY_AGENT_OVERLAY_RESET,
OVERLAY_MSG_INTERRUPT,
OVERLAY_SCREENSHOT_HIDE,
type OverlayInterruptRequest,
type OverlayInterruptResponse,
} from "@/lib/overlay-bridge";
Expand Down Expand Up @@ -45,3 +47,27 @@ describe("isOverlayAgentOverlayResetMessage", () => {
).toBe(false);
});
});

describe("isOverlayScreenshotHideMessage", () => {
it("accepts hide/restore messages with a boolean flag", () => {
expect(isOverlayScreenshotHideMessage({ type: OVERLAY_SCREENSHOT_HIDE, hidden: true })).toBe(
true,
);
expect(isOverlayScreenshotHideMessage({ type: OVERLAY_SCREENSHOT_HIDE, hidden: false })).toBe(
true,
);
});

it("rejects messages with a missing or non-boolean flag", () => {
expect(isOverlayScreenshotHideMessage({ type: OVERLAY_SCREENSHOT_HIDE })).toBe(false);
expect(
isOverlayScreenshotHideMessage({ type: OVERLAY_SCREENSHOT_HIDE, hidden: "yes" }),
).toBe(false);
});

it("rejects unrelated message types and non-objects", () => {
expect(isOverlayScreenshotHideMessage({ type: "something-else", hidden: true })).toBe(false);
expect(isOverlayScreenshotHideMessage(null)).toBe(false);
expect(isOverlayScreenshotHideMessage("nope")).toBe(false);
});
});
22 changes: 22 additions & 0 deletions apps/extension/src/lib/overlay-bridge.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,28 @@ export interface OverlayAutomationBypassMessage {
enabled: boolean;
}

/**
* Background → content: hide the control overlay for the duration of a
* screenshot so its breathing border and "stop" pill do not bleed into the
* captured image. The content script re-renders without the overlay and only
* acks the `hidden: true` request once the removal has painted, so the caller
* can capture a clean frame. The matching `hidden: false` restores it.
*/
export const OVERLAY_SCREENSHOT_HIDE = "bh-screenshot-hide";

export interface OverlayScreenshotHideMessage {
type: typeof OVERLAY_SCREENSHOT_HIDE;
hidden: boolean;
}

export function isOverlayScreenshotHideMessage(
message: unknown,
): message is OverlayScreenshotHideMessage {
if (!message || typeof message !== "object") return false;
const candidate = message as { type?: unknown; hidden?: unknown };
return candidate.type === OVERLAY_SCREENSHOT_HIDE && typeof candidate.hidden === "boolean";
}

/** Background → content: clear overlays that belong only inside an Agent tab. */
export const OVERLAY_AGENT_OVERLAY_RESET = "bh-agent-overlay-reset";

Expand Down
91 changes: 91 additions & 0 deletions apps/extension/src/tools/__tests__/observation.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ function makeScreenshotDeps(
get?: ScreenshotDeps["tabsApi"]["get"];
query?: ScreenshotDeps["tabsApi"]["query"];
captureVisibleTab?: ScreenshotDeps["captureApi"]["captureVisibleTab"];
hideControlOverlay?: ScreenshotDeps["hideControlOverlay"];
} = {},
): ScreenshotDeps {
const get =
Expand All @@ -50,6 +51,7 @@ function makeScreenshotDeps(
cdp: opts.cdp,
tabsApi,
captureApi: { ...tabsApi, captureVisibleTab },
hideControlOverlay: opts.hideControlOverlay,
};
}

Expand Down Expand Up @@ -192,6 +194,95 @@ describe("handleScreenshot", () => {
expect(res).toMatchObject({ code: "cdp_failed", message: /captureVisibleTab refused/ });
});

it("hides the control overlay before capturing and restores it after", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
await sm.start("aa11");
const calls: string[] = [];
const hideControlOverlay = vi.fn(async (_tabId: number, hidden: boolean) => {
calls.push(hidden ? "hide" : "restore");
});
const capture = vi.fn(async (_w: number) => {
calls.push("capture");
return `data:image/png;base64,${TINY_PNG}`;
});
const get = vi.fn(async () => ({ id: 9, windowId: 200, active: true }) as chrome.tabs.Tab);
const res = await handleScreenshot(
sm,
{ session_id: "aa11", tab_id: 9 },
makeScreenshotDeps({ captureVisibleTab: capture, get, query: vi.fn(), hideControlOverlay }),
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
expect(calls).toEqual(["hide", "capture", "restore"]);
expect(hideControlOverlay).toHaveBeenNthCalledWith(1, 9, true);
expect(hideControlOverlay).toHaveBeenNthCalledWith(2, 9, false);
});

it("restores the control overlay even when the capture fails", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
await sm.start("aa11");
const hideControlOverlay = vi.fn(async () => {});
const capture = vi.fn(async () => {
throw new Error("captureVisibleTab refused");
});
const get = vi.fn(async () => ({ id: 9, windowId: 100, active: true }) as chrome.tabs.Tab);
const res = await handleScreenshot(
sm,
{ session_id: "aa11", tab_id: 9 },
makeScreenshotDeps({ captureVisibleTab: capture, get, query: vi.fn(), hideControlOverlay }),
);
expect(res).toMatchObject({ code: "cdp_failed" });
expect(hideControlOverlay).toHaveBeenNthCalledWith(1, 9, true);
expect(hideControlOverlay).toHaveBeenNthCalledWith(2, 9, false);
});

it("captures anyway when hiding the control overlay rejects", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
await sm.start("aa11");
const hideControlOverlay = vi.fn(async () => {
throw new Error("no content script");
});
const capture = vi.fn(async (_w: number) => `data:image/png;base64,${TINY_PNG}`);
const get = vi.fn(async () => ({ id: 9, windowId: 100, active: true }) as chrome.tabs.Tab);
const res = await handleScreenshot(
sm,
{ session_id: "aa11", tab_id: 9 },
makeScreenshotDeps({ captureVisibleTab: capture, get, query: vi.fn(), hideControlOverlay }),
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
expect(res.image_base64).toBe(TINY_PNG);
expect(capture).toHaveBeenCalledTimes(1);
// Hide failed, so we never mark it hidden and never issue a restore.
expect(hideControlOverlay).toHaveBeenCalledTimes(1);
expect(hideControlOverlay).toHaveBeenCalledWith(9, true);
});

it("hides and restores the control overlay around a ref capture", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e5", 999, { tabId: 7 });
const { cdp } = makeFakeCdp({
"DOM.scrollIntoViewIfNeeded": () => ({}),
"DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }),
"Page.captureScreenshot": () => ({ data: TINY_PNG }),
});
const hideControlOverlay = vi.fn(async () => {});
const res = await handleScreenshot(
sm,
{ session_id: "aa11", ref: "@e5", tab_id: 7 },
makeScreenshotDeps({
cdp,
get: vi.fn(async () => ({ id: 7, windowId: 100, active: false }) as chrome.tabs.Tab),
query: vi.fn(),
captureVisibleTab: vi.fn(),
hideControlOverlay,
}),
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
expect(res.image_base64).toBe(TINY_PNG);
expect(hideControlOverlay).toHaveBeenNthCalledWith(1, 7, true);
expect(hideControlOverlay).toHaveBeenNthCalledWith(2, 7, false);
});

it("captures a clipped PNG when ref is given", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
Expand Down
18 changes: 16 additions & 2 deletions apps/extension/src/tools/dispatcher.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { OVERLAY_AUTOMATION_BYPASS } from "@/lib/overlay-bridge";
import { OVERLAY_AUTOMATION_BYPASS, OVERLAY_SCREENSHOT_HIDE } from "@/lib/overlay-bridge";
import type { SessionManager } from "@/session-manager/manager";
import type { Transport } from "@/transport/transport";
import type {
Expand Down Expand Up @@ -279,7 +279,21 @@ export class ToolDispatcher {
this.sessions,
req.params as ScreenshotParams,
this.cdp
? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi, captureApi: chromeTabsCaptureApi }
? {
cdp: this.cdp,
tabsApi: chromeTabsCaptureApi,
captureApi: chromeTabsCaptureApi,
hideControlOverlay: async (tabId, hidden) => {
try {
await chrome.tabs.sendMessage(tabId, {
type: OVERLAY_SCREENSHOT_HIDE,
hidden,
});
} catch {
// Content script may be unavailable on restricted pages.
}
},
}
: undefined,
);
case "tool.console":
Expand Down
Loading
Loading