diff --git a/apps/extension/src/entrypoints/content.ts b/apps/extension/src/entrypoints/content.ts index b9e9dee..d5e0a89 100644 --- a/apps/extension/src/entrypoints/content.ts +++ b/apps/extension/src/entrypoints/content.ts @@ -17,10 +17,13 @@ import { } from "@/lib/help-bridge"; import { isOverlayAgentOverlayResetMessage, + isOverlayScreenshotHideMessage, OVERLAY_AUTOMATION_BYPASS, OVERLAY_MSG_WHO_AM_I, + OVERLAY_SCREENSHOT_HIDE, type OverlayAgentOverlayResetMessage, type OverlayAutomationBypassMessage, + type OverlayScreenshotHideMessage, type OverlayWhoAmIResponse, } from "@/lib/overlay-bridge"; import { sendInterrupt } from "@/lib/overlay-interrupt-client"; @@ -49,6 +52,9 @@ export default defineContentScript({ let overlayHost: HTMLElement | null = null; let hostLossReported = false; let remountInProgress = false; + // Set while a screenshot capture asks us to suppress the control overlay + // so its breathing border / "stop" pill stay out of the image. + let screenshotHidden = false; const ui = await createShadowRootUi(ctx, { name: "browser-skill-overlay", @@ -87,7 +93,10 @@ export default defineContentScript({ requests: overlayState.borrowRequests, }), React.createElement(ControlOverlay, { - visible: overlayState.controlVisible && overlayState.activeHelp === null, + visible: + overlayState.controlVisible && + overlayState.activeHelp === null && + !screenshotHidden, interrupting: overlayState.interrupting, automationBypass: overlayState.automationBypassCount > 0, onInterrupt: handleInterrupt, @@ -136,10 +145,27 @@ export default defineContentScript({ | HelpRequestMessage | HelpCancelMessage | OverlayAgentOverlayResetMessage - | OverlayAutomationBypassMessage, + | OverlayAutomationBypassMessage + | OverlayScreenshotHideMessage, _sender: chrome.runtime.MessageSender, - sendResponse: (response: BorrowResponseMessage | HelpResponseMessage) => void, + sendResponse: ( + response: BorrowResponseMessage | HelpResponseMessage | OverlayScreenshotHideMessage, + ) => void, ) => { + if (isOverlayScreenshotHideMessage(message)) { + screenshotHidden = message.hidden; + renderOverlay(); + if (!message.hidden) return false; + // Ack only after the overlay removal has painted so the caller + // captures a frame that no longer contains the control chrome. + requestAnimationFrame(() => + requestAnimationFrame(() => + sendResponse({ type: OVERLAY_SCREENSHOT_HIDE, hidden: true }), + ), + ); + return true; + } + if ( message && typeof message === "object" && diff --git a/apps/extension/src/lib/__tests__/overlay-bridge.test.ts b/apps/extension/src/lib/__tests__/overlay-bridge.test.ts index 1b29407..785b35d 100644 --- a/apps/extension/src/lib/__tests__/overlay-bridge.test.ts +++ b/apps/extension/src/lib/__tests__/overlay-bridge.test.ts @@ -1,8 +1,10 @@ import { describe, expect, it } from "vitest"; import { isOverlayAgentOverlayResetMessage, + isOverlayScreenshotHideMessage, OVERLAY_AGENT_OVERLAY_RESET, OVERLAY_MSG_INTERRUPT, + OVERLAY_SCREENSHOT_HIDE, type OverlayInterruptRequest, type OverlayInterruptResponse, } from "@/lib/overlay-bridge"; @@ -45,3 +47,27 @@ describe("isOverlayAgentOverlayResetMessage", () => { ).toBe(false); }); }); + +describe("isOverlayScreenshotHideMessage", () => { + it("accepts hide/restore messages with a boolean flag", () => { + expect(isOverlayScreenshotHideMessage({ type: OVERLAY_SCREENSHOT_HIDE, hidden: true })).toBe( + true, + ); + expect(isOverlayScreenshotHideMessage({ type: OVERLAY_SCREENSHOT_HIDE, hidden: false })).toBe( + true, + ); + }); + + it("rejects messages with a missing or non-boolean flag", () => { + expect(isOverlayScreenshotHideMessage({ type: OVERLAY_SCREENSHOT_HIDE })).toBe(false); + expect( + isOverlayScreenshotHideMessage({ type: OVERLAY_SCREENSHOT_HIDE, hidden: "yes" }), + ).toBe(false); + }); + + it("rejects unrelated message types and non-objects", () => { + expect(isOverlayScreenshotHideMessage({ type: "something-else", hidden: true })).toBe(false); + expect(isOverlayScreenshotHideMessage(null)).toBe(false); + expect(isOverlayScreenshotHideMessage("nope")).toBe(false); + }); +}); diff --git a/apps/extension/src/lib/overlay-bridge.ts b/apps/extension/src/lib/overlay-bridge.ts index 1fc2c62..6309250 100644 --- a/apps/extension/src/lib/overlay-bridge.ts +++ b/apps/extension/src/lib/overlay-bridge.ts @@ -43,6 +43,28 @@ export interface OverlayAutomationBypassMessage { enabled: boolean; } +/** + * Background → content: hide the control overlay for the duration of a + * screenshot so its breathing border and "stop" pill do not bleed into the + * captured image. The content script re-renders without the overlay and only + * acks the `hidden: true` request once the removal has painted, so the caller + * can capture a clean frame. The matching `hidden: false` restores it. + */ +export const OVERLAY_SCREENSHOT_HIDE = "bh-screenshot-hide"; + +export interface OverlayScreenshotHideMessage { + type: typeof OVERLAY_SCREENSHOT_HIDE; + hidden: boolean; +} + +export function isOverlayScreenshotHideMessage( + message: unknown, +): message is OverlayScreenshotHideMessage { + if (!message || typeof message !== "object") return false; + const candidate = message as { type?: unknown; hidden?: unknown }; + return candidate.type === OVERLAY_SCREENSHOT_HIDE && typeof candidate.hidden === "boolean"; +} + /** Background → content: clear overlays that belong only inside an Agent tab. */ export const OVERLAY_AGENT_OVERLAY_RESET = "bh-agent-overlay-reset"; diff --git a/apps/extension/src/tools/__tests__/observation.test.ts b/apps/extension/src/tools/__tests__/observation.test.ts index a009e19..8bb2624 100644 --- a/apps/extension/src/tools/__tests__/observation.test.ts +++ b/apps/extension/src/tools/__tests__/observation.test.ts @@ -36,6 +36,7 @@ function makeScreenshotDeps( get?: ScreenshotDeps["tabsApi"]["get"]; query?: ScreenshotDeps["tabsApi"]["query"]; captureVisibleTab?: ScreenshotDeps["captureApi"]["captureVisibleTab"]; + hideControlOverlay?: ScreenshotDeps["hideControlOverlay"]; } = {}, ): ScreenshotDeps { const get = @@ -50,6 +51,7 @@ function makeScreenshotDeps( cdp: opts.cdp, tabsApi, captureApi: { ...tabsApi, captureVisibleTab }, + hideControlOverlay: opts.hideControlOverlay, }; } @@ -192,6 +194,95 @@ describe("handleScreenshot", () => { expect(res).toMatchObject({ code: "cdp_failed", message: /captureVisibleTab refused/ }); }); + it("hides the control overlay before capturing and restores it after", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + await sm.start("aa11"); + const calls: string[] = []; + const hideControlOverlay = vi.fn(async (_tabId: number, hidden: boolean) => { + calls.push(hidden ? "hide" : "restore"); + }); + const capture = vi.fn(async (_w: number) => { + calls.push("capture"); + return `data:image/png;base64,${TINY_PNG}`; + }); + const get = vi.fn(async () => ({ id: 9, windowId: 200, active: true }) as chrome.tabs.Tab); + const res = await handleScreenshot( + sm, + { session_id: "aa11", tab_id: 9 }, + makeScreenshotDeps({ captureVisibleTab: capture, get, query: vi.fn(), hideControlOverlay }), + ); + if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`); + expect(calls).toEqual(["hide", "capture", "restore"]); + expect(hideControlOverlay).toHaveBeenNthCalledWith(1, 9, true); + expect(hideControlOverlay).toHaveBeenNthCalledWith(2, 9, false); + }); + + it("restores the control overlay even when the capture fails", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + await sm.start("aa11"); + const hideControlOverlay = vi.fn(async () => {}); + const capture = vi.fn(async () => { + throw new Error("captureVisibleTab refused"); + }); + const get = vi.fn(async () => ({ id: 9, windowId: 100, active: true }) as chrome.tabs.Tab); + const res = await handleScreenshot( + sm, + { session_id: "aa11", tab_id: 9 }, + makeScreenshotDeps({ captureVisibleTab: capture, get, query: vi.fn(), hideControlOverlay }), + ); + expect(res).toMatchObject({ code: "cdp_failed" }); + expect(hideControlOverlay).toHaveBeenNthCalledWith(1, 9, true); + expect(hideControlOverlay).toHaveBeenNthCalledWith(2, 9, false); + }); + + it("captures anyway when hiding the control overlay rejects", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + await sm.start("aa11"); + const hideControlOverlay = vi.fn(async () => { + throw new Error("no content script"); + }); + const capture = vi.fn(async (_w: number) => `data:image/png;base64,${TINY_PNG}`); + const get = vi.fn(async () => ({ id: 9, windowId: 100, active: true }) as chrome.tabs.Tab); + const res = await handleScreenshot( + sm, + { session_id: "aa11", tab_id: 9 }, + makeScreenshotDeps({ captureVisibleTab: capture, get, query: vi.fn(), hideControlOverlay }), + ); + if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`); + expect(res.image_base64).toBe(TINY_PNG); + expect(capture).toHaveBeenCalledTimes(1); + // Hide failed, so we never mark it hidden and never issue a restore. + expect(hideControlOverlay).toHaveBeenCalledTimes(1); + expect(hideControlOverlay).toHaveBeenCalledWith(9, true); + }); + + it("hides and restores the control overlay around a ref capture", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + const ctx = await sm.start("aa11"); + ctx.refStore.set("e5", 999, { tabId: 7 }); + const { cdp } = makeFakeCdp({ + "DOM.scrollIntoViewIfNeeded": () => ({}), + "DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }), + "Page.captureScreenshot": () => ({ data: TINY_PNG }), + }); + const hideControlOverlay = vi.fn(async () => {}); + const res = await handleScreenshot( + sm, + { session_id: "aa11", ref: "@e5", tab_id: 7 }, + makeScreenshotDeps({ + cdp, + get: vi.fn(async () => ({ id: 7, windowId: 100, active: false }) as chrome.tabs.Tab), + query: vi.fn(), + captureVisibleTab: vi.fn(), + hideControlOverlay, + }), + ); + if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`); + expect(res.image_base64).toBe(TINY_PNG); + expect(hideControlOverlay).toHaveBeenNthCalledWith(1, 7, true); + expect(hideControlOverlay).toHaveBeenNthCalledWith(2, 7, false); + }); + it("captures a clipped PNG when ref is given", async () => { const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); const ctx = await sm.start("aa11"); diff --git a/apps/extension/src/tools/dispatcher.ts b/apps/extension/src/tools/dispatcher.ts index 669727f..cf43a00 100644 --- a/apps/extension/src/tools/dispatcher.ts +++ b/apps/extension/src/tools/dispatcher.ts @@ -1,4 +1,4 @@ -import { OVERLAY_AUTOMATION_BYPASS } from "@/lib/overlay-bridge"; +import { OVERLAY_AUTOMATION_BYPASS, OVERLAY_SCREENSHOT_HIDE } from "@/lib/overlay-bridge"; import type { SessionManager } from "@/session-manager/manager"; import type { Transport } from "@/transport/transport"; import type { @@ -279,7 +279,21 @@ export class ToolDispatcher { this.sessions, req.params as ScreenshotParams, this.cdp - ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi, captureApi: chromeTabsCaptureApi } + ? { + cdp: this.cdp, + tabsApi: chromeTabsCaptureApi, + captureApi: chromeTabsCaptureApi, + hideControlOverlay: async (tabId, hidden) => { + try { + await chrome.tabs.sendMessage(tabId, { + type: OVERLAY_SCREENSHOT_HIDE, + hidden, + }); + } catch { + // Content script may be unavailable on restricted pages. + } + }, + } : undefined, ); case "tool.console": diff --git a/apps/extension/src/tools/observation.ts b/apps/extension/src/tools/observation.ts index 96b54d0..6bfac2f 100644 --- a/apps/extension/src/tools/observation.ts +++ b/apps/extension/src/tools/observation.ts @@ -97,6 +97,14 @@ export interface ScreenshotDeps { cdp?: SharedCdpRunner; tabsApi: ChromeTabsApi; captureApi: ChromeTabsCaptureApi; + /** + * Optionally suppress the content-script control overlay (breathing border + * and "stop" pill) while the capture runs so it does not appear in the + * image. Resolves once the overlay has hidden and painted; the caller + * restores it with `hidden = false` afterwards. Best-effort: on a restricted + * page with no content script this rejects and the shot proceeds as-is. + */ + hideControlOverlay?: (tabId: number, hidden: boolean) => Promise; } function defaultScreenshotDeps(): ScreenshotDeps { @@ -146,6 +154,12 @@ async function captureElementScreenshot( } } +/** + * Capture a screenshot of a tab (`tool.screenshot`). With a `ref` it clips to + * that element via CDP; otherwise it captures the visible tab. In both cases + * the control overlay is hidden for the duration of the shot (when + * `deps.hideControlOverlay` is provided) so its chrome stays out of the image. + */ export async function handleScreenshot( manager: SessionManager, params: ScreenshotParams, @@ -156,53 +170,77 @@ export async function handleScreenshot( const ctx = ctxOrErr; const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi); if (isRpcError(target)) return target; - const dialogCursor = deps.cdp ? markDialogCursor(deps.cdp, target.tabId) : 0; - const withShotDialogs = (result: T) => - deps.cdp ? attachDialogs(deps.cdp, target.tabId, dialogCursor, result) : result; - - const ref = typeof params.ref === "string" && params.ref.length > 0 ? params.ref : null; - if (ref) { - if (!deps.cdp) { - return { code: "cdp_failed", message: "screenshot ref capture requires CDP" }; - } - const node = resolveSnapshotRef(ctx, ref, target.tabId); - if (isRpcError(node)) return node; - deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); - const captured = await captureElementScreenshot(deps.cdp, target.tabId, node.backendNodeId); - if (isRpcError(captured)) return captured; - return withShotDialogs({ - image_base64: captured.image_base64, - width: captured.width, - height: captured.height, - format: "png", - tab_id: target.tabId, - }); - } - if (!target.active) { - return rpcError( - "invalid_params", - "tab_not_active", - `tab ${target.tabId} is not active; screenshot can only capture the visible tab`, - ); + // Suppress the control overlay (breathing border / "stop" pill) for the + // duration of the capture so it does not appear in the image. Best-effort: + // a restricted page with no content script just captures as-is. + let controlOverlayHidden = false; + if (deps.hideControlOverlay) { + try { + await deps.hideControlOverlay(target.tabId, true); + controlOverlayHidden = true; + } catch (err) { + console.debug("[bsk screenshot] hide control overlay failed", err); + } } try { - const dataUrl = await deps.captureApi.captureVisibleTab(target.windowId, { format: "png" }); - const image_base64 = stripDataUrlPrefix(dataUrl); - const dims = parsePngDimensions(image_base64) ?? { width: 0, height: 0 }; - return withShotDialogs({ - image_base64, - width: dims.width, - height: dims.height, - format: "png", - tab_id: target.tabId, - }); - } catch (err) { - return { - code: "cdp_failed", - message: err instanceof Error ? err.message : String(err), - }; + const dialogCursor = deps.cdp ? markDialogCursor(deps.cdp, target.tabId) : 0; + const withShotDialogs = (result: T) => + deps.cdp ? attachDialogs(deps.cdp, target.tabId, dialogCursor, result) : result; + + const ref = typeof params.ref === "string" && params.ref.length > 0 ? params.ref : null; + if (ref) { + if (!deps.cdp) { + return { code: "cdp_failed", message: "screenshot ref capture requires CDP" }; + } + const node = resolveSnapshotRef(ctx, ref, target.tabId); + if (isRpcError(node)) return node; + deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); + const captured = await captureElementScreenshot(deps.cdp, target.tabId, node.backendNodeId); + if (isRpcError(captured)) return captured; + return withShotDialogs({ + image_base64: captured.image_base64, + width: captured.width, + height: captured.height, + format: "png", + tab_id: target.tabId, + }); + } + + if (!target.active) { + return rpcError( + "invalid_params", + "tab_not_active", + `tab ${target.tabId} is not active; screenshot can only capture the visible tab`, + ); + } + + try { + const dataUrl = await deps.captureApi.captureVisibleTab(target.windowId, { format: "png" }); + const image_base64 = stripDataUrlPrefix(dataUrl); + const dims = parsePngDimensions(image_base64) ?? { width: 0, height: 0 }; + return withShotDialogs({ + image_base64, + width: dims.width, + height: dims.height, + format: "png", + tab_id: target.tabId, + }); + } catch (err) { + return { + code: "cdp_failed", + message: err instanceof Error ? err.message : String(err), + }; + } + } finally { + if (controlOverlayHidden && deps.hideControlOverlay) { + try { + await deps.hideControlOverlay(target.tabId, false); + } catch (err) { + console.debug("[bsk screenshot] restore control overlay failed", err); + } + } } }