From 73d1c7665b5f9c76c90b0e0f33aea2e127b6f685 Mon Sep 17 00:00:00 2001 From: graydovee Date: Sat, 10 Oct 2026 12:27:15 +0800 Subject: [PATCH 1/4] =?UTF-8?q?feat(browser):=20=E5=A2=9E=E5=8A=A0=20brows?= =?UTF-8?q?er=5Fscreenshot=20=E8=A7=86=E5=8F=A3=E6=88=AA=E5=9B=BE=E5=B7=A5?= =?UTF-8?q?=E5=85=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 扩展侧用 chrome.tabs.captureVisibleTab 截取受控标签页的可视区域， 返回 base64 与媒体类型；宿主侧把像素存入附件存储，以图片内容块 返回给模型，持久化结果只保留元数据。授权与 browser_snapshot 同为 读取档；png 超过字节上限时自动降级为 jpeg 重试一次。 --- .../src/background/authorization.ts | 12 +- .../dsh-browser/src/background/tools.ts | 108 ++++++++++- .../dsh-browser/tests/screenshot.spec.ts | 123 ++++++++++++ packages/browser/bridge-browser/src/index.ts | 12 +- .../bridge-browser/src/screenshot-tools.ts | 175 ++++++++++++++++++ .../tests/screenshot-tools.spec.ts | 142 ++++++++++++++ 6 files changed, 563 insertions(+), 9 deletions(-) create mode 100644 extensions/dsh-browser/tests/screenshot.spec.ts create mode 100644 packages/browser/bridge-browser/src/screenshot-tools.ts create mode 100644 packages/browser/bridge-browser/tests/screenshot-tools.spec.ts diff --git a/extensions/dsh-browser/src/background/authorization.ts b/extensions/dsh-browser/src/background/authorization.ts index d263a0587..4e957f838 100644 --- a/extensions/dsh-browser/src/background/authorization.ts +++ b/extensions/dsh-browser/src/background/authorization.ts @@ -5,7 +5,7 @@ import type { TabFrame } from './frames.ts' import type { ApprovalPrompt } from '../security/approval.ts' import { getUiLocale, type UiLocale } from '../i18n.ts' -const PAGE_READS = new Set(['browser_snapshot', 'browser_get_text']) +const PAGE_READS = new Set(['browser_snapshot', 'browser_get_text', 'browser_screenshot']) const STATE_CHANGING_ACTIONS = new Set([ 'browser_click', 'browser_type', @@ -26,15 +26,17 @@ export function approvalPromptForCall( ): ApprovalPrompt | undefined { if (PAGE_READS.has(call.name)) { if (sharePageContent !== 'ask') return undefined - const targetFrames = call.name === 'browser_snapshot' - ? frames - : frames.filter((frame) => frame.frameId === requestedFrame(call.args)) + const targetFrames = call.name === 'browser_get_text' + ? frames.filter((frame) => frame.frameId === requestedFrame(call.args)) + : frames return { kind: 'read', action: call.name, summary: call.name === 'browser_snapshot' ? localized(locale, 'Read the current page and accessible iframes', '读取当前页面及可访问 iframe') - : localized(locale, 'Read text from the specified area of the current page', '读取当前页面的指定文本区域'), + : call.name === 'browser_screenshot' + ? localized(locale, 'Capture a screenshot of the current page', '截取当前页面的可视区域') + : localized(locale, 'Read text from the specified area of the current page', '读取当前页面的指定文本区域'), origins: uniqueOrigins(targetFrames, frames), canTrust: false, } diff --git a/extensions/dsh-browser/src/background/tools.ts b/extensions/dsh-browser/src/background/tools.ts index 0fc3069a1..0f49e9319 100644 --- a/extensions/dsh-browser/src/background/tools.ts +++ b/extensions/dsh-browser/src/background/tools.ts @@ -655,6 +655,104 @@ async function dispatchTabManagementTool( return unavailable(`Unsupported tab-management tool: ${call.name}`) } +/** Parse the model-supplied screenshot capture options. */ +function screenshotCaptureOptions(call: ToolCall): { format: 'png' | 'jpeg'; quality: number } | ToolAnswer { + const format = call.args.format === undefined ? 'png' : call.args.format + if (format !== 'png' && format !== 'jpeg') { + return { ok: false, error: { code: 'action-failed', message: 'format must be "png" or "jpeg".' } } + } + const quality = call.args.quality === undefined ? 85 : call.args.quality + if (typeof quality !== 'number' || !Number.isInteger(quality) || quality < 1 || quality > 100) { + return { ok: false, error: { code: 'action-failed', message: 'quality must be an integer between 1 and 100.' } } + } + return { format, quality } +} + +/** Decode a captureVisibleTab data URL into base64 pixels and media type. */ +function decodeCaptureDataUrl(dataUrl: string): { data: string; mediaType: string } | undefined { + const comma = dataUrl.indexOf(',') + if (comma === -1) return undefined + const header = /^data:([^;,]+);base64$/.exec(dataUrl.slice(0, comma)) + if (header === null) return undefined + const data = dataUrl.slice(comma + 1) + return data === '' ? undefined : { data, mediaType: header[1] ?? '' } +} + +async function measureImageBytes(data: string, mediaType: string): Promise<{ width: number; height: number } | undefined> { + try { + const binary = atob(data) + const bytes = new Uint8Array(binary.length) + for (let index = 0; index < binary.length; index += 1) bytes[index] = binary.charCodeAt(index) + const bitmap = await createImageBitmap(new Blob([bytes as BlobPart], { type: mediaType })) + try { + return { width: bitmap.width, height: bitmap.height } + } finally { + bitmap.close() + } + } catch { + // Dimensions are advisory metadata; the capture stays deliverable. + return undefined + } +} + +/** + * Capture the controlled tab's viewport. `captureVisibleTab` photographs the + * active tab of the window, so a controlled-but-inactive tab is a hard error + * rather than a silent capture of the wrong page. + */ +async function dispatchScreenshot( + tab: Pick & { active?: boolean; windowId?: number }, + call: ToolCall, + signal?: AbortSignal, + targetStillAllowed?: () => boolean, +): Promise { + if (isCancelled(call, signal)) return cancelled() + if (targetStillAllowed?.() === false) return targetChanged() + const options = screenshotCaptureOptions(call) + if ('ok' in options) return options + let active = tab.active + let windowId = tab.windowId + if (active === undefined || windowId === undefined) { + const fresh = await findTab(tab.id!) + if (fresh === undefined) return unavailable('The controlled tab closed before the screenshot was captured.') + active = fresh.active + windowId = fresh.windowId + } + if (isCancelled(call, signal)) return cancelled() + if (targetStillAllowed?.() === false) return targetChanged() + if (active !== true) { + return unavailable('The controlled tab is not the active tab of its window, so its viewport cannot be captured. Activate the tab first, then retry.') + } + if (windowId === undefined) { + return unavailable('The controlled tab has no window, so its viewport cannot be captured.') + } + let dataUrl: string + try { + dataUrl = await chrome.tabs.captureVisibleTab(windowId, { + format: options.format, + ...(options.format === 'jpeg' ? { quality: options.quality } : {}), + }) + } catch (error: unknown) { + const detail = error instanceof Error ? error.message : String(error) + return unavailable(`The screenshot could not be captured: ${detail}`) + } + if (isCancelled(call, signal)) return cancelled() + if (targetStillAllowed?.() === false) return targetChanged() + const captured = decodeCaptureDataUrl(dataUrl) + if (captured === undefined) return unavailable('The browser returned the screenshot in an unsupported format.') + const dimensions = await measureImageBytes(captured.data, captured.mediaType) + return { + ok: true, + result: { + image: { + data: captured.data, + mediaType: captured.mediaType, + ...dimensions === undefined ? {} : dimensions, + }, + }, + } +} + /** * Dispatch one tool call to the selected tab's content script. * @param call - the tool call to execute. @@ -689,7 +787,7 @@ export async function dispatchToolCall( // Privacy boundary: with sharing off, no page content may leave the page. if (!tabManagement.unrestrictedAccess && sharePageContent === 'off' - && (call.name === 'browser_snapshot' || call.name === 'browser_get_text')) { + && (call.name === 'browser_snapshot' || call.name === 'browser_get_text' || call.name === 'browser_screenshot')) { return { ok: false, error: { code: 'action-failed', message: 'Page content sharing is disabled in Settings > Page content sharing.' } } } const tab = targetTab ?? (await chrome.tabs.query({ active: true, lastFocusedWindow: true }))[0] @@ -703,7 +801,7 @@ export async function dispatchToolCall( if (targetStillAllowed?.() === false) return targetChanged() const frameError = validateFrameTarget(call, frames) if (frameError !== undefined) return frameError - if (!isInjectablePage(tab.url) && !TAB_NATIVE_TOOLS.has(call.name)) { + if (!isInjectablePage(tab.url) && !TAB_NATIVE_TOOLS.has(call.name) && call.name !== 'browser_screenshot') { return unavailable('The current page DOM is protected by the browser. Only snapshot metadata, navigate, back, forward, and reload are available on this page.') } const targetError = validateElementTarget(call, tab.id, frames) @@ -734,9 +832,15 @@ export async function dispatchToolCall( if (refreshedTargetError !== undefined) return refreshedTargetError } if (!isInjectablePage(tab.url)) { + if (call.name === 'browser_screenshot') { + return unavailable('A screenshot cannot be captured on this page: the browser protects pages outside http and https.') + } return await dispatchTabNativeTool(tab.id, tab.url, tab.title, tab.windowId, call, effectiveBudget, signal, targetStillAllowed, tabManagement.commitAction) ?? unavailable('The current page DOM is protected by the browser.') } + if (call.name === 'browser_screenshot') { + return await dispatchScreenshot(tab, call, signal, targetStillAllowed) + } try { return await dispatchOnce( tab.id, diff --git a/extensions/dsh-browser/tests/screenshot.spec.ts b/extensions/dsh-browser/tests/screenshot.spec.ts new file mode 100644 index 000000000..5d08b0d05 --- /dev/null +++ b/extensions/dsh-browser/tests/screenshot.spec.ts @@ -0,0 +1,123 @@ +// @vitest-environment jsdom +import { afterEach, describe, expect, it, vi } from 'vitest' +import { dispatchToolCall, type ToolCall } from '../src/background/tools.ts' + +/** 1x1 transparent PNG. */ +const PNG_PIXEL = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==' + +function stubChrome(tab: Partial, capture?: (options: unknown) => string) { + const full: chrome.tabs.Tab = { + id: 7, + windowId: 3, + index: 0, + active: true, + highlighted: false, + pinned: false, + incognito: false, + selected: false, + discarded: false, + autoDiscardable: true, + groupId: -1, + url: 'https://example.com/', + title: 'Example', + ...tab, + } + const captureVisibleTab = vi.fn(async (_windowId: number, options: unknown) => { + if (capture !== undefined) return capture(options) + return `data:image/png;base64,${PNG_PIXEL}` + }) + vi.stubGlobal('chrome', { + tabs: { + query: vi.fn(async () => [full]), + get: vi.fn(async () => ({ ...full })), + captureVisibleTab, + remove: vi.fn(async () => undefined), + }, + webNavigation: { + getAllFrames: vi.fn(async () => [{ frameId: 0, parentFrameId: -1, documentId: 'document-7', url: full.url ?? '' }]), + }, + }) + return { captureVisibleTab } +} + +const call = (args: Record = {}): ToolCall => ({ id: 'tool-1', name: 'browser_screenshot', args }) + +describe('browser_screenshot dispatch', () => { + afterEach(() => { + vi.unstubAllGlobals() + }) + + it('captures the active http tab and returns base64 image metadata', async () => { + stubChrome({ id: 7, windowId: 3, active: true }) + const answer = await dispatchToolCall(call(), 'auto') + expect(answer.ok).toBe(true) + if (!answer.ok) return + const image = (answer.result as { image: { data: string; mediaType: string } }).image + expect(image.data).toBe(PNG_PIXEL) + expect(image.mediaType).toBe('image/png') + }) + + it('passes format and quality through to captureVisibleTab', async () => { + const { captureVisibleTab } = stubChrome({ active: true }) + await dispatchToolCall(call({ format: 'jpeg', quality: 60 }), 'auto') + expect(captureVisibleTab).toHaveBeenCalledWith(3, { format: 'jpeg', quality: 60 }) + }) + + it('defaults to png without a quality option', async () => { + const { captureVisibleTab } = stubChrome({ active: true }) + await dispatchToolCall(call(), 'auto') + expect(captureVisibleTab).toHaveBeenCalledWith(3, { format: 'png' }) + }) + + it('rejects an invalid format', async () => { + stubChrome({ active: true }) + const answer = await dispatchToolCall(call({ format: 'webp' }), 'auto') + expect(answer.ok).toBe(false) + expect(answer.error?.message).toContain('format') + }) + + it('rejects an out-of-range quality', async () => { + stubChrome({ active: true }) + const answer = await dispatchToolCall(call({ format: 'jpeg', quality: 0 }), 'auto') + expect(answer.ok).toBe(false) + expect(answer.error?.message).toContain('quality') + }) + + it('refuses a controlled tab that is not active', async () => { + stubChrome({ active: false }) + const answer = await dispatchToolCall(call(), 'auto') + expect(answer.ok).toBe(false) + expect(answer.error?.message).toContain('not the active tab') + }) + + it('refuses protected browser pages with a screenshot-specific message', async () => { + stubChrome({ url: 'chrome://settings/', active: true }) + const answer = await dispatchToolCall(call(), 'auto') + expect(answer.ok).toBe(false) + expect(answer.error?.message).toContain('screenshot') + }) + + it('surfaces capture failures from the browser', async () => { + stubChrome({ active: true }, () => { throw new Error('cannot access contents of the page') }) + const answer = await dispatchToolCall(call(), 'auto') + expect(answer.ok).toBe(false) + expect(answer.error?.message).toContain('cannot access contents') + }) + + it('is blocked when page content sharing is off', async () => { + stubChrome({ active: true }) + const answer = await dispatchToolCall(call(), 'off') + expect(answer.ok).toBe(false) + expect(answer.error?.message).toContain('sharing is disabled') + }) + + it('requests read approval when sharing is ask', async () => { + stubChrome({ active: true }) + const authorize = vi.fn(async (_prompt: { kind: string; action: string }) => 'approved' as const) + await dispatchToolCall(call(), 'ask', undefined, authorize) + expect(authorize).toHaveBeenCalledTimes(1) + const prompt = authorize.mock.calls[0]![0]! + expect(prompt.kind).toBe('read') + expect(prompt.action).toBe('browser_screenshot') + }) +}) diff --git a/packages/browser/bridge-browser/src/index.ts b/packages/browser/bridge-browser/src/index.ts index 1d77aa767..266f082e1 100644 --- a/packages/browser/bridge-browser/src/index.ts +++ b/packages/browser/bridge-browser/src/index.ts @@ -29,6 +29,7 @@ import { dshHomePath } from '@deepseek-ai/dsh-home-paths' import { BridgeServer } from './server.ts' import { BrowserContextInjector } from './browser-context.ts' import { registerBrowserTools } from './tools.ts' +import { registerScreenshotTool } from './screenshot-tools.ts' import { BRIDGE_CONFIG_PATH, BRIDGE_PATH, @@ -255,7 +256,13 @@ function mountBridge( snapshotMaxChars: resolved.snapshotMaxChars, maxInteractiveItems: resolved.maxInteractiveItems, }) - return () => { for (const dispose of disposers.values()) dispose() } + const disposeScreenshot = registerScreenshotTool(ctx, server, { + toolTimeoutMs: resolved.toolTimeoutMs, + }) + return () => { + disposeScreenshot() + for (const dispose of disposers.values()) dispose() + } }, 'bridge-browser: browser tools') // Optional system-prompt contribution: a one-line hint only — the model is @@ -267,7 +274,8 @@ function mountBridge( order: 107, text: 'A browser bridge may be connected. To read or operate the user\'s active browser page, call browser_snapshot ' + '(text-only; numbered items are the click/type targets), unless the current turn already includes a plugin-provided ' - + 'followed-page browser_snapshot. Reuse that injected snapshot and its indices directly. Never assume page content you have not snapshotted.', + + 'followed-page browser_snapshot. Reuse that injected snapshot and its indices directly. Never assume page content you have not snapshotted. ' + + 'When visual appearance matters, call browser_screenshot to capture the controlled tab viewport as an image.', }), 'bridge-browser: system prompt section') } diff --git a/packages/browser/bridge-browser/src/screenshot-tools.ts b/packages/browser/bridge-browser/src/screenshot-tools.ts new file mode 100644 index 000000000..60081998b --- /dev/null +++ b/packages/browser/bridge-browser/src/screenshot-tools.ts @@ -0,0 +1,175 @@ +/** + * The screenshot tool: captures the controlled tab's viewport through the + * extension and returns the image as a durable attachment block. + * + * The wire result carries base64 pixels; this side stores them exactly once + * in the attachment store and hands the model an image ContentBlock plus a + * short text summary. The persisted tool value keeps metadata only — no + * base64 — so sessions stay small. Whether the active model can consume the + * image is the runtime's projection concern (text-only routes receive a + * placeholder), and programmatic callers keep the attachment reference. + * + * @module + */ + +import type { Context } from '@deepseek-ai/cordis' +import type { ContentBlock } from '@deepseek-ai/dsh-llm' +import type { ImageAttachmentRef } from '@deepseek-ai/dsh-attachment' +import { defineTool } from '@deepseek-ai/dsh-tools' +import type { BridgeServer } from './server.ts' + +/** The extension's screenshot wire payload. */ +interface ScreenshotImage { + data: string + mediaType: string + width?: number + height?: number +} + +interface ScreenshotResult { + image: ScreenshotImage +} + +/** Options shared with the text browser tools. */ +export interface ScreenshotToolOptions { + toolTimeoutMs: number +} + +const SUPPORTED_MEDIA_TYPES = new Set(['image/png', 'image/jpeg']) + +/** + * Register `browser_screenshot` on `ctx.tools`. + * @param ctx - Cordis context with the tools service; the optional + * `attachments` service stores the captured pixels. + * @param bridge - the authenticated bridge server. + * @param options - resolved tool budgets. + * @returns a disposer for the caller's effect to own. + */ +export function registerScreenshotTool( + ctx: Context, + bridge: BridgeServer, + options: ScreenshotToolOptions, +): () => void { + // exec → projected content for this execution, mirroring the MCP adapter's + // projection handoff: execute() does the async storage, projectContent() + // stays synchronous. + const projections = new WeakMap() + + const requestScreenshot = (exec: { agent?: unknown; signal: AbortSignal }, args: Record): Promise => { + const sessionId = exec.agent === undefined ? undefined : String((exec.agent as { id: unknown }).id) + const call = sessionId === undefined + ? bridge.requestTool('browser_screenshot', args, exec.signal, options.toolTimeoutMs) + : bridge.requestTool('browser_screenshot', args, exec.signal, options.toolTimeoutMs, sessionId) + return call as Promise + } + + const storeImage = async (image: ScreenshotImage): Promise => { + const attachments = ctx.get('attachments') as + | { saveImages(inputs: ReadonlyArray<{ data: Uint8Array; mediaType: string }>): Promise } + | undefined + if (attachments === undefined) { + throw new Error('No attachment store is mounted, so the screenshot cannot be returned as an image.') + } + const mediaType = image.mediaType + if (!SUPPORTED_MEDIA_TYPES.has(mediaType)) { + throw new Error(`The browser returned an unsupported screenshot media type: ${mediaType}`) + } + const data = Buffer.from(image.data, 'base64') + const [ref] = await attachments.saveImages([{ data: new Uint8Array(data), mediaType }]) + return ref as ImageAttachmentRef + } + + const tool = defineTool({ + name: 'browser_screenshot', + description: 'Capture the current viewport of the controlled tab as an image and return it for visual inspection. ' + + 'The image is attached as an image content block. Only the active tab of its window can be captured, ' + + 'and only http/https pages.', + parameters: { + format: { + type: 'string', + enum: ['png', 'jpeg'], + description: 'Image format. png is lossless; jpeg is smaller. Defaults to png.', + }, + quality: { + type: 'number', + description: 'JPEG quality, 1-100. Defaults to 85. Ignored for png.', + }, + }, + timeoutMs: options.toolTimeoutMs, + output: { + schema: { + type: 'object', + additionalProperties: false, + properties: { + text: { type: 'string', required: true }, + image: { + type: 'object', + additionalProperties: false, + properties: { + attachmentId: { type: 'string', required: true }, + mediaType: { type: 'string', required: true }, + bytes: { type: 'number', required: true }, + width: { type: 'number', required: true }, + height: { type: 'number', required: true }, + }, + }, + }, + }, + render: (_args: unknown, value: unknown) => { + const result = value as { text: string } + return [{ type: 'text' as const, text: result.text }] + }, + }, + execute: async (args, exec) => { + const a = args as { format?: 'png' | 'jpeg'; quality?: number } + const wireArgs = (format: 'png' | 'jpeg', quality?: number): Record => ({ + format, + ...(format === 'jpeg' && quality !== undefined ? { quality } : {}), + }) + + const format = a.format ?? 'png' + const result = await requestScreenshot(exec, wireArgs(format, a.quality)) + let ref: ImageAttachmentRef + try { + ref = await storeImage(result.image) + } catch (error) { + // A lossless png can exceed the per-image byte limit on large or + // hidpi viewports; retry once as jpeg before giving up. + if (format === 'png' && isAdmissionLimit(error)) { + const fallback = await requestScreenshot(exec, wireArgs('jpeg', 80)) + ref = await storeImage(fallback.image) + } else { + throw error + } + } + + const summary = `Captured the controlled tab viewport (${ref.width}x${ref.height} ${ref.mediaType}, ${formatBytes(ref.bytes)}).` + const metadata = { + attachmentId: ref.attachmentId, + mediaType: ref.mediaType, + bytes: ref.bytes, + width: ref.width, + height: ref.height, + } + projections.set(exec as object, [ + { type: 'image', attachment: ref }, + { type: 'text', text: summary }, + ]) + return { text: summary, image: metadata } + }, + projectContent: (exec) => projections.get(exec as object), + }) + + const dispose = ctx.tools.register(tool) + return () => dispose() +} + +/** Whether an image-admission refusal was a size limit the retry can address. */ +function isAdmissionLimit(error: unknown): boolean { + return error instanceof Error && 'code' in error + && (error as { code?: unknown }).code === 'IMAGE_TOO_LARGE' +} + +function formatBytes(bytes: number): string { + return bytes >= 1024 ? `${(bytes / 1024).toFixed(1)} KB` : `${bytes} B` +} diff --git a/packages/browser/bridge-browser/tests/screenshot-tools.spec.ts b/packages/browser/bridge-browser/tests/screenshot-tools.spec.ts new file mode 100644 index 000000000..21f1168da --- /dev/null +++ b/packages/browser/bridge-browser/tests/screenshot-tools.spec.ts @@ -0,0 +1,142 @@ +import { describe, expect, it, vi } from 'vitest' +import type { Context } from '@deepseek-ai/cordis' +import type { BridgeServer } from '../src/server.ts' +import { registerScreenshotTool } from '../src/screenshot-tools.ts' + +interface StoredImage { data: Uint8Array; mediaType: string } + +/** 1x1 transparent PNG bytes. */ +const PNG_BASE64 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==' +const JPEG_BASE64 = '/9j/4AAQSkZJRgABAQEAYABgAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwcJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/wAALCAABAAEBAREA/8QAFAABAQAAAAAAAAAAAAAAAAAAAAv/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/8QAFBEBAAAAAAAAAAAAAAAAAAAAAP/aAAwDAQACEQMRAD8AmgA//9k=' + +function makeHarness(options: { + attachments?: { saveImages: (inputs: ReadonlyArray) => Promise } | null + capture?: (args: Record) => { image: { data: string; mediaType: string; width?: number; height?: number } } +} = {}) { + const registered: { name: string; definition: Record }[] = [] + const saved: StoredImage[] = [] + const attachments = options.attachments === null + ? undefined + : options.attachments ?? { + saveImages: vi.fn(async (inputs: ReadonlyArray) => { + saved.push(...inputs) + const bytes = inputs.map((input) => input.data.byteLength) + return inputs.map((input, index) => ({ + attachmentId: `att-${index + 1}`, + mediaType: input.mediaType, + bytes: bytes[index]!, + width: 1, + height: 1, + })) + }), + } + const ctx = { + tools: { + register: vi.fn((definition: { name: string }) => { + registered.push({ name: definition.name, definition: definition as Record }) + return () => {} + }), + }, + get: vi.fn((name: string) => (name === 'attachments' ? attachments : undefined)), + } as unknown as Context + const requestTool = vi.fn(async (_name: string, args: Record) => { + const capture = options.capture ?? (() => ({ image: { data: PNG_BASE64, mediaType: 'image/png' } })) + return capture(args) + }) + const bridge = { requestTool } as unknown as BridgeServer + const dispose = registerScreenshotTool(ctx, bridge, { toolTimeoutMs: 1_000 }) + const tool = registered.find((r) => r.name === 'browser_screenshot')?.definition + if (tool === undefined) throw new Error('screenshot tool was not registered') + return { ctx, requestTool, saved, tool, dispose } +} + +const exec = () => ({ signal: new AbortController().signal }) + +describe('registerScreenshotTool', () => { + it('registers browser_screenshot and disposes', () => { + const { tool, dispose } = makeHarness() + expect(tool.name).toBe('browser_screenshot') + expect(typeof tool.execute).toBe('function') + expect(typeof tool.projectContent).toBe('function') + dispose() + }) + + it('stores the captured pixels once and returns metadata only', async () => { + const { tool, saved, requestTool } = makeHarness() + const result = await (tool.execute as (args: unknown, e: unknown) => Promise)({}, exec()) as { + text: string + image: { attachmentId: string; mediaType: string; bytes: number; width: number; height: number; data?: string } + } + expect(requestTool).toHaveBeenCalledWith('browser_screenshot', { format: 'png' }, expect.anything(), 1_000) + expect(saved).toHaveLength(1) + expect(saved[0]!.mediaType).toBe('image/png') + expect(Buffer.from(saved[0]!.data).toString('base64')).toBe(PNG_BASE64) + expect(result.image.attachmentId).toBe('att-1') + expect(result.image.mediaType).toBe('image/png') + // The durable value must not carry the pixels. + expect(JSON.stringify(result)).not.toContain(PNG_BASE64) + expect(typeof result.text).toBe('string') + }) + + it('projects an image content block for the executing call', async () => { + const { tool } = makeHarness() + const run = exec() + await (tool.execute as (args: unknown, e: unknown) => Promise)({}, run) + const blocks = (tool.projectContent as (e: unknown) => unknown)(run) as Array<{ type: string }> + expect(blocks).toHaveLength(2) + expect(blocks[0]!.type).toBe('image') + expect(blocks[0]).toHaveProperty('attachment.attachmentId', 'att-1') + expect(blocks[1]!.type).toBe('text') + // Other executions have no projection. + expect((tool.projectContent as (e: unknown) => unknown)(exec())).toBeUndefined() + }) + + it('forwards jpeg format and quality to the extension', async () => { + const { tool, requestTool } = makeHarness() + await (tool.execute as (args: unknown, e: unknown) => Promise)({ format: 'jpeg', quality: 70 }, exec()) + expect(requestTool).toHaveBeenCalledWith('browser_screenshot', { format: 'jpeg', quality: 70 }, expect.anything(), 1_000) + }) + + it('retries once as jpeg when png admission hits the byte limit', async () => { + const formats: string[] = [] + const saved: StoredImage[] = [] + const attachments = { + saveImages: vi.fn(async (inputs: ReadonlyArray) => { + if (inputs[0]!.mediaType === 'image/png') { + const error = Error('Image exceeds the configured image-byte limit.') as Error & { code?: string } + error.code = 'IMAGE_TOO_LARGE' + throw error + } + saved.push(...inputs) + return [{ attachmentId: 'att-jpeg', mediaType: 'image/jpeg', bytes: 631, width: 1, height: 1 }] + }), + } + const { tool } = makeHarness({ + attachments, + capture: (args) => { + formats.push(String(args.format)) + return args.format === 'jpeg' + ? { image: { data: JPEG_BASE64, mediaType: 'image/jpeg' } } + : { image: { data: PNG_BASE64, mediaType: 'image/png' } } + }, + }) + const result = await (tool.execute as (args: unknown, e: unknown) => Promise)({}, exec()) as { image: { attachmentId: string } } + expect(formats).toEqual(['png', 'jpeg']) + expect(result.image.attachmentId).toBe('att-jpeg') + expect(saved[0]!.mediaType).toBe('image/jpeg') + }) + + it('fails when no attachment store is mounted', async () => { + const { tool } = makeHarness({ attachments: null }) + await expect((tool.execute as (args: unknown, e: unknown) => Promise)({}, exec())) + .rejects.toThrow(/attachment store/i) + }) + + it('fails on an unsupported media type from the extension', async () => { + const { tool } = makeHarness({ + capture: () => ({ image: { data: 'AAAA', mediaType: 'image/webp' } }), + }) + await expect((tool.execute as (args: unknown, e: unknown) => Promise)({}, exec())) + .rejects.toThrow(/unsupported screenshot media type/) + }) +}) From af0aa9ac971e032debfe3cb4f8e9f7f4e6310e3a Mon Sep 17 00:00:00 2001 From: graydovee Date: Sat, 10 Oct 2026 12:27:22 +0800 Subject: [PATCH 2/4] =?UTF-8?q?fix(panel):=20=E6=94=AF=E6=8C=81=E7=B2=98?= =?UTF-8?q?=E8=B4=B4=E5=92=8C=E6=8B=96=E6=8B=BD=E6=B7=BB=E5=8A=A0=E5=9B=BE?= =?UTF-8?q?=E7=89=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 输入框增加 onPaste、composer-box 增加 onDrop：截图后可直接 ⌘V 粘贴或拖入图片，复用现有上传校验（数量、字节、尺寸、媒体类型）； 宿主不支持图片输入时给出与上传按钮相同的提示。 --- extensions/dsh-browser/src/panel/App.tsx | 39 ++++++++++++++++- .../tests/panel-image-transfer.spec.ts | 43 +++++++++++++++++++ 2 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 extensions/dsh-browser/tests/panel-image-transfer.spec.ts diff --git a/extensions/dsh-browser/src/panel/App.tsx b/extensions/dsh-browser/src/panel/App.tsx index 6d3d93602..2a455186b 100644 --- a/extensions/dsh-browser/src/panel/App.tsx +++ b/extensions/dsh-browser/src/panel/App.tsx @@ -185,6 +185,18 @@ function isAuthFailure(cause: unknown): boolean { && (cause.code.includes('credential') || /\b401\b|\b403\b|invalid[ _-]?key/i.test(cause.message)) } +/** Extract image files from a paste or drop payload; empty when it carries none. */ +export function transferredImageFiles(dataTransfer: DataTransfer | null): File[] { + if (dataTransfer === null) return [] + const files: File[] = [] + for (const item of dataTransfer.items) { + if (item.kind !== 'file' || !item.type.startsWith('image/')) continue + const file = item.getAsFile() + if (file !== null) files.push(file) + } + return files +} + async function discoverOnce( api: PanelApi, profile: RelayProfileDraft, @@ -2294,7 +2306,22 @@ export function App(): React.JSX.Element { )} {error !== null &&
{error}
}