feat(browser): add a viewport screenshot tool and sidebar image paste

extensions/dsh-browser/manifest.firefox.json

@@ -28,8 +28,7 @@
     "notifications"
   ],
   "host_permissions": [
-    "http://*/*",
-    "https://*/*"
+    "<all_urls>"
   ],
   "background": {
     "scripts": [

extensions/dsh-browser/manifest.json

@@ -16,8 +16,7 @@
     "notifications"
   ],
   "host_permissions": [
-    "http://*/*",
-    "https://*/*"
+    "<all_urls>"
   ],
   "background": {
     "service_worker": "background.js",

extensions/dsh-browser/src/background/authorization.ts

@@ -24,11 +24,23 @@ export function approvalPromptForCall(
   frames: TabFrame[],
   locale: UiLocale = getUiLocale(),
 ): ApprovalPrompt | undefined {
+  // A screenshot photographs exactly what the user sees — passwords, private
+  // messages, account balances included — so it always prompts, regardless of
+  // the page-content sharing preference. Only unrestricted access skips it.
+  if (call.name === 'browser_screenshot') {
+    return {
+      kind: 'read',
+      action: call.name,
+      summary: localized(locale, 'Capture a screenshot of the current page (may contain sensitive content)', '截取当前页面的可视区域截图（可能包含敏感信息）'),
+      origins: uniqueOrigins(frames, frames),
+      canTrust: false,
+    }
+  }
   if (PAGE_READS.has(call.name)) {
     if (sharePageContent !== 'ask') return undefined
-    const targetFrames = call.name === 'browser_snapshot'
-      ? frames
-      : frames.filter((frame) => frame.frameId === requestedFrame(call.args))
+    const targetFrames = call.name === 'browser_get_text'
+      ? frames.filter((frame) => frame.frameId === requestedFrame(call.args))
+      : frames
     return {
       kind: 'read',
       action: call.name,

extensions/dsh-browser/src/background/tools.ts

@@ -655,6 +655,104 @@ async function dispatchTabManagementTool(
   return unavailable(`Unsupported tab-management tool: ${call.name}`)
 }
 
+/** Parse the model-supplied screenshot capture options. */
+function screenshotCaptureOptions(call: ToolCall): { format: 'png' | 'jpeg'; quality: number } | ToolAnswer {
+  const format = call.args.format === undefined ? 'png' : call.args.format
+  if (format !== 'png' && format !== 'jpeg') {
+    return { ok: false, error: { code: 'action-failed', message: 'format must be "png" or "jpeg".' } }
+  }
+  const quality = call.args.quality === undefined ? 85 : call.args.quality
+  if (typeof quality !== 'number' || !Number.isInteger(quality) || quality < 1 || quality > 100) {
+    return { ok: false, error: { code: 'action-failed', message: 'quality must be an integer between 1 and 100.' } }
+  }
+  return { format, quality }
+}
+
+/** Decode a captureVisibleTab data URL into base64 pixels and media type. */
+function decodeCaptureDataUrl(dataUrl: string): { data: string; mediaType: string } | undefined {
+  const comma = dataUrl.indexOf(',')
+  if (comma === -1) return undefined
+  const header = /^data:([^;,]+);base64$/.exec(dataUrl.slice(0, comma))
+  if (header === null) return undefined
+  const data = dataUrl.slice(comma + 1)
+  return data === '' ? undefined : { data, mediaType: header[1] ?? '' }
+}
+
+async function measureImageBytes(data: string, mediaType: string): Promise<{ width: number; height: number } | undefined> {
+  try {
+    const binary = atob(data)
+    const bytes = new Uint8Array(binary.length)
+    for (let index = 0; index < binary.length; index += 1) bytes[index] = binary.charCodeAt(index)
+    const bitmap = await createImageBitmap(new Blob([bytes as BlobPart], { type: mediaType }))
+    try {
+      return { width: bitmap.width, height: bitmap.height }
+    } finally {
+      bitmap.close()
+    }
+  } catch {
+    // Dimensions are advisory metadata; the capture stays deliverable.
+    return undefined
+  }
+}
+
+/**
+ * Capture the controlled tab's viewport. `captureVisibleTab` photographs the
+ * active tab of the window, so a controlled-but-inactive tab is a hard error
+ * rather than a silent capture of the wrong page.
+ */
+async function dispatchScreenshot(
+  tab: Pick<chrome.tabs.Tab, 'id' | 'url'> & { active?: boolean; windowId?: number },
+  call: ToolCall,
+  signal?: AbortSignal,
+  targetStillAllowed?: () => boolean,
+): Promise<ToolAnswer> {
+  if (isCancelled(call, signal)) return cancelled()
+  if (targetStillAllowed?.() === false) return targetChanged()
+  const options = screenshotCaptureOptions(call)
+  if ('ok' in options) return options
+  let active = tab.active
+  let windowId = tab.windowId
+  if (active === undefined || windowId === undefined) {
+    const fresh = await findTab(tab.id!)
+    if (fresh === undefined) return unavailable('The controlled tab closed before the screenshot was captured.')
+    active = fresh.active
+    windowId = fresh.windowId
+  }
+  if (isCancelled(call, signal)) return cancelled()
+  if (targetStillAllowed?.() === false) return targetChanged()
+  if (active !== true) {
+    return unavailable('The controlled tab is not the active tab of its window, so its viewport cannot be captured. Activate the tab first, then retry.')
+  }
+  if (windowId === undefined) {
+    return unavailable('The controlled tab has no window, so its viewport cannot be captured.')
+  }
+  let dataUrl: string
+  try {
+    dataUrl = await chrome.tabs.captureVisibleTab(windowId, {
+      format: options.format,
+      ...(options.format === 'jpeg' ? { quality: options.quality } : {}),
+    })
+  } catch (error: unknown) {
+    const detail = error instanceof Error ? error.message : String(error)
+    return unavailable(`The screenshot could not be captured: ${detail}`)
+  }
+  if (isCancelled(call, signal)) return cancelled()
+  if (targetStillAllowed?.() === false) return targetChanged()
+  const captured = decodeCaptureDataUrl(dataUrl)
+  if (captured === undefined) return unavailable('The browser returned the screenshot in an unsupported format.')
+  const dimensions = await measureImageBytes(captured.data, captured.mediaType)
+  return {
+    ok: true,
+    result: {
+      image: {
+        data: captured.data,
+        mediaType: captured.mediaType,
+        ...dimensions === undefined ? {} : dimensions,
+      },
+    },
+  }
+}
+
 /**
  * Dispatch one tool call to the selected tab's content script.
  * @param call - the tool call to execute.
@@ -689,7 +787,7 @@ export async function dispatchToolCall(
   // Privacy boundary: with sharing off, no page content may leave the page.
   if (!tabManagement.unrestrictedAccess
     && sharePageContent === 'off'
-    && (call.name === 'browser_snapshot' || call.name === 'browser_get_text')) {
+    && (call.name === 'browser_snapshot' || call.name === 'browser_get_text' || call.name === 'browser_screenshot')) {
     return { ok: false, error: { code: 'action-failed', message: 'Page content sharing is disabled in Settings > Page content sharing.' } }
   }
   const tab = targetTab ?? (await chrome.tabs.query({ active: true, lastFocusedWindow: true }))[0]
@@ -703,7 +801,7 @@ export async function dispatchToolCall(
   if (targetStillAllowed?.() === false) return targetChanged()
   const frameError = validateFrameTarget(call, frames)
   if (frameError !== undefined) return frameError
-  if (!isInjectablePage(tab.url) && !TAB_NATIVE_TOOLS.has(call.name)) {
+  if (!isInjectablePage(tab.url) && !TAB_NATIVE_TOOLS.has(call.name) && call.name !== 'browser_screenshot') {
     return unavailable('The current page DOM is protected by the browser. Only snapshot metadata, navigate, back, forward, and reload are available on this page.')
   }
   const targetError = validateElementTarget(call, tab.id, frames)
@@ -734,9 +832,15 @@ export async function dispatchToolCall(
     if (refreshedTargetError !== undefined) return refreshedTargetError
   }
   if (!isInjectablePage(tab.url)) {
+    if (call.name === 'browser_screenshot') {
+      return unavailable('A screenshot cannot be captured on this page: the browser protects pages outside http and https.')
+    }
     return await dispatchTabNativeTool(tab.id, tab.url, tab.title, tab.windowId, call, effectiveBudget, signal, targetStillAllowed, tabManagement.commitAction)
       ?? unavailable('The current page DOM is protected by the browser.')
   }
+  if (call.name === 'browser_screenshot') {
+    return await dispatchScreenshot(tab, call, signal, targetStillAllowed)
+  }
   try {
     return await dispatchOnce(
       tab.id,

extensions/dsh-browser/src/panel/App.tsx

@@ -185,6 +185,18 @@ function isAuthFailure(cause: unknown): boolean {
     && (cause.code.includes('credential') || /\b401\b|\b403\b|invalid[ _-]?key/i.test(cause.message))
 }
 
+/** Extract image files from a paste or drop payload; empty when it carries none. */
+export function transferredImageFiles(dataTransfer: DataTransfer | null): File[] {
+  if (dataTransfer === null) return []
+  const files: File[] = []
+  for (const item of dataTransfer.items) {
+    if (item.kind !== 'file' || !item.type.startsWith('image/')) continue
+    const file = item.getAsFile()
+    if (file !== null) files.push(file)
+  }
+  return files
+}
+
 async function discoverOnce(
   api: PanelApi,
   profile: RelayProfileDraft,
@@ -2294,7 +2306,22 @@ export function App(): React.JSX.Element {
       )}
       {error !== null && <div className="error">{error}</div>}
       <footer className="composer">
-        <div className="composer-box">
+        <div
+          className="composer-box"
+          onDragOver={(event) => {
+            if (event.dataTransfer.types.includes('Files')) event.preventDefault()
+          }}
+          onDrop={(event) => {
+            const files = [...event.dataTransfer.files].filter((file) => file.type.startsWith('image/'))
+            if (files.length === 0) return
+            event.preventDefault()
+            if (imageLimits === null) {
+              setError(copy.app.imageUnavailable)
+              return
+            }
+            void addImageFiles(files)
+          }}
+        >
           {selection !== null && (
             <SelectionQuote
               selection={{ ...selection, quote: selection.text }}
@@ -2337,6 +2364,16 @@ export function App(): React.JSX.Element {
                 void send()
               }
             }}
+            onPaste={(e) => {
+              const files = transferredImageFiles(e.clipboardData)
+              if (files.length === 0) return
+              e.preventDefault()
+              if (imageLimits === null) {
+                setError(copy.app.imageUnavailable)
+                return
+              }
+              void addImageFiles(files)
+            }}
             placeholder={state === 'connected' ? copy.app.connectedPlaceholder : copy.app.disconnectedPlaceholder}
             disabled={!sessionReady || busy}
             rows={2}

extensions/dsh-browser/tests/panel-image-transfer.spec.ts

@@ -0,0 +1,43 @@
+// @vitest-environment jsdom
+import { describe, expect, it } from 'vitest'
+import { transferredImageFiles } from '../src/panel/App.tsx'
+
+function transferData(items: Array<{ kind: string; type: string; file?: File }>): DataTransfer {
+  // jsdom has no DataTransfer constructor; the handler only reads `items`.
+  return {
+    items: items.map((item) => ({
+      kind: item.kind,
+      type: item.type,
+      getAsFile: () => item.file ?? null,
+    })),
+  } as unknown as DataTransfer
+}
+
+describe('transferredImageFiles', () => {
+  const png = new File([new Uint8Array([137, 80, 78, 71])], 'shot.png', { type: 'image/png' })
+  const txt = new File(['hi'], 'note.txt', { type: 'text/plain' })
+
+  it('collects image files from a paste payload', () => {
+    expect(transferredImageFiles(transferData([
+      { kind: 'string', type: 'text/plain' },
+      { kind: 'file', type: 'image/png', file: png },
+    ]))).toEqual([png])
+  })
+
+  it('ignores non-image files', () => {
+    expect(transferredImageFiles(transferData([
+      { kind: 'file', type: 'text/plain', file: txt },
+    ]))).toEqual([])
+  })
+
+  it('returns empty for a text-only paste and a null payload', () => {
+    expect(transferredImageFiles(transferData([{ kind: 'string', type: 'text/plain' }]))).toEqual([])
+    expect(transferredImageFiles(null)).toEqual([])
+  })
+
+  it('skips declared image items whose file could not be produced', () => {
+    expect(transferredImageFiles(transferData([
+      { kind: 'file', type: 'image/png' },
+    ]))).toEqual([])
+  })
+})