From 67c1430ad7f8ea99279087b866cefdc75be48b0d Mon Sep 17 00:00:00 2001 From: dsh-browser-dev Date: Sat, 10 Oct 2026 03:02:20 +0800 Subject: [PATCH 1/4] feat(extension): add browser_screenshot with viewport capture The tool captures the visible area of the controlled tab as jpeg (default) or png with an optional 1-100 quality, and returns the pixels as base64 beside its text answer so the bridge can save them. chrome.tabs.captureVisibleTab comes first; because it needs /activeTab for the photographed tab and always shoots the active tab, a refused call falls back to Page.captureScreenshot over the DevTools protocol, which needs no activation. The answer names the route that produced the image. The debugger is an optional permission, declared in optional_permissions for both builds and granted on demand from a new Settings section that calls chrome.permissions.request from a user gesture. A missing grant fails with the exact switch that enables it. A screenshot is a page read: it joins the sharing-off privacy gate and the ask-mode read prompt. It is also the one tool the protected-page gate lets through, because the browser takes the picture, not the page. --- extensions/dsh-browser/manifest.firefox.json | 3 + extensions/dsh-browser/manifest.json | 3 + .../src/background/authorization.ts | 20 +- .../dsh-browser/src/background/tools.ts | 171 +++++++++++++- extensions/dsh-browser/src/panel/App.tsx | 48 ++++ extensions/dsh-browser/src/panel/strings.ts | 20 ++ extensions/dsh-browser/src/panel/styles.css | 24 ++ .../tests/background-tools.spec.ts | 223 +++++++++++++++++- .../dsh-browser/tests/firefox-build.spec.ts | 15 ++ 9 files changed, 518 insertions(+), 9 deletions(-) diff --git a/extensions/dsh-browser/manifest.firefox.json b/extensions/dsh-browser/manifest.firefox.json index 8d6dda156..f56db6e43 100644 --- a/extensions/dsh-browser/manifest.firefox.json +++ b/extensions/dsh-browser/manifest.firefox.json @@ -27,6 +27,9 @@ "alarms", "notifications" ], + "optional_permissions": [ + "debugger" + ], "host_permissions": [ "http://*/*", "https://*/*" diff --git a/extensions/dsh-browser/manifest.json b/extensions/dsh-browser/manifest.json index d1311c27b..ccc972a86 100644 --- a/extensions/dsh-browser/manifest.json +++ b/extensions/dsh-browser/manifest.json @@ -15,6 +15,9 @@ "alarms", "notifications" ], + "optional_permissions": [ + "debugger" + ], "host_permissions": [ "http://*/*", "https://*/*" diff --git a/extensions/dsh-browser/src/background/authorization.ts b/extensions/dsh-browser/src/background/authorization.ts index d263a0587..3a86f1740 100644 --- a/extensions/dsh-browser/src/background/authorization.ts +++ b/extensions/dsh-browser/src/background/authorization.ts @@ -5,7 +5,7 @@ import type { TabFrame } from './frames.ts' import type { ApprovalPrompt } from '../security/approval.ts' import { getUiLocale, type UiLocale } from '../i18n.ts' -const PAGE_READS = new Set(['browser_snapshot', 'browser_get_text']) +const PAGE_READS = new Set(['browser_snapshot', 'browser_get_text', 'browser_screenshot']) const STATE_CHANGING_ACTIONS = new Set([ 'browser_click', 'browser_type', @@ -32,9 +32,7 @@ export function approvalPromptForCall( return { kind: 'read', action: call.name, - summary: call.name === 'browser_snapshot' - ? localized(locale, 'Read the current page and accessible iframes', '读取当前页面及可访问 iframe') - : localized(locale, 'Read text from the specified area of the current page', '读取当前页面的指定文本区域'), + summary: readSummary(call, locale), origins: uniqueOrigins(targetFrames, frames), canTrust: false, } @@ -106,6 +104,20 @@ export function originFromUrl(value: string): string | undefined { } } +/** + * Reader-facing purpose of one page-reading tool, shown in the `ask`-mode read + * prompt. A screenshot is a read: it carries page pixels, not just text. + */ +function readSummary(call: ToolCall, locale: UiLocale): string { + if (call.name === 'browser_get_text') { + return localized(locale, 'Read text from the specified area of the current page', '读取当前页面的指定文本区域') + } + if (call.name === 'browser_screenshot') { + return localized(locale, 'Capture the visible area of the current page as an image', '截取当前页面的可见区域为图片') + } + return localized(locale, 'Read the current page and accessible iframes', '读取当前页面及可访问 iframe') +} + function summarizeAction(call: ToolCall, locale: UiLocale): string { const frame = typeof call.args.frame === 'number' && call.args.frame !== 0 ? localized(locale, `, iframe ${call.args.frame}`, `，iframe ${call.args.frame}`) diff --git a/extensions/dsh-browser/src/background/tools.ts b/extensions/dsh-browser/src/background/tools.ts index 0fc3069a1..5055ec3e0 100644 --- a/extensions/dsh-browser/src/background/tools.ts +++ b/extensions/dsh-browser/src/background/tools.ts @@ -1,6 +1,7 @@ /** * Tool dispatch: executes `tool.call` frames in an explicitly selected tab via - * the content script and answers with the text-only result. + * the content script or the DevTools protocol and answers with text, carrying + * captured image bytes alongside it when a screenshot was requested. * * The background service owns tab-affinity policy. Direct callers may omit a * target for backward-compatible active-tab dispatch in isolated tests. @@ -254,6 +255,162 @@ async function dispatchTabNativeTool( return { ok: true, result: { text, navigationPending: true } } } +/** + * Give the compositor one frame to present a tab that was just brought forward. + * `captureVisibleTab` photographs whatever is painted, so shooting immediately + * after `tabs.update` can still catch the previous tab. + */ +async function settleAfterActivation(): Promise { + await new Promise((resolve) => setTimeout(resolve, 120)) +} + +/** Build the answer for a viewport capture, naming the route that produced it. */ +function viewportScreenshotAnswer(base64: string, format: string, route: string, note: string): ToolAnswer { + if (base64 === '') return unavailable('The browser returned an empty screenshot.') + // `format` arrives as the bare subtype ('jpeg' / 'png'): the data URL's own + // `image/...` prefix is stripped before it reaches here. + const bare = format.replace(/^image\//, '') + return { + ok: true, + result: { + text: `Captured the visible area of the controlled tab${note} as image/${bare} ${route}.`, + screenshot: { mime: `image/${bare}`, base64 }, + }, + } +} + +/** + * Capture the visible area of one tab and hand the pixels back to the bridge. + * + * `captureVisibleTab` always photographs the active tab of a window, so a + * background target is brought to the front first — otherwise the model would + * receive a picture of whatever the user happened to be looking at instead. + * + * That call also needs the extension to hold `` or `activeTab` for the + * photographed tab, which is exactly the permission a hosted build can withhold + * from the user's site-access choice; when it is, the call fails with "Either the + * '' or 'activeTab' permission is required". The DevTools protocol is + * therefore the fallback for any failure of that call, and it needs no + * activation at all: its pixels come from the renderer. + */ +async function dispatchScreenshot( + tabId: number, + windowId: number | undefined, + call: ToolCall, + signal?: AbortSignal, +): Promise { + if (isCancelled(call, signal)) return cancelled() + const format = call.args.format === 'png' ? 'png' : 'jpeg' + const requestedQuality = typeof call.args.quality === 'number' ? call.args.quality : 85 + const quality = Math.min(100, Math.max(1, Math.round(requestedQuality))) + try { + const tab = await chrome.tabs.get(tabId) + let activated = false + if (!tab.active) { + await chrome.tabs.update(tabId, { active: true }) + activated = true + // Give the compositor one frame to present the newly activated tab. + await settleAfterActivation() + } + if (isCancelled(call, signal)) return cancelled() + const dataUrl = await chrome.tabs.captureVisibleTab( + windowId ?? tab.windowId, + format === 'png' ? { format: 'png' } : { format: 'jpeg', quality }, + ) + const match = /^data:([^;,]+);base64,(.*)$/s.exec(dataUrl) + if (match === null) return unavailable('The browser returned a screenshot in an unexpected format.') + const note = activated ? ' (the tab was brought to the front first)' : '' + return viewportScreenshotAnswer(match[2] ?? '', match[1] ?? format, 'via chrome.tabs.captureVisibleTab', note) + } catch (error: unknown) { + const detail = error instanceof Error ? error.message : String(error) + return await captureViewportThroughDebugger(tabId, format, quality, detail) + } +} + +/** + * Fallback viewport capture, used when `captureVisibleTab` fails for any reason. + * It waits one frame first so a tab that was just activated has presented, then + * shoots through the protocol. + */ +async function captureViewportThroughDebugger( + tabId: number, + format: string, + quality: number, + captureDetail: string, +): Promise { + return await runWithDebugger(tabId, async (target) => { + try { + await settleAfterActivation() + const answer = await chrome.debugger.sendCommand(target, 'Page.captureScreenshot', { + format, + ...format === 'jpeg' ? { quality } : {}, + fromSurface: true, + }) as { data?: string } + const base64 = typeof answer?.data === 'string' ? answer.data : '' + if (base64 === '') { + return unavailable(`The screenshot could not be taken: ${captureDetail}`) + } + return viewportScreenshotAnswer( + base64, + format, + `through the debugger, because captureVisibleTab was refused: ${captureDetail}`, + '', + ) + } catch (error: unknown) { + const fallbackDetail = error instanceof Error ? error.message : String(error) + return unavailable( + `The screenshot could not be taken: ${captureDetail} (the debugger fallback failed too: ${fallbackDetail})`, + ) + } + }) +} + +/** + * Run one DevTools-protocol task against a tab, attaching first and always + * detaching afterwards. + * + * MV3 gives the extension no way to read renderer pixels or synthesise trusted + * input, so the debugger is the only route to either. Attaching is what raises + * the browser's "being controlled" banner, which is why a session is never held + * open between calls — detaching in `finally` is what takes the banner back down. + * + * `debugger` is an optional permission, so the user has to grant it once from + * the side panel before any protocol route can run; when it is missing the + * answer names the exact switch that grants it. + */ +async function runWithDebugger( + tabId: number, + run: (target: chrome.debugger.Debuggee) => Promise, +): Promise { + let granted = false + try { + granted = await chrome.permissions.contains({ permissions: ['debugger'] }) + } catch { + granted = false + } + if (!granted) { + return unavailable( + 'This needs the "debugger" permission, which this extension has not been granted. Open the side panel Settings and turn on "Screenshot capture (debugger)" to allow it.', + ) + } + const target = { tabId } + try { + await chrome.debugger.attach(target, '1.3') + } catch (error: unknown) { + const detail = error instanceof Error ? error.message : String(error) + return unavailable(`Could not attach the debugger: ${detail}. DevTools open on this tab will block it.`) + } + try { + return await run(target) + } finally { + try { + await chrome.debugger.detach(target) + } catch { + // Already detached; nothing left to release. + } + } +} + function cancelled(): ToolAnswer { return { ok: false, error: { code: 'bridge-closed', message: 'The browser tool call was cancelled.' } } } @@ -689,7 +846,7 @@ export async function dispatchToolCall( // Privacy boundary: with sharing off, no page content may leave the page. if (!tabManagement.unrestrictedAccess && sharePageContent === 'off' - && (call.name === 'browser_snapshot' || call.name === 'browser_get_text')) { + && (call.name === 'browser_snapshot' || call.name === 'browser_get_text' || call.name === 'browser_screenshot')) { return { ok: false, error: { code: 'action-failed', message: 'Page content sharing is disabled in Settings > Page content sharing.' } } } const tab = targetTab ?? (await chrome.tabs.query({ active: true, lastFocusedWindow: true }))[0] @@ -703,7 +860,9 @@ export async function dispatchToolCall( if (targetStillAllowed?.() === false) return targetChanged() const frameError = validateFrameTarget(call, frames) if (frameError !== undefined) return frameError - if (!isInjectablePage(tab.url) && !TAB_NATIVE_TOOLS.has(call.name)) { + // A viewport capture is taken by the browser itself rather than by the page, + // so it needs no injectable document and stays available on protected pages. + if (!isInjectablePage(tab.url) && !TAB_NATIVE_TOOLS.has(call.name) && call.name !== 'browser_screenshot') { return unavailable('The current page DOM is protected by the browser. Only snapshot metadata, navigate, back, forward, and reload are available on this page.') } const targetError = validateElementTarget(call, tab.id, frames) @@ -733,6 +892,12 @@ export async function dispatchToolCall( const refreshedTargetError = validateElementTarget(call, tab.id, executionFrames) if (refreshedTargetError !== undefined) return refreshedTargetError } + // A screenshot is taken by the browser or the renderer rather than by the + // page, so it never goes through the content script. The plain tab capture + // comes first and the DevTools protocol is the fallback. + if (call.name === 'browser_screenshot') { + return await dispatchScreenshot(tab.id, tab.windowId, call, signal) + } if (!isInjectablePage(tab.url)) { return await dispatchTabNativeTool(tab.id, tab.url, tab.title, tab.windowId, call, effectiveBudget, signal, targetStillAllowed, tabManagement.commitAction) ?? unavailable('The current page DOM is protected by the browser.') diff --git a/extensions/dsh-browser/src/panel/App.tsx b/extensions/dsh-browser/src/panel/App.tsx index 6d3d93602..2f8d2a799 100644 --- a/extensions/dsh-browser/src/panel/App.tsx +++ b/extensions/dsh-browser/src/panel/App.tsx @@ -647,6 +647,8 @@ export function App(): React.JSX.Element { const [approvalQueue, setApprovalQueue] = useState([]) const [tabAffinity, setTabAffinity] = useState(null) const [trustedOriginInput, setTrustedOriginInput] = useState('') + const [debuggerGranted, setDebuggerGranted] = useState(null) + const [permissionNotice, setPermissionNotice] = useState(null) const [error, setError] = useState(null) const [showSessionPicker, setShowSessionPicker] = useState(false) const [loadingSessions, setLoadingSessions] = useState(false) @@ -1633,6 +1635,40 @@ export function App(): React.JSX.Element { } } + /** + * Ask for the optional `debugger` permission behind the screenshot fallback. + * + * It is a user-gesture-only prompt, which is why the request lives on this + * explicit button rather than inside tool dispatch, and why nothing else in + * the panel ever calls it. + */ + async function grantDebuggerPermission(): Promise { + try { + const granted = await chrome.permissions.request({ permissions: ['debugger'] }) + setDebuggerGranted(granted) + setPermissionNotice(granted ? copy.settings.screenshotGranted : copy.settings.screenshotDenied) + } catch (cause) { + setPermissionNotice(copy.settings.screenshotGrantFailed(cause instanceof Error ? cause.message : String(cause))) + } + } + + /** Reflect the stored grant state every time the settings view is opened. */ + useEffect(() => { + if (!showSettings) return + let cancelled = false + const read = async (): Promise => { + try { + const granted = await chrome.permissions.contains({ permissions: ['debugger'] }) + if (!cancelled) setDebuggerGranted(granted) + } catch { + // Without the API there is no grant to report, and the button stays usable. + if (!cancelled) setDebuggerGranted(false) + } + } + void read() + return () => { cancelled = true } + }, [showSettings]) + /** Load relay profiles from the llm-pi-ai settings namespace once per settings visit. */ useEffect(() => { if (!showSettings || relayLoaded) return @@ -2024,6 +2060,18 @@ export function App(): React.JSX.Element {