From 0e9f9935fe16efc87751b762adeaa04cf9a06202 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Sun, 20 Sep 2026 15:33:09 +0800 Subject: [PATCH] feat(extension): answer remote gateway UI requests outside the tool queue MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A gateway that runs tasks for a user renders its own view of the running task: a preview of the page the agent is working on, and a control that brings that tab to the front. Neither can be served by the tool RPCs, because those are serialized per session, so both go dark exactly when the user wants to look — during a long navigation, or while request_help waits for an answer. Answer two optional requests, ui.task_preview and ui.task_focus, on the socket before the transport sees them. They never enter the tool dispatcher, never start a session, and exist only on authenticated remote sockets. The preview captures the task's own tab, keeps the control and help overlays visible so a periodic poll does not make them flicker, and returns a JPEG frame bounded to 640 pixels wide. Captures are coalesced per task, at most one runs per tab, a poll that arrives while Chrome still holds one is refused rather than queued behind it, and authorization, the document revision and the debugger identity are re-checked before a frame is returned. session.stop drains an in-flight capture before it detaches the debugger and closes tabs. --- apps/extension/src/entrypoints/background.ts | 20 +- .../src/lib/__tests__/task-preview.test.ts | 193 ++++++++++++++++++ .../src/lib/__tests__/ui-channel.test.ts | 55 +++++ apps/extension/src/lib/task-preview.ts | 192 +++++++++++++++++ apps/extension/src/lib/ui-channel.ts | 57 ++++++ apps/extension/src/tools/session.ts | 9 + docs/remote-extension-connection.md | 15 +- 7 files changed, 539 insertions(+), 2 deletions(-) create mode 100644 apps/extension/src/lib/__tests__/task-preview.test.ts create mode 100644 apps/extension/src/lib/__tests__/ui-channel.test.ts create mode 100644 apps/extension/src/lib/task-preview.ts create mode 100644 apps/extension/src/lib/ui-channel.ts diff --git a/apps/extension/src/entrypoints/background.ts b/apps/extension/src/entrypoints/background.ts index 830bbbe9..d547b673 100644 --- a/apps/extension/src/entrypoints/background.ts +++ b/apps/extension/src/entrypoints/background.ts @@ -27,6 +27,8 @@ import { import { POPUP_PORT_NAME, type PopupInbound, type PopupOutbound } from "@/lib/popup-bridge"; import { recordFrameCoordinator } from "@/lib/recording/frame-coordinator"; import { attachSessionsLiveFlag } from "@/lib/sessions-live-flag"; +import { captureTaskPreview, taskTarget } from "@/lib/task-preview"; +import { attachUiChannel } from "@/lib/ui-channel"; import { attachLongScreenshot } from "@/long-screenshot/background"; import { createDisconnectCleanup } from "@/session-manager/disconnect-cleanup"; import { attachSessionEventHandler } from "@/session-manager/event-handler"; @@ -63,7 +65,23 @@ export default defineBackground(() => { url: __BSK_DAEMON_WS_URL__, webSocketFactory: (url) => { if (!connectionPreferenceValid) throw new Error("Connection settings are unavailable"); - return remoteSocket(url, remoteEndpoint); + const socket = remoteSocket(url, remoteEndpoint); + // Registered before the transport listens, so `ui.*` frames are answered + // here instead of reaching the tool dispatcher. Remote gateways only. + if (remoteEndpoint) + attachUiChannel(socket, { + focus: async (sessionId) => { + const tabId = await taskTarget(sessions, sessionId); + const task = sessions.get(sessionId); + const tab = await chrome.tabs.get(tabId); + if (!task || !isAgentControlledTab(task, tabId)) throw new Error("Task unavailable"); + await chrome.tabs.update(tabId, { active: true }); + await chrome.windows.update(tab.windowId, { focused: true }); + return { focused: true }; + }, + preview: (sessionId) => captureTaskPreview(sessions, cdp, sessionId), + }); + return socket; }, }); const sessions = new SessionManager({ remote: () => remoteEndpoint !== null }); diff --git a/apps/extension/src/lib/__tests__/task-preview.test.ts b/apps/extension/src/lib/__tests__/task-preview.test.ts new file mode 100644 index 00000000..f35efa10 --- /dev/null +++ b/apps/extension/src/lib/__tests__/task-preview.test.ts @@ -0,0 +1,193 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { ChromiumCdp } from "@/browser-driver/chromium-cdp"; +import type { SessionManager } from "@/session-manager/manager"; +import { captureTaskPreview, withTaskPreviewStop } from "../task-preview"; + +afterEach(() => { + vi.unstubAllGlobals(); + vi.useRealTimers(); + vi.restoreAllMocks(); +}); + +function fixture() { + const task = { + remote: true, + agentWindowId: 10, + refStore: { documentRevision: () => 1 }, + agentCreatedTabs: new Set([5]), + borrowedTabs: new Map(), + }; + const manager = { get: () => task } as unknown as SessionManager; + const cdp = { + getAttachmentId: vi.fn(() => "attachment"), + acquireBackgroundExecution: vi.fn(async () => {}), + send: vi.fn(async () => ({ data: btoa("jpeg") })), + } as unknown as ChromiumCdp; + const draw = vi.fn(); + const close = vi.fn(); + const sizes: number[] = []; + vi.stubGlobal("chrome", { + tabs: { + query: vi.fn(async () => [{ id: 5, windowId: 10, active: true }]), + get: vi.fn(async () => ({ id: 5, windowId: 10, title: "task" })), + sendMessage: vi.fn(async () => ({ ok: true })), + }, + }); + vi.stubGlobal( + "createImageBitmap", + vi.fn(async () => ({ width: 1280, height: 720, close })), + ); + vi.stubGlobal( + "OffscreenCanvas", + class { + width: number; + height: number; + constructor(w: number, h: number) { + this.width = w; + this.height = h; + sizes.push(w, h); + } + getContext() { + return { drawImage: draw }; + } + async convertToBlob() { + return new Blob(["thumbnail"]); + } + }, + ); + return { task, manager, cdp, sizes, close }; +} + +describe("task preview", () => { + it("keeps the control and help overlays visible across repeated captures", async () => { + const f = fixture(); + for (let i = 0; i < 3; i++) { + await expect(captureTaskPreview(f.manager, f.cdp, "visible")).resolves.toMatchObject({ + tab_id: 5, + image_base64: btoa("thumbnail"), + }); + } + expect(f.cdp.acquireBackgroundExecution).toHaveBeenCalledWith("visible", 5); + expect(f.cdp.send).toHaveBeenCalledTimes(3); + expect(chrome.tabs.sendMessage).not.toHaveBeenCalled(); + }); + + it("coalesces concurrent polls and bounds the encoded frame on HiDPI screens", async () => { + const f = fixture(); + const first = captureTaskPreview(f.manager, f.cdp, "one"); + expect(captureTaskPreview(f.manager, f.cdp, "one")).toBe(first); + expect(await first).toMatchObject({ + tab_id: 5, + format: "jpeg", + image_base64: btoa("thumbnail"), + }); + expect(f.sizes).toEqual([640, 360]); + expect(f.close).toHaveBeenCalled(); + expect(f.cdp.send).toHaveBeenCalledExactlyOnceWith(5, "Page.captureScreenshot", { + format: "jpeg", + quality: 50, + fromSurface: true, + captureBeyondViewport: false, + }); + }); + + it("refuses a further poll while Chrome still holds a capture, and recovers after it", async () => { + vi.useFakeTimers(); + const f = fixture(); + let finish!: (frame: { data: string }) => void; + vi.mocked(f.cdp.send).mockReturnValueOnce( + new Promise((resolve) => { + finish = resolve; + }) as never, + ); + const stuck = expect(captureTaskPreview(f.manager, f.cdp, "stuck")).rejects.toThrow( + "timed out after 3000ms", + ); + await vi.advanceTimersByTimeAsync(3_000); + await stuck; + await expect(captureTaskPreview(f.manager, f.cdp, "stuck")).rejects.toThrow( + "still running in Chrome", + ); + expect(f.cdp.send).toHaveBeenCalledTimes(1); + finish({ data: btoa("late") }); + await vi.advanceTimersByTimeAsync(0); + await expect(captureTaskPreview(f.manager, f.cdp, "stuck")).resolves.toMatchObject({ + tab_id: 5, + }); + expect(vi.getTimerCount()).toBe(0); + }); + + it("lets session.stop proceed once a stuck capture has timed out", async () => { + vi.useFakeTimers(); + const f = fixture(); + vi.mocked(f.cdp.send).mockReturnValueOnce(new Promise(() => {}) as never); + const stuck = expect(captureTaskPreview(f.manager, f.cdp, "stop")).rejects.toThrow("timed out"); + const stop = vi.fn(async () => "stopped"); + const released = withTaskPreviewStop(f.manager, "stop", stop); + await expect(captureTaskPreview(f.manager, f.cdp, "stop")).rejects.toThrow("stopping"); + expect(stop).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(3_000); + await stuck; + await expect(released).resolves.toBe("stopped"); + expect(stop).toHaveBeenCalledOnce(); + }); + + it("blocks previews throughout cleanup and restores them after a failed stop", async () => { + const f = fixture(); + let reject!: (error: Error) => void; + const pending = new Promise((_, fail) => { + reject = fail; + }); + const stopping = withTaskPreviewStop(f.manager, "retry", () => pending); + const checked = expect(stopping).rejects.toThrow("return failed"); + await expect(captureTaskPreview(f.manager, f.cdp, "retry")).rejects.toThrow("stopping"); + expect(f.cdp.acquireBackgroundExecution).not.toHaveBeenCalled(); + reject(new Error("return failed")); + await checked; + await expect(captureTaskPreview(f.manager, f.cdp, "retry")).resolves.toMatchObject({ + tab_id: 5, + }); + }); + + it("does not capture an unowned active user tab", async () => { + const f = fixture(); + f.task.agentCreatedTabs.clear(); + await expect(captureTaskPreview(f.manager, f.cdp, "empty")).rejects.toThrow( + "Task tab unavailable", + ); + expect(f.cdp.send).not.toHaveBeenCalled(); + }); + + it("uses an owned tab when an unauthorized user tab is active", async () => { + const f = fixture(); + vi.mocked(chrome.tabs.query).mockResolvedValue([ + { id: 9, windowId: 10, active: true }, + { id: 5, windowId: 10, active: false }, + ] as chrome.tabs.Tab[]); + await expect(captureTaskPreview(f.manager, f.cdp, "user-active")).resolves.toMatchObject({ + tab_id: 5, + }); + expect(f.cdp.send).toHaveBeenCalledWith(5, "Page.captureScreenshot", expect.anything()); + }); + + it("discards the frame when authorization ends during the capture", async () => { + const f = fixture(); + vi.mocked(f.cdp.acquireBackgroundExecution).mockImplementation(async () => { + f.task.agentCreatedTabs.clear(); + }); + await expect(captureTaskPreview(f.manager, f.cdp, "ended")).rejects.toThrow( + "Task ended during capture", + ); + }); + + it("discards the frame when the debugger attachment changes", async () => { + const f = fixture(); + vi.mocked(f.cdp.send).mockImplementation(async () => { + vi.mocked(f.cdp.getAttachmentId).mockReturnValue("replacement"); + return { data: btoa("jpeg") } as never; + }); + await expect(captureTaskPreview(f.manager, f.cdp, "replaced")).rejects.toThrow( + "Task ended during capture", + ); + }); +}); diff --git a/apps/extension/src/lib/__tests__/ui-channel.test.ts b/apps/extension/src/lib/__tests__/ui-channel.test.ts new file mode 100644 index 00000000..69935484 --- /dev/null +++ b/apps/extension/src/lib/__tests__/ui-channel.test.ts @@ -0,0 +1,55 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { attachUiChannel } from "../ui-channel"; + +afterEach(() => vi.unstubAllGlobals()); + +function fixture() { + vi.stubGlobal("WebSocket", { OPEN: 1 }); + const socket = Object.assign(new EventTarget(), { readyState: 1, send: vi.fn() }); + const handlers = { + focus: vi.fn(async () => ({ focused: true })), + preview: vi.fn(async () => ({ image_base64: "jpeg" })), + }; + attachUiChannel(socket as unknown as WebSocket, handlers); + const native = vi.fn(); + socket.addEventListener("message", native); + const request = async (body: unknown) => { + socket.dispatchEvent(new MessageEvent("message", { data: JSON.stringify(body) })); + await new Promise((resolve) => setTimeout(resolve, 0)); + }; + return { socket, handlers, native, request }; +} + +describe("remote UI channel", () => { + it("answers UI requests without reaching the native transport", async () => { + const f = fixture(); + for (const name of ["preview", "focus"] as const) { + await f.request({ id: name, method: `ui.task_${name}`, params: { session_id: "task" } }); + expect(f.handlers[name]).toHaveBeenCalledWith("task"); + } + expect(f.native).not.toHaveBeenCalled(); + await f.request({ id: "native", method: "tool.snapshot", params: {} }); + expect(f.native).toHaveBeenCalledOnce(); + }); + + it("rejects an invalid session id before any browser work", async () => { + const f = fixture(); + await f.request({ id: "bad", method: "ui.task_focus", params: { session_id: 42 } }); + expect(f.handlers.focus).not.toHaveBeenCalled(); + expect(JSON.parse(f.socket.send.mock.calls[0]![0])).toMatchObject({ + error: { code: "invalid_params" }, + }); + }); + + it("reports a failed request and drops a late reply on a closed socket", async () => { + const f = fixture(); + f.handlers.focus.mockRejectedValueOnce(new Error("unavailable")); + await f.request({ id: "focus", method: "ui.task_focus", params: { session_id: "task" } }); + expect(JSON.parse(f.socket.send.mock.calls[0]![0])).toMatchObject({ + error: { code: "not_found" }, + }); + f.socket.readyState = 3; + await f.request({ id: "late", method: "ui.task_preview", params: { session_id: "task" } }); + expect(f.socket.send).toHaveBeenCalledTimes(1); + }); +}); diff --git a/apps/extension/src/lib/task-preview.ts b/apps/extension/src/lib/task-preview.ts new file mode 100644 index 00000000..cbbd62e7 --- /dev/null +++ b/apps/extension/src/lib/task-preview.ts @@ -0,0 +1,192 @@ +import type { ChromiumCdp } from "@/browser-driver/chromium-cdp"; +import { isAgentControlledTab, type SessionManager } from "@/session-manager/manager"; + +/** Longest a poll waits for Chrome before the caller is told to try again. */ +const CAPTURE_DEADLINE_MS = 3_000; +/** Widest encoded preview frame, in image pixels. */ +const PREVIEW_WIDTH = 640; + +interface PreviewState { + pending?: Promise; + stopping: number; +} +const previews = new WeakMap(); +function previewState(task: object): PreviewState { + let state = previews.get(task); + if (!state) { + state = { stopping: 0 }; + previews.set(task, state); + } + return state; +} + +/** + * One UI frame of the task's own tab, captured outside the tool queue so a + * preview still answers while the session is navigating or waiting for human + * help. Concurrent polls share the frame that is already in flight. + */ +export function captureTaskPreview( + manager: SessionManager, + cdp: ChromiumCdp, + sessionId: string, +): Promise { + const task = manager.get(sessionId); + if (!task?.remote) return Promise.reject(new Error("Task unavailable")); + const state = previewState(task); + if (state.stopping) return Promise.reject(new Error("Task is stopping")); + if (state.pending) return state.pending; + const work = capture(manager, cdp, sessionId).finally(() => { + if (state.pending === work) state.pending = undefined; + }); + state.pending = work; + return work; +} + +/** + * Drains an in-flight preview before `session.stop` detaches the debugger and + * closes the task's tabs, and refuses new ones for the duration. Session state + * is untouched: a stop that fails can be retried and previews resume. + */ +export async function withTaskPreviewStop( + manager: SessionManager, + sessionId: string | undefined, + stop: () => Promise, +): Promise { + const task = sessionId ? manager.get(sessionId) : undefined; + if (!task?.remote) return stop(); + const state = previewState(task); + state.stopping++; + try { + await state.pending?.catch(() => {}); + return await stop(); + } finally { + state.stopping--; + } +} + +async function capture(manager: SessionManager, cdp: ChromiumCdp, sessionId: string) { + const task = manager.get(sessionId); + if (!task?.remote) throw new Error("Task unavailable"); + const tabId = await taskTarget(manager, sessionId); + if (!isAgentControlledTab(task, tabId)) throw new Error("Task tab unavailable"); + const tab = await chrome.tabs.get(tabId); + // The preview does not pass through the dispatcher, so it acquires the same + // explicit background-execution claim the tools use before reading a page + // that is not in the foreground. `session.stop` drains this capture before + // releasing the claim and closing the tab. + await cdp.acquireBackgroundExecution(sessionId, tabId); + const attachment = cdp.getAttachmentId(tabId); + const revision = task.refStore.documentRevision(tabId); + if (manager.get(sessionId) !== task || !isAgentControlledTab(task, tabId)) + throw new Error("Task ended during capture"); + // UI frames keep the extension's own control and help overlays: hiding and + // restoring them on every poll would make them flicker in the user's browser. + // Tool screenshots suppress the overlays separately, for unobstructed content. + // Only the viewport is captured; the bitmap is scaled below without a layout read. + const shot = await captureViewport(cdp, tabId); + const stillOurs = () => + manager.get(sessionId) === task && + isAgentControlledTab(task, tabId) && + cdp.getAttachmentId(tabId) === attachment && + task.refStore.documentRevision(tabId) === revision; + if (!stillOurs()) throw new Error("Task ended during capture"); + const data = await downscale(shot.data); + if (!stillOurs()) throw new Error("Task ended during capture"); + return { + image_base64: data, + format: "jpeg", + tab_id: tabId, + title: tab.title ?? "", + captured_at: new Date().toISOString(), + }; +} + +const capturesInFlight = new WeakMap>>(); + +/** + * At most one capture per tab, and a bounded wait for the caller. + * + * A local deadline cannot cancel a CDP command, so the fence is released only + * when Chrome actually answers (or the debugger detaches). A poll that arrives + * while a capture is stuck is refused instead of queueing another one behind it. + */ +async function captureViewport(cdp: ChromiumCdp, tabId: number): Promise<{ data: string }> { + let inFlight = capturesInFlight.get(cdp); + if (!inFlight) { + inFlight = new Map(); + capturesInFlight.set(cdp, inFlight); + } + if (inFlight.has(tabId)) { + throw new Error(`Previous preview capture is still running in Chrome (tab ${tabId})`); + } + const shot = cdp.send<{ data: string }>(tabId, "Page.captureScreenshot", { + format: "jpeg", + quality: 50, + fromSurface: true, + captureBeyondViewport: false, + }); + inFlight.set(tabId, shot); + const release = () => { + if (inFlight.get(tabId) === shot) inFlight.delete(tabId); + }; + void shot.then(release, release); + let deadline: ReturnType | undefined; + try { + return await Promise.race([ + shot, + new Promise((_, reject) => { + deadline = setTimeout( + () => reject(new Error(`Preview capture timed out after ${CAPTURE_DEADLINE_MS}ms`)), + CAPTURE_DEADLINE_MS, + ); + }), + ]); + } finally { + clearTimeout(deadline); + } +} + +/** + * Viewport captures are returned in physical display pixels, so a HiDPI screen + * produces a bitmap several times wider than the preview needs. Bound the + * encoded image itself rather than leaving that to the gateway. + */ +async function downscale(jpegBase64: string): Promise { + const bitmap = await createImageBitmap( + new Blob([Uint8Array.from(atob(jpegBase64), (c) => c.charCodeAt(0))], { type: "image/jpeg" }), + ); + try { + if (bitmap.width <= PREVIEW_WIDTH) return jpegBase64; + const canvas = new OffscreenCanvas( + PREVIEW_WIDTH, + Math.max(1, Math.round((bitmap.height * PREVIEW_WIDTH) / bitmap.width)), + ); + const context = canvas.getContext("2d"); + if (!context) throw new Error("Preview canvas unavailable"); + context.drawImage(bitmap, 0, 0, canvas.width, canvas.height); + const jpeg = await canvas.convertToBlob({ type: "image/jpeg", quality: 0.5 }); + return btoa( + Array.from(new Uint8Array(await jpeg.arrayBuffer()), (b) => String.fromCharCode(b)).join(""), + ); + } finally { + bitmap.close(); + } +} + +/** + * The task's active tab, or another tab it owns when the user has selected an + * unauthorized page inside the Agent Window. Window membership alone never + * grants access, so an unowned tab is never captured or focused. + */ +export async function taskTarget(manager: SessionManager, sessionId: string): Promise { + const task = manager.get(sessionId); + if (!task?.remote) throw new Error("Task unavailable"); + const tabs = await chrome.tabs.query({ windowId: task.agentWindowId }); + if (manager.get(sessionId) !== task) throw new Error("Task unavailable"); + const owned = tabs.filter( + (tab) => typeof tab.id === "number" && isAgentControlledTab(task, tab.id), + ); + const tabId = (owned.find((tab) => tab.active) ?? owned[0])?.id; + if (tabId === undefined) throw new Error("Task tab unavailable"); + return tabId; +} diff --git a/apps/extension/src/lib/ui-channel.ts b/apps/extension/src/lib/ui-channel.ts new file mode 100644 index 00000000..18d58635 --- /dev/null +++ b/apps/extension/src/lib/ui-channel.ts @@ -0,0 +1,57 @@ +/** + * Optional UI request channel for authenticated remote gateways. + * + * A gateway that runs tasks for a user usually renders its own view of the + * running task: a small preview of the page the agent is working on, and a + * control that brings that tab to the front. Both have to answer while the + * session's tool queue is busy — during a long navigation, or while + * `request_help` waits for the user — which is exactly when the ordinary tool + * RPCs cannot run, because they are serialized per session. + * + * These requests are therefore answered here and never enter the tool + * dispatcher. Attach this to the socket before `WSTransport` so a `ui.*` frame + * is consumed by this listener instead of being reported as an unknown method. + * Only authenticated remote sockets carry the channel; a local daemon socket + * keeps the native protocol unchanged. + */ +export interface UiChannelHandlers { + /** Activates the task's own tab and raises its window. Never starts a task. */ + focus(sessionId: string): Promise; + /** Returns one downscaled JPEG frame of the task's own tab. */ + preview(sessionId: string): Promise; +} + +export function attachUiChannel(socket: WebSocket, handlers: UiChannelHandlers): void { + socket.addEventListener("message", (event) => { + let request: { id?: unknown; method?: unknown; params?: { session_id?: unknown } }; + try { + request = JSON.parse(event.data); + if (!request || typeof request !== "object") return; + } catch { + return; + } + const methods: Record Promise> = { + "ui.task_focus": (id) => handlers.focus(id), + "ui.task_preview": (id) => handlers.preview(id), + }; + if (typeof request.method !== "string" || !Object.hasOwn(methods, request.method)) return; + // The frame is ours: keep it away from the native transport listener. + event.stopImmediatePropagation(); + if (typeof request.id !== "string" || !request.id) return; + const id = request.id; + const sessionId = request.params?.session_id; + const send = (body: object) => { + if (socket.readyState === WebSocket.OPEN) socket.send(JSON.stringify({ id, ...body })); + }; + if (typeof sessionId !== "string" || !sessionId) { + send({ error: { code: "invalid_params", message: "session_id is required" } }); + return; + } + void Promise.resolve() + .then(() => methods[request.method as string]!(sessionId)) + .then( + (result) => send({ result }), + () => send({ error: { code: "not_found", message: "Browser task unavailable" } }), + ); + }); +} diff --git a/apps/extension/src/tools/session.ts b/apps/extension/src/tools/session.ts index 000366d4..c27ee3b5 100644 --- a/apps/extension/src/tools/session.ts +++ b/apps/extension/src/tools/session.ts @@ -1,3 +1,4 @@ +import { withTaskPreviewStop } from "@/lib/task-preview"; import { DEFAULT_INTERACTION_PREFERENCES, type InteractionPreferenceStore, @@ -196,6 +197,14 @@ export async function handleSessionStop( manager: SessionManager, params: SessionStopParams, deps: SessionStopDeps = {}, +): Promise { + return withTaskPreviewStop(manager, params?.session_id, () => stopSession(manager, params, deps)); +} + +async function stopSession( + manager: SessionManager, + params: SessionStopParams, + deps: SessionStopDeps, ): Promise { if (!params?.session_id) { return { diff --git a/docs/remote-extension-connection.md b/docs/remote-extension-connection.md index 4aa1a4d1..b3fa1d3e 100644 --- a/docs/remote-extension-connection.md +++ b/docs/remote-extension-connection.md @@ -155,6 +155,19 @@ The extension uses the same protocol whether it connects to `bsk` or a compatibl 5. Renewal uses the same POST endpoint and the current token, with `action: "renew"` and a new `next_token`. Keep the same `device_id`. Invalidate the old credential for new connections. An exact retry of the same old/new pair returns the original successful response; the old credential must not rotate to a different replacement. Revocation invalidates retries too. An already connected socket can remain open during renewal while its device grant is valid. 6. After upgrade, support the existing native handshake and RPC frames. Bind all RPC routing, responses, events and session state to the authenticated device. Do not trust its self-reported browser ID as authorization to another device's tasks. Close active sockets on expiration or revocation and cancel their pending work. +### Optional UI channel + +A gateway that renders its own view of a running task can send two extra request frames on the authenticated socket. The extension answers them outside the tool queue, so they still work while the session is navigating or waiting for `request_help`, and neither one ever starts a session: + +```json +{"id":"ui-1","method":"ui.task_preview","params":{"session_id":"server-owned-session"}} +``` + +- `ui.task_preview` returns `image_base64`, `format: "jpeg"`, `tab_id`, `title` and `captured_at`. The encoded frame is at most 640 pixels wide. Captures are coalesced per task, at most one runs per tab, and a poll that arrives while Chrome still holds one is refused rather than queued behind it. Authorization, the document revision and the debugger identity are re-checked before a frame is returned. These frames keep the extension's control and help overlays visible, so a periodic preview does not make them flicker in the user's browser; tool screenshots continue to suppress them for unobstructed page content. +- `ui.task_focus` activates the task's own tab and raises its window, returning `{ "focused": true }`. + +Both are restricted to a tab the task owns; window membership alone never qualifies, and an unowned tab inside the Agent Window is neither captured nor focused. Errors use the native envelope, `{ "id": "…", "error": { "code": "…", "message": "…" } }`. The channel exists only on remote sockets: an extension connected to a local daemon does not serve these methods. `session.stop` drains an in-flight preview before it detaches the debugger and closes the task's tabs. + A gateway can bridge this protocol to a local `bsk` daemon on its server, keeping that daemon's loopback/IPC boundary private. A gateway that terminates device authentication owns that authentication lifecycle and routing isolation. Merely forwarding its credentials to the built-in server will not authorize them: the built-in server accepts its own issued grants. ## Browser permissions and task lifetime @@ -167,7 +180,7 @@ This also applies to tabs or windows opened by a page through `target="_blank"`, Disconnecting cancels task work, returns borrowed tabs and closes task-created tabs. User-created tabs survive cleanup. Failed returns preserve the window and must be resolved before reconnecting. Reconnection starts new tasks; commands and sessions are never replayed. Failed remote authentication does not select a local connection automatically. -Remote upload and download are unsupported in this version and return the `unsupported` error. Existing local file transfer behavior is unchanged. Screenshots and other existing RPC content results remain supported. There is no gateway preview or focus side protocol, background task tab group, or alternative window model. +Remote upload and download are unsupported in this version and return the `unsupported` error. Existing local file transfer behavior is unchanged. Screenshots and other existing RPC content results remain supported. There is no background task tab group or alternative window model. Device credentials live in extension-origin IndexedDB; ordinary extension settings contain only the selected connection mode and a non-secret revision. Fresh local profiles and explicitly selected local mode do not read that credential database. If remote storage fails, the popup reports the error and the extension does not fall back automatically. Explicitly selecting the local connection can recover startup even when the credential database is unavailable. Legacy remote settings still migrate before ordinary settings access is restored. The standalone server persists hashed credentials with private file permissions and atomic writes. Treat the whole browser profile and `BSK_HOME` as trusted local data. Protect TLS private keys separately.