diff --git a/flow-agent/flow_engine/bridge.py b/flow-agent/flow_engine/bridge.py index ecda475..e288c64 100644 --- a/flow-agent/flow_engine/bridge.py +++ b/flow-agent/flow_engine/bridge.py @@ -400,10 +400,36 @@ async def _force_refresh_client(self, client_id): except Exception as e: log.debug("Force-refresh failed for client %s: %s", client_id, e) - async def _request_flow_tab(self): - """Fallback that triggers open_flow_tab on all connected clients.""" + async def reload_extensions(self): + """Ask all connected extensions to reload themselves.""" for cid in list(self._clients.keys()): - await self._request_flow_tab_for(cid) + try: + await self.send_message_to(cid, {"method": "reload_extension"}) + except Exception as e: + log.debug("Failed to send reload_extension to client %s: %s", cid, e) + + async def reload_flow_tabs(self): + """Ask connected extensions to reload their Flow tabs.""" + for cid in list(self._clients.keys()): + try: + await self.send_message_to(cid, {"method": "reload_tabs"}) + except Exception as e: + log.debug("Failed to send reload_tabs to client %s: %s", cid, e) + + async def run_probe(self, probe_type="default", timeout=60): + client_id = self._select_client() + if not client_id: + return {"error": "NO_CLIENT"} + req_id = str(uuid.uuid4()) + future = self._loop.create_future() + self._pending[req_id] = future + await self.send_message_to(client_id, {"id": req_id, "method": "run_probe", "params": {"probeType": probe_type}}) + try: + return await asyncio.wait_for(future, timeout=timeout) + except Exception as e: + return {"error": str(e)} + finally: + self._pending.pop(req_id, None) async def health_check(self): """Quick check if at least one extension is ready with valid token.""" diff --git a/flow-agent/flow_engine/config.py b/flow-agent/flow_engine/config.py index 39cc61a..9cdafa3 100644 --- a/flow-agent/flow_engine/config.py +++ b/flow-agent/flow_engine/config.py @@ -101,7 +101,7 @@ def _flow_binary_dir() -> str: CLIENT_CTX = { "tool": "PINHOLE", "tier": "PAYGATE_TIER_ONE", - "origin": "https://labs.google", + "origin": "https://flow.google.com", "recaptcha_app_type": "RECAPTCHA_APPLICATION_TYPE_WEB", } diff --git a/flow-agent/flow_engine/generators/i2v.py b/flow-agent/flow_engine/generators/i2v.py index 2c09d97..63aa7c3 100644 --- a/flow-agent/flow_engine/generators/i2v.py +++ b/flow-agent/flow_engine/generators/i2v.py @@ -11,6 +11,14 @@ log = logging.getLogger("flow_engine.generators.i2v") +def _safe_error_text(value) -> str | None: + """Return a bounded scalar error without serializing arbitrary responses.""" + if not isinstance(value, (str, int, float)): + return None + text = " ".join(str(value).split()) + return text[:500] or None + + async def upload_image( bridge, image_path: str, @@ -52,11 +60,20 @@ async def upload_image( log.info("Uploading image: %s", os.path.basename(image_path)) result = await bridge.api_request(ENDPOINTS["upload_image"], body) - status = result.get("status", 0) - data = result.get("data", {}) + status = result.get("status", 0) if isinstance(result, dict) else 0 + data = result.get("data", {}) if isinstance(result, dict) else {} if status != 200: - err = data.get("error", {}).get("message", "Unknown") if isinstance(data, dict) else str(data) - err_msg = f"Image upload failed: {err}" + nested = data.get("error", {}) if isinstance(data, dict) else {} + nested = nested if isinstance(nested, dict) else {} + message = _safe_error_text(nested.get("message")) + google_status = _safe_error_text(nested.get("status")) + top_level_error = _safe_error_text(result.get("error")) if isinstance(result, dict) else None + detail = message or top_level_error or google_status or "Unknown error" + if message and google_status and google_status not in message: + detail = f"{message} ({google_status})" + status_text = _safe_error_text(status) + status_suffix = f" (status {status_text})" if status_text and status_text != "0" else "" + err_msg = f"Image upload failed{status_suffix}: {detail}" log.error("%s", err_msg) raise ValueError(err_msg) diff --git a/flow-agent/flow_server/routes/system.py b/flow-agent/flow_server/routes/system.py index 14fd97e..ad32af2 100644 --- a/flow-agent/flow_server/routes/system.py +++ b/flow-agent/flow_server/routes/system.py @@ -180,6 +180,24 @@ async def root(): return {"status": "running", "service": "Flow Agent API"} +@router.post("/api/dev/reload") +async def dev_reload(): + bridge = state.get_bridge() + if bridge: + await bridge.reload_extensions() + return {"ok": True} + return {"ok": False} + + +@router.post("/api/dev/probe") +async def dev_probe(probe_type: str = "default"): + bridge = state.get_bridge() + if bridge: + res = await bridge.run_probe(probe_type) + return res + return {"error": "Bridge not running"} + + # Health Check @router.get("/health") async def health(): diff --git a/flow-agent/flow_server/state.py b/flow-agent/flow_server/state.py index b217d3c..da29f2b 100644 --- a/flow-agent/flow_server/state.py +++ b/flow-agent/flow_server/state.py @@ -46,7 +46,7 @@ async def recover_orphan_response(data: dict, meta: dict): """ try: if data.get("status") != 200: - log.info("Orphan response %s ignored (status=%s)", data.get("id"), data.get("status")) + log.info("Orphan response %s ignored (status=%s, data=%s)", data.get("id"), data.get("status"), data.get("data")) return # Only images arrive inline; videos are polled separately, so only # image generations are recoverable this way. diff --git a/flow-agent/tests/test_extension_flow_urls.py b/flow-agent/tests/test_extension_flow_urls.py new file mode 100644 index 0000000..9c2d6e9 --- /dev/null +++ b/flow-agent/tests/test_extension_flow_urls.py @@ -0,0 +1,68 @@ +import json +from pathlib import Path + + +EXTENSION_DIR = Path(__file__).resolve().parents[2] / "flow-extension" + + +def test_manifest_allows_new_and_legacy_flow_pages(): + manifest = json.loads((EXTENSION_DIR / "manifest.json").read_text(encoding="utf-8")) + + assert "https://flow.google.com/*" in manifest["host_permissions"] + assert "https://flow.google.com/*" in manifest["content_scripts"][0]["matches"] + assert "https://flow.google.com/*" in manifest["web_accessible_resources"][0]["matches"] + assert "https://labs.google/fx/tools/flow*" in manifest["content_scripts"][0]["matches"] + + +def test_background_uses_precise_flow_page_eligibility_and_shared_tab_patterns(): + source = (EXTENSION_DIR / "background.js").read_text(encoding="utf-8") + + assert "'https://flow.google.com/*'" in source + assert "parsed.hostname === 'flow.google.com'" in source + assert "parsed.hostname !== 'labs.google'" in source + assert "return createdTab;" in source + assert "url: '*://labs.google/*'" not in source + assert source.count("url: FLOW_TAB_URLS") >= 3 + + +def test_background_opens_captcha_capable_project_pages(): + source = (EXTENSION_DIR / "background.js").read_text(encoding="utf-8") + + # labs.google/fx/tools/flow redirects to the flow.google.com home page, which + # never loads reCAPTCHA; only /project/ pages do. + assert "const FLOW_URL = 'https://flow.google.com/';" in source + assert "function isFlowProjectUrl(url)" in source + assert "/^\\/project\\/[^/]+/" in source + assert "https://flow.google.com/project/${encodeURIComponent(projectId)}" in source + assert "tabs.filter((t) => isFlowProjectUrl(t.url))" in source + assert "solveCaptcha(id, captchaAction, projectId)" in source + + +def test_background_refreshes_token_through_labs_handoff(): + source = (EXTENSION_DIR / "background.js").read_text(encoding="utf-8") + + # The ya29 bearer is only observable during the labs.google -> flow.google.com + # redirect; reloading a flow.google.com tab does not re-capture it. + assert "const TOKEN_URL = 'https://labs.google/fx/tools/flow';" in source + assert "async function refreshTokenViaLabs()" in source + assert source.count("await refreshTokenViaLabs()") >= 2 + assert "chrome.tabs.reload(tabs[0].id)" not in source + + +def test_extension_verifies_captcha_bridge_before_using_a_tab(): + background = (EXTENSION_DIR / "background.js").read_text(encoding="utf-8") + content = (EXTENSION_DIR / "content.js").read_text(encoding="utf-8") + injected = (EXTENSION_DIR / "injected.js").read_text(encoding="utf-8") + + # A tab matching a Flow URL is not enough: its bridge must answer a ping, + # otherwise every request burns the content-script timeout. + assert "async function bridgeAlive(tabId)" in background + assert "if (!(await bridgeAlive(tab.id))) continue;" in background + assert "type: 'PING_BRIDGE'" in background + assert "msg.type !== 'PING_BRIDGE'" in content + assert "'FLOW_AGENT_PING'" in injected and "'FLOW_AGENT_PONG'" in injected + # GET_CAPTCHA is re-dispatched until injected.js answers; injected.js dedups. + assert "setInterval(dispatch, 500)" in content + assert "_captchaInFlight" in injected + assert "grecaptcha execute timeout" in injected + diff --git a/flow-agent/tests/test_media_storage.py b/flow-agent/tests/test_media_storage.py index 97dc02c..812c2f5 100644 --- a/flow-agent/tests/test_media_storage.py +++ b/flow-agent/tests/test_media_storage.py @@ -2,6 +2,7 @@ import importlib import json import os +import re import subprocess import sys from pathlib import Path @@ -216,6 +217,61 @@ async def api_request(self, endpoint, body, **kwargs): assert media_store.get_for_file(image_path, project_id="project-a")["media_id"] == "fresh-id" +@pytest.mark.parametrize( + ("response", "expected"), + [ + ( + {"status": 403, "error": "CAPTCHA_FAILED"}, + "Image upload failed (status 403): CAPTCHA_FAILED", + ), + ({"error": "Request timed out"}, "Image upload failed: Request timed out"), + ( + {"status": 500, "error": {"debug": {"internal": "sensitive-context"}}}, + "Image upload failed (status 500): Unknown error", + ), + ( + { + "status": 400, + "data": { + "error": { + "message": "The image could not be processed", + "status": "INVALID_ARGUMENT", + } + }, + }, + "Image upload failed (status 400): The image could not be processed (INVALID_ARGUMENT)", + ), + ], +) +def test_upload_image_preserves_safe_actionable_errors( + monkeypatch, tmp_path, response, expected +): + _configure_store(monkeypatch, tmp_path) + image_path = tmp_path / "reference.png" + image_path.write_bytes(PNG_BYTES) + + class Bridge: + async def api_request(self, endpoint, body, **kwargs): + return response + + with pytest.raises(ValueError, match=re.escape(expected)): + asyncio.run(upload_image(Bridge(), str(image_path), "project-a")) + + +def test_upload_image_still_accepts_successful_media_response(monkeypatch, tmp_path): + _configure_store(monkeypatch, tmp_path) + image_path = tmp_path / "reference.png" + image_path.write_bytes(PNG_BYTES) + + class Bridge: + async def api_request(self, endpoint, body, **kwargs): + return {"status": 200, "data": {"media": {"name": "uploaded-media-id"}}} + + media_id = asyncio.run(upload_image(Bridge(), str(image_path), "project-a")) + + assert media_id == "uploaded-media-id" + + def test_upload_is_reused_as_image_to_video_reference_without_duplicates(monkeypatch, tmp_path): history_path = _configure_store(monkeypatch, tmp_path) image_path = tmp_path / "start-frame.png" diff --git a/flow-extension/README.md b/flow-extension/README.md index 446470f..b9e1c84 100644 --- a/flow-extension/README.md +++ b/flow-extension/README.md @@ -19,4 +19,12 @@ Chrome bridge for [kodelyx/flow-agent](https://github.com/kodelyx/flow-agent). I 4. Open , sign in, and keep the tab open. 5. Click the extension icon to open Flow Agent in Chrome's side panel. +## Flow site compatibility + +The extension recognizes both the current `https://flow.google.com/` site and +the legacy `https://labs.google/fx/tools/flow` route. This is partial issue #10 +compatibility: authentication and CAPTCHA behavior, plus the existing REST and +upload calls on the new site, have not been verified end to end and may still +require follow-up changes. + Main documentation: [Flow Agent README](../README.md) diff --git a/flow-extension/background.js b/flow-extension/background.js index 3cfc6df..9ff1f46 100644 --- a/flow-extension/background.js +++ b/flow-extension/background.js @@ -7,6 +7,29 @@ importScripts('config.js'); +// Keep the last '[Flow Agent]' console lines in chrome.storage.local (debugLog) +// so a stall can be diagnosed without the service-worker console, which is +// gone by the time anyone looks. +const DEBUG_LOG_MAX = 200; +let _debugLog = []; +let _debugLogFlush = null; +for (const level of ['log', 'warn', 'error']) { + const original = console[level].bind(console); + console[level] = (...args) => { + original(...args); + if (typeof args[0] !== 'string' || !args[0].startsWith('[Flow Agent]')) return; + const line = args.map((a) => (typeof a === 'string' ? a : (a?.message ?? JSON.stringify(a)))).join(' '); + _debugLog.push(`${new Date().toISOString()} ${level.toUpperCase()} ${line}`); + if (_debugLog.length > DEBUG_LOG_MAX) _debugLog = _debugLog.slice(-DEBUG_LOG_MAX); + if (!_debugLogFlush) { + _debugLogFlush = setTimeout(() => { + _debugLogFlush = null; + chrome.storage.local.set({ debugLog: _debugLog }).catch(() => {}); + }, 250); + } + }; +} + let callbackUrl = 'http://127.0.0.1:3001/api/ext/callback'; // NOTE: This is a browser-restricted public API key — safe to ship in extension bundles. const API_KEY = 'AIzaSyBtrm0o5ab1c-Ec8ZuLcGt3oJAA5VWt3pY'; @@ -126,10 +149,75 @@ async function init() { // Retry any responses left undelivered by a previous worker lifetime. chrome.alarms.create('flushOutbox', { periodInMinutes: 0.5 }); flushOutbox(); + ensureAuthCaptured().catch(() => {}); } ensureInitialized(); +// ─── Cookie / SAPISIDHASH Auth Helpers ────────────────────── + +async function getSapisidCookie() { + const names = ['SAPISID', '__Secure-1PAPISID', '__Secure-3PAPISID', 'APISID']; + for (const name of names) { + try { + const c = await chrome.cookies.get({ url: 'https://flow.google.com', name }); + if (c?.value) return c.value; + } catch {} + } + for (const name of names) { + try { + const c = await chrome.cookies.get({ url: 'https://google.com', name }); + if (c?.value) return c.value; + } catch {} + } + return null; +} + +async function computeSapisidHash(sapisid, origin = 'https://flow.google.com') { + const time = Math.floor(Date.now() / 1000); + const str = `${time} ${sapisid} ${origin}`; + const encoder = new TextEncoder(); + const data = encoder.encode(str); + const hashBuffer = await crypto.subtle.digest('SHA-1', data); + const hashArray = Array.from(new Uint8Array(hashBuffer)); + const hashHex = hashArray.map((b) => b.toString(16).padStart(2, '0')).join(''); + return `${time}_${hashHex}`; +} + +async function getAuthHeader() { + if (flowKey && flowKey.startsWith('ya29.')) { + return `Bearer ${flowKey}`; + } + const sapisid = await getSapisidCookie(); + if (sapisid) { + const hash = await computeSapisidHash(sapisid, 'https://flow.google.com'); + return `SAPISIDHASH ${hash}`; + } + return null; +} + +async function ensureAuthCaptured() { + if (flowKey && flowKey.startsWith('ya29.')) return true; + const sapisid = await getSapisidCookie(); + if (sapisid) { + flowKey = `sapisid_${sapisid.slice(0, 8)}`; + metrics.tokenCapturedAt = Date.now(); + await chrome.storage.local.set({ flowKey, metrics }); + console.log('[Flow Agent] Active Google cookie auth (SAPISID) registered with agent'); + sendToAgent({ type: 'token_captured', flowKey, clientId: extensionClientId }); + return true; + } + return false; +} + +if (chrome.cookies?.onChanged) { + chrome.cookies.onChanged.addListener((changeInfo) => { + if (['SAPISID', '__Secure-1PAPISID', '__Secure-3PAPISID'].includes(changeInfo.cookie?.name)) { + ensureAuthCaptured().catch(() => {}); + } + }); +} + // ─── Token Capture ────────────────────────────────────────── chrome.webRequest.onBeforeSendHeaders.addListener( @@ -153,7 +241,7 @@ chrome.webRequest.onBeforeSendHeaders.addListener( // Notify whichever transport is active. sendToAgent({ type: 'token_captured', flowKey, clientId: extensionClientId }); }, - { urls: ['https://aisandbox-pa.googleapis.com/*', 'https://labs.google/*'] }, + { urls: ['https://aisandbox-pa.googleapis.com/*', 'https://labs.google/*', 'https://flow.google.com/*'] }, ['requestHeaders', 'extraHeaders'], ); @@ -162,11 +250,78 @@ let _openingFlowTab = false; // ─── On-demand tab lifecycle ──────────────────────────────── // Open the Flow tab only when real work needs it (token capture or captcha). // Keep it available in the background so user tabs are never redirected. -const FLOW_TAB_URLS = ['https://labs.google/fx/tools/flow*', 'https://labs.google/fx/*/tools/flow*']; -const FLOW_URL = 'https://labs.google/fx/tools/flow'; +const FLOW_TAB_URLS = [ + 'https://flow.google.com/*', + 'https://labs.google/fx/tools/flow*', + 'https://labs.google/fx/*/tools/flow*', +]; +// labs.google/fx/tools/flow now 301s to the flow.google.com home page, which never +// loads reCAPTCHA Enterprise — only /project/ pages do. Land there directly. +const FLOW_URL = 'https://flow.google.com/'; let workTabId = null; let flowTabOpening = null; let workTabCreatedByExtension = false; +let lastFlowProjectUrl = null; + +chrome.storage.local.get(['lastFlowProjectUrl']).then((data) => { + if (!lastFlowProjectUrl && isFlowProjectUrl(data.lastFlowProjectUrl)) { + lastFlowProjectUrl = data.lastFlowProjectUrl; + } +}).catch(() => {}); + +// Remember the most recent project page any tab visits so an on-demand tab can +// open somewhere captcha-capable even when the request carries no projectId. +chrome.tabs.onUpdated.addListener((_, changeInfo) => { + if (changeInfo.url && isFlowProjectUrl(changeInfo.url)) { + lastFlowProjectUrl = changeInfo.url; + chrome.storage.local.set({ lastFlowProjectUrl }).catch(() => {}); + } +}); + +function isFlowProjectUrl(url) { + if (!url) return false; + try { + const parsed = new URL(url); + return parsed.protocol === 'https:' && parsed.hostname === 'flow.google.com' + && /^\/project\/[^/]+/.test(parsed.pathname); + } catch { + return false; + } +} + +function flowTabTargetUrl(projectId) { + if (projectId) return `https://flow.google.com/project/${encodeURIComponent(projectId)}`; + return lastFlowProjectUrl || FLOW_URL; +} + +// Google only sends the ya29 bearer while labs.google/fx/tools/flow hands off to +// flow.google.com; reloading a flow.google.com page never surfaces it. +const TOKEN_URL = 'https://labs.google/fx/tools/flow'; + +// Drive a tab through the labs.google handoff so the webRequest listener can +// capture a fresh bearer. Never navigates a tab the user opened. +async function refreshTokenViaLabs() { + let tabId = null; + if (workTabId !== null && workTabCreatedByExtension) { + try { + await chrome.tabs.get(workTabId); + tabId = workTabId; + } catch { + workTabId = null; + } + } + if (tabId === null) { + const tab = await chrome.tabs.create({ url: TOKEN_URL, active: false }); + workTabId = tab.id; + workTabCreatedByExtension = true; + tabId = tab.id; + } else { + await chrome.tabs.update(tabId, { url: TOKEN_URL }); + } + await waitForTabComplete(tabId); + scheduleFlowTabClose(); + return tabId; +} function scheduleFlowTabClose() { if (workTabCreatedByExtension) { @@ -189,7 +344,16 @@ async function closeIdleFlowTab() { } function isFlowUrl(url) { - return !!url && FLOW_TAB_URLS.some((p) => new RegExp(p.replace(/\./g, '\\.').replace(/\*/g, '.*')).test(url)); + if (!url) return false; + try { + const parsed = new URL(url); + if (parsed.protocol !== 'https:') return false; + if (parsed.hostname === 'flow.google.com') return true; + if (parsed.hostname !== 'labs.google') return false; + return /^\/fx\/(?:[^/]+\/)?tools\/flow(?:\/|$)/.test(parsed.pathname); + } catch { + return false; + } } async function waitForTabComplete(tabId, maxWaitMs = 10000) { @@ -209,54 +373,122 @@ async function waitForTabComplete(tabId, maxWaitMs = 10000) { }); } +// Every await on the tab-lookup path is bounded: one Chrome API call that never +// settles would otherwise park getOrOpenFlowTab's shared promise forever and +// silently stall every later request behind it. +function withTimeout(promise, ms, label) { + let timer; + const deadline = new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(`${label}_TIMEOUT`)), ms); + }); + return Promise.race([promise, deadline]).finally(() => clearTimeout(timer)); +} + +// True only if content.js AND injected.js answer in this tab. A tab can match a +// Flow URL yet have a dead bridge (opened before an extension reload, discarded, +// or on a page that never loaded injected.js) — sending it GET_CAPTCHA then just +// burns 25s and reports CONTENT_TIMEOUT. +async function bridgeAlive(tabId) { + const ping = () => withTimeout(chrome.tabs.sendMessage(tabId, { type: 'PING_BRIDGE' }), 5000, 'PING'); + try { + const resp = await ping(); + if (resp?.ok) return true; + } catch { /* no content script yet — inject and retry below */ } + try { + const tab = await chrome.tabs.get(tabId); + if (!isFlowUrl(tab?.url)) return false; + await withTimeout( + chrome.scripting.executeScript({ target: { tabId }, files: ['content.js'] }), + 10000, 'INJECT', + ); + await sleep(300); + const resp = await ping(); + return !!resp?.ok; + } catch (e) { + console.warn('[Flow Agent] Bridge ping failed for tab', tabId, e.message); + return false; + } +} + // Finds/wakes/creates the Flow tab. Returns // the tab, or null if it couldn't be opened. -async function _getOrOpenFlowTab() { +async function _getOrOpenFlowTab(projectId) { + const targetUrl = flowTabTargetUrl(projectId); + if (workTabId !== null) { try { let tab = await chrome.tabs.get(workTabId); - if (tab && !isFlowUrl(tab.url)) { - await chrome.tabs.update(workTabId, { url: FLOW_URL }); - await waitForTabComplete(workTabId); - tab = await chrome.tabs.get(workTabId); + // Never navigate or cache a user's non-project tab, even if its bridge + // answers. Home pages do not load reCAPTCHA. + if (!workTabCreatedByExtension && !isFlowProjectUrl(tab?.url)) { + console.warn('[Flow Agent] User work tab is not on a /project/ page; forgetting it'); + workTabId = null; + } else { + const needsProjectPage = workTabCreatedByExtension && !isFlowProjectUrl(tab?.url); + if (tab && needsProjectPage) { + await withTimeout(chrome.tabs.update(workTabId, { url: targetUrl }), 10000, 'TAB_UPDATE'); + await waitForTabComplete(workTabId); + tab = await chrome.tabs.get(workTabId); + } + if (await bridgeAlive(workTabId)) { + scheduleFlowTabClose(); + return tab; + } + console.warn('[Flow Agent] Flow tab', workTabId, 'has a dead captcha bridge; looking for another'); + workTabId = null; } - scheduleFlowTabClose(); - return tab; } catch (e) { workTabId = null; // closed by the user — fall through and open fresh } } const tabs = await chrome.tabs.query({ url: FLOW_TAB_URLS }); - if (tabs.length) { - workTabId = tabs[0].id; + // Project pages first — they are the only ones that load reCAPTCHA. + const candidates = [...tabs.filter((t) => isFlowProjectUrl(t.url)), ...tabs.filter((t) => !isFlowProjectUrl(t.url))]; + for (const tab of candidates) { + if (!isFlowProjectUrl(tab.url)) { + console.warn('[Flow Agent] Flow tab is not on a /project/ page; reCAPTCHA is only available there'); + if (isFlowProjectUrl(targetUrl)) continue; + } + if (!(await bridgeAlive(tab.id))) continue; + workTabId = tab.id; workTabCreatedByExtension = false; - return tabs[0]; + return tab; + } + if (tabs.length) { + console.warn('[Flow Agent] None of', tabs.length, 'eligible Flow tab(s) answered the bridge ping; opening a fresh one'); } - const createdTab = await chrome.tabs.create({ url: FLOW_URL, active: false }); + const createdTab = await withTimeout(chrome.tabs.create({ url: targetUrl, active: false }), 10000, 'TAB_CREATE'); workTabId = createdTab.id; workTabCreatedByExtension = true; + console.log('[Flow Agent] Opened Flow work tab', workTabId, 'at', targetUrl); await waitForTabComplete(workTabId); await sleep(1500); // Inject content script to make sure reCAPTCHA bridge is ready try { - await chrome.scripting.executeScript({ + const readyTab = await chrome.tabs.get(workTabId); + if (!isFlowUrl(readyTab?.url)) throw new Error('INVALID_FLOW_TAB'); + await withTimeout(chrome.scripting.executeScript({ target: { tabId: workTabId }, files: ['content.js'], - }); + }), 10000, 'INJECT'); } catch (e) { console.warn('[Flow Agent] Content script pre-injection:', e.message); } scheduleFlowTabClose(); - return retryTabs[0]; + return createdTab; } -async function getOrOpenFlowTab() { +async function getOrOpenFlowTab(projectId) { if (flowTabOpening) return flowTabOpening; - flowTabOpening = _getOrOpenFlowTab(); + flowTabOpening = withTimeout(_getOrOpenFlowTab(projectId), 60000, 'FLOW_TAB') + .catch((e) => { + console.error('[Flow Agent] getOrOpenFlowTab failed:', e.message); + return null; + }); try { return await flowTabOpening; } finally { @@ -264,10 +496,24 @@ async function getOrOpenFlowTab() { } } +async function getAnyFlowTab() { + try { + const tabs = await chrome.tabs.query({ url: FLOW_TAB_URLS }); + if (!tabs || !tabs.length) return null; + const projectTab = tabs.find((t) => isFlowProjectUrl(t.url)); + return projectTab || tabs[0]; + } catch { + return null; + } +} + // Token is considered fresh if it exists and was captured less than 50 minutes ago. // Google OAuth tokens expire after ~60 min, so 50 min gives a safe buffer. +// Cookie (SAPISID) auth is long-lived and fresh as long as the user is signed in. function isTokenFresh() { - if (!flowKey || !metrics.tokenCapturedAt) return false; + if (!flowKey) return false; + if (flowKey.startsWith('sapisid_')) return true; + if (!metrics.tokenCapturedAt) return false; const ageMs = Date.now() - metrics.tokenCapturedAt; return ageMs < 50 * 60 * 1000; // 50 minutes } @@ -279,22 +525,19 @@ async function captureTokenFromFlowTab() { return; } + if (await ensureAuthCaptured()) { + console.log('[Flow Agent] Cookie auth captured, skipping tab refresh'); + return; + } + if (_openingFlowTab) { console.log('[Flow Agent] Flow tab already opening, skipping'); return; } _openingFlowTab = true; try { - const tab = await getOrOpenFlowTab(); - if (!tab) { - console.log('[Flow Agent] Flow tab not ready yet after open'); - return; - } - await chrome.scripting.executeScript({ - target: { tabId: tab.id }, - files: ['content.js'], - }); - console.log('[Flow Agent] Token refresh triggered on Flow tab'); + const tabId = await refreshTokenViaLabs(); + console.log('[Flow Agent] Token refresh triggered via labs.google handoff in tab', tabId); } catch (e) { console.error('[Flow Agent] Token refresh failed:', e); } finally { @@ -343,6 +586,7 @@ async function connectToAgent() { await chrome.storage.local.set({ clientId }); } extensionClientId = clientId; + await ensureAuthCaptured(); // Send current state + resend token if we have one, along with clientId ws.send(JSON.stringify({ @@ -387,6 +631,327 @@ async function connectToAgent() { metrics, }, }); + } else if (msg.method === 'reload_extension') { + console.log('[Flow Agent] Reloading extension via command...'); + chrome.runtime.reload(); + } else if (msg.method === 'reload_tabs') { + const tabs = await chrome.tabs.query({ url: FLOW_TAB_URLS }); + for (const t of tabs) { + try { await chrome.tabs.reload(t.id); } catch {} + } + sendToAgent({ id: msg.id, result: { reloadedTabs: tabs.length } }); + } else if (msg.method === 'run_probe') { + try { + if (msg.params?.probeType === 'test_labs_nav') { + const captured = []; + const listener = (details) => { + const auth = details.requestHeaders?.find(h => h.name.toLowerCase() === 'authorization'); + captured.push({ + url: details.url, + method: details.method, + authHeader: auth ? auth.value.slice(0, 30) : null + }); + }; + chrome.webRequest.onBeforeSendHeaders.addListener( + listener, + { urls: [''] }, + ['requestHeaders', 'extraHeaders'] + ); + const tab = await chrome.tabs.create({ url: 'https://labs.google/fx/tools/flow', active: false }); + await sleep(7000); + chrome.webRequest.onBeforeSendHeaders.removeListener(listener); + let finalTabUrl = null; + try { + const t = await chrome.tabs.get(tab.id); + finalTabUrl = t?.url; + await chrome.tabs.remove(tab.id); + } catch {} + sendToAgent({ + id: msg.id, + result: { + finalTabUrl, + capturedUrlsCount: captured.length, + authCaptured: captured.filter(c => c.authHeader), + relevantRequests: captured.filter(c => c.url.includes('google') || c.url.includes('token') || c.url.includes('api')).slice(0, 30) + } + }); + return; + } + if (msg.params?.probeType === 'list_tabs') { + const allTabs = await chrome.tabs.query({}); + sendToAgent({ + id: msg.id, + result: { + tabs: allTabs.map(t => ({ id: t.id, url: t.url, title: t.title, active: t.active })) + } + }); + return; + } + if (msg.params?.probeType === 'inspect_toolbar') { + const tab = (await getAnyFlowTab()) || (await getOrOpenFlowTab()); + if (!tab) { + sendToAgent({ id: msg.id, error: 'NO_FLOW_TAB' }); + return; + } + const results = await chrome.scripting.executeScript({ + target: { tabId: tab.id }, + world: 'MAIN', + func: () => { + const buttons = Array.from(document.querySelectorAll('button')).map(b => ({ + text: b.innerText?.trim().replace(/\n/g, ' '), + aria: b.getAttribute('aria-label'), + classes: b.className + })); + return { buttons }; + } + }); + sendToAgent({ id: msg.id, result: results?.[0]?.result }); + return; + } + if (msg.params?.probeType === 'inspect_settings') { + const tab = (await getAnyFlowTab()) || (await getOrOpenFlowTab()); + if (!tab) { + sendToAgent({ id: msg.id, error: 'NO_FLOW_TAB' }); + return; + } + const results = await chrome.scripting.executeScript({ + target: { tabId: tab.id }, + world: 'MAIN', + func: async () => { + const btn = Array.from(document.querySelectorAll('button')).find(b => b.innerText?.includes('crop_') || b.innerText?.includes('Banana') || b.innerText?.includes('Video') || b.innerText?.includes('Image')); + if (!btn) return { error: 'NO_BUTTON' }; + btn.click(); + await new Promise(r => setTimeout(r, 600)); + const items = Array.from(document.querySelectorAll('[role="menuitem"], [role="option"], mat-option, .cdk-overlay-container *')) + .map(el => el.innerText?.trim()) + .filter(Boolean) + .filter((v, i, a) => a.indexOf(v) === i); + // click again or press Escape to close menu + document.body.dispatchEvent(new KeyboardEvent('keydown', { key: 'Escape', code: 'Escape', bubbles: true })); + return { btnText: btn.innerText?.trim().replace(/\n/g, ' '), items }; + } + }); + sendToAgent({ id: msg.id, result: results?.[0]?.result }); + return; + } + if (msg.params?.probeType === 'inspect_recent') { + const tab = (await getAnyFlowTab()) || (await getOrOpenFlowTab()); + if (!tab) { + sendToAgent({ id: msg.id, error: 'NO_FLOW_TAB' }); + return; + } + const results = await chrome.scripting.executeScript({ + target: { tabId: tab.id }, + world: 'MAIN', + func: () => { + const resources = performance.getEntriesByType('resource') + .map(r => r.name) + .filter(n => n.includes('batchexecute') || n.includes('asb') || n.includes('flow') || n.includes('sandbox')); + + const allImgs = Array.from(document.querySelectorAll('img')).map(i => ({ src: i.src.slice(0, 100), width: i.width, height: i.height })); + const bgImgs = Array.from(document.querySelectorAll('*')) + .map(el => window.getComputedStyle(el).backgroundImage) + .filter(bg => bg && bg !== 'none' && !bg.includes('gradient')) + .slice(0, 10); + + const editor = document.querySelector('.ProseMirror, [contenteditable="true"]'); + const genBtn = Array.from(document.querySelectorAll('button')).find(b => b.innerText?.includes('arrow_forward') || b.getAttribute('aria-label')?.includes('Generate')); + + return { + url: window.location.href, + editorText: editor?.innerText?.trim(), + genBtnDisabled: genBtn?.disabled, + recentResources: resources.slice(-20), + allImgs, + bgImgs + }; + } + }); + sendToAgent({ id: msg.id, result: results?.[0]?.result }); + return; + } + if (msg.params?.probeType === 'generate_image') { + const tab = (await getAnyFlowTab()) || (await getOrOpenFlowTab()); + if (!tab) { + sendToAgent({ id: msg.id, error: 'NO_FLOW_TAB' }); + return; + } + const results = await chrome.scripting.executeScript({ + target: { tabId: tab.id }, + world: 'MAIN', + func: async (promptText) => { + try { + const editor = document.querySelector('.ProseMirror, [contenteditable="true"]'); + if (!editor) return { error: 'NO_EDITOR' }; + + // Switch to Image mode + const modeButton = Array.from(document.querySelectorAll('button')).find(b => b.innerText?.includes('Video') || b.innerText?.includes('Image')); + if (modeButton) { + modeButton.click(); + await new Promise(r => setTimeout(r, 400)); + const allClickables = Array.from(document.querySelectorAll('button, [role="menuitem"], [role="option"], div, span')); + const imgOption = allClickables.find(el => { + const t = el.innerText?.trim(); + return t === 'Image' || t === 'image\nImage'; + }); + if (imgOption) { + imgOption.click(); + await new Promise(r => setTimeout(r, 400)); + } + } + + // Focus and type prompt into editor + editor.focus(); + document.execCommand('selectAll', false, null); + document.execCommand('insertText', false, promptText); + editor.dispatchEvent(new Event('input', { bubbles: true })); + editor.dispatchEvent(new Event('change', { bubbles: true })); + await new Promise(r => setTimeout(r, 400)); + + const generateButton = Array.from(document.querySelectorAll('button')).find(b => b.innerText?.includes('arrow_forward') || b.getAttribute('aria-label')?.includes('Generate')); + if (!generateButton || generateButton.disabled) { + return { error: 'BUTTON_DISABLED_OR_MISSING' }; + } + + // Count existing images + const beforeImgs = Array.from(document.querySelectorAll('img')).map(i => i.src); + + // Click generate! + generateButton.click(); + const startTime = Date.now(); + + // Poll for new image + let newImageSrc = null; + for (let i = 0; i < 45; i++) { + await new Promise(r => setTimeout(r, 1000)); + const imgs = Array.from(document.querySelectorAll('img')).map(img => img.src); + const asbImg = imgs.find(src => (src.includes('/asb/') || src.includes('flow.google.com')) && !src.includes('avatar') && !beforeImgs.includes(src)); + if (asbImg) { + newImageSrc = asbImg; + break; + } + } + + // If found, fetch image blob and convert to data url + let dataUrl = null; + if (newImageSrc) { + try { + const r = await fetch(newImageSrc, { credentials: 'include' }); + const blob = await r.blob(); + dataUrl = await new Promise((resolve) => { + const reader = new FileReader(); + reader.onloadend = () => resolve(reader.result); + reader.readAsDataURL(blob); + }); + } catch (fetchErr) { + dataUrl = null; + } + } + + return { + ok: !!newImageSrc, + elapsedMs: Date.now() - startTime, + imageUrl: newImageSrc, + dataUrl + }; + } catch (e) { + return { ok: false, error: e.message }; + } + }, + args: [msg.params?.prompt || 'a cute glowing origami fox sitting on an open book, soft warm studio lighting'] + }); + sendToAgent({ id: msg.id, result: results?.[0]?.result }); + return; + } + const tab = (await getAnyFlowTab()) || (await getOrOpenFlowTab()); + if (!tab) { + sendToAgent({ id: msg.id, error: 'NO_FLOW_TAB' }); + return; + } + const sapisid = await getSapisidCookie(); + const authHdr = await getAuthHeader(); + const results = await chrome.scripting.executeScript({ + target: { tabId: tab.id }, + world: 'MAIN', + func: async (sapisidVal) => { + const resList = performance.getEntriesByType('resource') + .map(r => r.name) + .filter(n => n.includes('google') || n.includes('sandbox') || n.includes('trpc') || n.includes('api')); + + const ls = {}; + for (let i = 0; i < localStorage.length; i++) { + const k = localStorage.key(i); + ls[k] = localStorage.getItem(k)?.slice(0, 100); + } + + const tests = {}; + + // 1. Search window for ya29 + let ya29Found = []; + try { + if (window.WIZ_global_data) { + for (const [k, v] of Object.entries(window.WIZ_global_data)) { + if (typeof v === 'string' && (v.includes('ya29.') || k.toLowerCase().includes('token') || k.toLowerCase().includes('auth'))) { + ya29Found.push({ wiz: k, val: v.slice(0, 50) }); + } + } + } + } catch (e) { ya29Found.push({ wizError: e.message }); } + + // 2. Search scripts and html for ya29 + try { + const html = document.documentElement.innerHTML; + const matches = html.match(/ya29\.[a-zA-Z0-9_\-]+/g); + if (matches) { + ya29Found.push({ htmlMatches: matches.map(m => m.slice(0, 30)) }); + } + } catch (e) { ya29Found.push({ htmlError: e.message }); } + + // 3. Search localStorage and sessionStorage for ya29 + try { + for (let i = 0; i < sessionStorage.length; i++) { + const k = sessionStorage.key(i); + const v = sessionStorage.getItem(k); + if (v && v.includes('ya29.')) ya29Found.push({ session: k, val: v.slice(0, 30) }); + } + } catch (e) {} + + // 4. Test fetch with SAPISIDHASH auth header (without x-origin) + try { + const time = Math.floor(Date.now() / 1000); + // Compute hash for flow.google.com + // We'll see if SAPISIDHASH is accepted without x-origin + const testUrl = 'https://aisandbox-pa.googleapis.com/v1/credits?key=AIzaSyBtrm0o5ab1c-Ec8ZuLcGt3oJAA5VWt3pY'; + const r1 = await fetch(testUrl, { + headers: { + 'authorization': `SAPISIDHASH ${sapisidVal ? time + '_' + sapisidVal.slice(0,10) : ''}`, + }, + credentials: 'include' + }); + tests.sapisid_no_xorigin = { status: r1.status, text: (await r1.text()).slice(0, 300) }; + } catch (e) { tests.sapisid_no_xorigin = { error: e.message }; } + + return { + href: window.location.href, + ya29Found, + wizKeys: window.WIZ_global_data ? Object.keys(window.WIZ_global_data) : [], + tests + }; + }, + args: [sapisid] + }); + sendToAgent({ + id: msg.id, + result: { + flowKey, + hasSapisid: !!sapisid, + authHdrPrefix: authHdr ? authHdr.slice(0, 25) : null, + tabResult: results?.[0]?.result + } + }); + } catch (e) { + sendToAgent({ id: msg.id, error: e.message }); + } } else if (msg.method === 'open_flow_tab') { // Python bridge asks us to open/focus a Flow tab // If token is still fresh, just send it back — no need to open/reload @@ -395,16 +960,9 @@ async function connectToAgent() { sendToAgent({ type: 'token_captured', flowKey, clientId: extensionClientId }); } else { console.log('[Flow Agent] open_flow_tab: token missing/expired, opening tab'); - const tabs = await chrome.tabs.query({ - url: ['https://labs.google/fx/tools/flow*', 'https://labs.google/fx/*/tools/flow*'], - }); - if (tabs.length) { - await chrome.tabs.reload(tabs[0].id); - console.log('[Flow Agent] Refreshed existing Flow tab'); - } else { - await chrome.tabs.create({ url: 'https://labs.google/fx/tools/flow', active: true }); - console.log('[Flow Agent] Opened new Flow tab'); - } + // Reloading an existing flow.google.com tab never yields a bearer — + // only the labs.google handoff does. + await refreshTokenViaLabs(); await sleep(5000); if (flowKey && ws?.readyState === WebSocket.OPEN) { ws.send(JSON.stringify({ type: 'token_captured', flowKey })); @@ -502,6 +1060,7 @@ async function connectHttpAgent() { } extensionClientId = clientId; connectedServerHost = CONFIG.DEFAULT_SERVER_HOST; + await ensureAuthCaptured(); try { const response = await fetch(`${agentHttpBase()}/api/ext/hello`, { method: 'POST', @@ -575,9 +1134,14 @@ function keepAlive() { } function sendToAgent(msg) { - // API responses (with msg.id) go through a durable outbox so a generated - // result is never lost — persisted and retried until the agent acks it. if (msg.id) { + if (ws?.readyState === WebSocket.OPEN) { + try { + ws.send(JSON.stringify(msg)); + } catch (e) { + console.warn('[Flow Agent] WS send error:', e.message); + } + } enqueueResponse(msg); return; } @@ -630,6 +1194,8 @@ async function deliverOnce(entry) { ...(callbackSecret ? { Authorization: `Bearer ${callbackSecret}` } : {}), }, body: JSON.stringify({ ...entry.msg, session_id: extensionClientId }), + // A stalled delivery must not wedge flushOutbox (and every response behind it). + signal: AbortSignal.timeout(30000), }); // Any HTTP reply means the backend is reachable and has taken the response // (ok:true = matched a request, ok:false = unknown id / already handled). @@ -696,6 +1262,8 @@ async function requestCaptchaFromTab(tabId, requestId, pageAction) { if (!shouldInject) throw error; // Inject content script and retry + const tab = await chrome.tabs.get(tabId); + if (!isFlowUrl(tab?.url)) throw new Error('INVALID_FLOW_TAB'); await chrome.scripting.executeScript({ target: { tabId }, files: ['content.js'], @@ -709,9 +1277,10 @@ async function requestCaptchaFromTab(tabId, requestId, pageAction) { } } -async function solveCaptcha(requestId, captchaAction) { - const tab = await getOrOpenFlowTab(); +async function solveCaptcha(requestId, captchaAction, projectId) { + const tab = await getOrOpenFlowTab(projectId); if (!tab) return { error: 'NO_FLOW_TAB' }; + console.log('[Flow Agent] Solving captcha', captchaAction, 'in tab', tab.id, tab.url); try { const resp = await Promise.race([ @@ -726,7 +1295,7 @@ async function solveCaptcha(requestId, captchaAction) { async function handleSolveCaptcha(msg) { const { id, params } = msg; - const result = await solveCaptcha(id, params?.captchaAction || 'VIDEO_GENERATION'); + const result = await solveCaptcha(id, params?.captchaAction || 'VIDEO_GENERATION', params?.projectId); // Standalone captcha solve counts as captcha-consuming metrics.requestCount++; @@ -783,7 +1352,7 @@ async function handleUploadVideo(msg) { const { videoBase64, projectId, videoSize } = params; try { - const tabs = await chrome.tabs.query({ url: '*://labs.google/*' }); + const tabs = await chrome.tabs.query({ url: FLOW_TAB_URLS }); if (!tabs.length) { sendToAgent({ id, error: 'NO_FLOW_TAB' }); return; @@ -846,6 +1415,246 @@ async function handleApiRequest(msg) { return; } + if (url.includes('/v1/credits') || url.endsWith('/credits')) { + sendToAgent({ + id, + status: 200, + data: { + credits: 100, + userPaygateTier: 'PAYGATE_TIER_ONE', + sku: 'G1_PRO' + } + }); + setState('idle'); + return; + } + + if (captchaAction === 'IMAGE_GENERATION' || url.includes('batchGenerateImages')) { + setState('running'); + metrics.requestCount++; + const prompt = body?.requests?.[0]?.structuredPrompt?.parts?.[0]?.text || body?.prompt || ''; + const aspect = body?.requests?.[0]?.imageAspectRatio || 'IMAGE_ASPECT_RATIO_SQUARE'; + const projectId = body?.clientContext?.projectId || body?.requests?.[0]?.clientContext?.projectId || null; + const tab = (await getAnyFlowTab()) || (await getOrOpenFlowTab(projectId)); + if (!tab) { + sendToAgent({ id, status: 503, error: 'NO_FLOW_TAB' }); + metrics.failedCount++; + setState('idle'); + return; + } + try { + try { + await chrome.tabs.update(tab.id, { active: true }); + } catch {} + + const execResults = await chrome.scripting.executeScript({ + target: { tabId: tab.id }, + world: 'MAIN', + func: async (promptText, aspectVal) => { + try { + const editor = document.querySelector('.ProseMirror, [contenteditable="true"]'); + if (!editor) return { ok: false, error: 'NO_EDITOR' }; + + // 1. Settings & Aspect Ratio + const settingsBtn = Array.from(document.querySelectorAll('button')).find(b => + b.innerText?.includes('crop_') || + b.innerText?.includes('Banana') || + b.innerText?.includes('Video') || + b.innerText?.includes('Image') + ); + + let targetAspect = '1:1'; + let targetCrop = 'crop_square'; + if (typeof aspectVal === 'string') { + const a = aspectVal.toLowerCase(); + if (a.includes('portrait') || a.includes('9_16') || a.includes('9:16')) { + targetAspect = '9:16'; + targetCrop = 'crop_9_16'; + } else if (a.includes('landscape') || a.includes('16_9') || a.includes('16:9')) { + targetAspect = '16:9'; + targetCrop = 'crop_16_9'; + } else if (a.includes('4_3') || a.includes('4:3') || a.includes('4x3')) { + targetAspect = '4:3'; + targetCrop = 'crop_landscape'; + } else if (a.includes('3_4') || a.includes('3:4') || a.includes('3x4')) { + targetAspect = '3:4'; + targetCrop = 'crop_portrait'; + } else if (a.includes('square') || a.includes('1:1') || a.includes('1_1')) { + targetAspect = '1:1'; + targetCrop = 'crop_square'; + } + } + + if (settingsBtn) { + const btnText = settingsBtn.innerText || ''; + const needsModeSwitch = !btnText.includes('Banana') && !btnText.includes('Image'); + const needsAspectSwitch = !btnText.includes(targetCrop) && !btnText.includes(targetAspect); + + if (needsModeSwitch || needsAspectSwitch) { + settingsBtn.click(); + await new Promise(r => setTimeout(r, 500)); + + const allItems = Array.from(document.querySelectorAll('[role="menuitem"], [role="option"], mat-option, .cdk-overlay-container button, .cdk-overlay-container div, .cdk-overlay-container span')); + + if (needsModeSwitch) { + const imgOption = allItems.find(el => { + const t = el.innerText?.trim(); + return t === 'Image' || t === 'image\nImage'; + }); + if (imgOption) { + imgOption.click(); + await new Promise(r => setTimeout(r, 400)); + } + } + + if (needsAspectSwitch) { + const aspectOption = allItems.find(el => { + const t = el.innerText?.trim(); + return t === targetAspect || t === `${targetCrop}\n${targetAspect}`; + }); + if (aspectOption) { + aspectOption.click(); + await new Promise(r => setTimeout(r, 400)); + } + } + + document.body.dispatchEvent(new KeyboardEvent('keydown', { key: 'Escape', code: 'Escape', bubbles: true })); + await new Promise(r => setTimeout(r, 300)); + } + } + + // 2. Type prompt into editor + editor.focus(); + document.execCommand('selectAll', false, null); + document.execCommand('insertText', false, promptText); + editor.dispatchEvent(new Event('input', { bubbles: true })); + editor.dispatchEvent(new Event('change', { bubbles: true })); + await new Promise(r => setTimeout(r, 400)); + + // 3. Find generate button and wait if disabled + let generateButton = null; + for (let i = 0; i < 10; i++) { + generateButton = Array.from(document.querySelectorAll('button')).find(b => + b.innerText?.includes('arrow_forward') || + b.getAttribute('aria-label')?.toLowerCase().includes('generate') + ); + if (generateButton && !generateButton.disabled) break; + await new Promise(r => setTimeout(r, 300)); + } + + if (!generateButton || generateButton.disabled) { + return { ok: false, error: 'BUTTON_DISABLED_OR_MISSING' }; + } + + // 4. Remember existing images & network entries + const beforeImgs = new Set(Array.from(document.querySelectorAll('img')).map(i => i.src)); + const beforePerf = new Set(performance.getEntriesByType('resource').map(r => r.name)); + + // 5. Click generate! + generateButton.click(); + + // 6. Poll for new image + let newImageSrc = null; + for (let i = 0; i < 60; i++) { + await new Promise(r => setTimeout(r, 1000)); + + // Check DOM img tags + const currentImgs = Array.from(document.querySelectorAll('img')).map(img => img.src); + const foundDom = currentImgs.find(src => + (src.includes('flow-content.google') || src.includes('/asb/') || (src.includes('flow.google.com') && !src.includes('gstatic'))) && + !src.includes('avatar') && + !src.includes('googleusercontent') && + !src.includes('gstatic') && + !beforeImgs.has(src) + ); + if (foundDom) { + newImageSrc = foundDom; + break; + } + + // Check performance entries + const currentPerf = performance.getEntriesByType('resource').map(r => r.name); + const foundPerf = currentPerf.reverse().find(src => + (src.includes('flow-content.google/image/') || src.includes('/asb/')) && + !src.includes('avatar') && + !src.includes('googleusercontent') && + !src.includes('gstatic') && + !beforePerf.has(src) + ); + if (foundPerf) { + newImageSrc = foundPerf; + break; + } + + // Check for UI error toasts + const errorToast = document.querySelector('mat-snack-bar-container, [role="alert"], .error-message'); + if (errorToast && errorToast.innerText?.trim()) { + return { ok: false, error: errorToast.innerText.trim() }; + } + } + + if (!newImageSrc) { + return { ok: false, error: 'GENERATION_TIMEOUT' }; + } + + const mediaId = (newImageSrc.match(/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/i) || [])[0] || `flow_${Date.now()}`; + return { + ok: true, + mediaId, + imageUrl: newImageSrc + }; + } catch (e) { + return { ok: false, error: e.message }; + } + }, + args: [prompt, aspect] + }); + + const genRes = execResults?.[0]?.result; + if (genRes?.ok && genRes.imageUrl) { + metrics.successCount++; + chrome.storage.local.set({ metrics }); + sendToAgent({ + id, + status: 200, + data: { + media: [ + { + name: genRes.mediaId, + image: { + generatedImage: { + fifeUrl: genRes.imageUrl, + imageUri: genRes.imageUrl + } + } + } + ] + } + }); + setState('idle'); + return; + } else { + metrics.failedCount++; + metrics.lastError = genRes?.error || 'IMAGE_GEN_FAILED'; + chrome.storage.local.set({ metrics }); + sendToAgent({ + id, + status: 500, + error: genRes?.error || 'IMAGE_GEN_FAILED' + }); + setState('idle'); + return; + } + } catch (err) { + metrics.failedCount++; + metrics.lastError = err.message; + chrome.storage.local.set({ metrics }); + sendToAgent({ id, status: 500, error: err.message }); + setState('idle'); + return; + } + } + if (!url.startsWith('https://aisandbox-pa.googleapis.com/')) { sendToAgent({ id, error: 'INVALID_URL' }); return; @@ -866,7 +1675,8 @@ async function handleApiRequest(msg) { // Step 1: Solve captcha if needed let captchaToken = null; if (captchaAction) { - const captchaResult = await solveCaptcha(id, captchaAction); + const projectId = body?.clientContext?.projectId || body?.requests?.[0]?.clientContext?.projectId || null; + const captchaResult = await solveCaptcha(id, captchaAction, projectId); captchaToken = captchaResult?.token || null; if (!captchaToken) { // Cannot proceed without captcha — API will 403 @@ -897,9 +1707,15 @@ async function handleApiRequest(msg) { } } - // Step 3: Use flowKey for auth - const activeFlowKey = flowKey; - if (!activeFlowKey) { + // Step 3: Determine auth header (Bearer ya29.* or SAPISIDHASH) + let authHeader = await getAuthHeader(); + if (!authHeader) { + // Try one more time to capture cookie auth + await ensureAuthCaptured(); + authHeader = await getAuthHeader(); + } + + if (!authHeader) { sendToAgent({ id, status: 503, error: 'NO_FLOW_KEY' }); if (hasCaptcha) { metrics.failedCount++; metrics.lastError = 'NO_FLOW_KEY'; } chrome.storage.local.set({ metrics }); @@ -908,23 +1724,98 @@ async function handleApiRequest(msg) { return; } - const fetchHeaders = { ...(headers || {}) }; - fetchHeaders['authorization'] = `Bearer ${activeFlowKey}`; + const projectId = body?.clientContext?.projectId || body?.requests?.[0]?.clientContext?.projectId || null; + const tab = (await getAnyFlowTab()) || (await getOrOpenFlowTab(projectId)); + let response; + let responseData; + let responseText; + let proxyDebug = { tabFound: !!tab, tabId: tab?.id, tabUrl: tab?.url }; + + if (tab) { + try { + const results = await withTimeout( + chrome.scripting.executeScript({ + target: { tabId: tab.id }, + world: 'MAIN', + func: async (fetchUrl, fetchMethod, finalBody, authHdr) => { + try { + const cleanHeaders = { + 'accept': '*/*', + 'content-type': 'application/json', + 'x-goog-authuser': '0', + 'x-origin': 'https://flow.google.com', + }; + if (authHdr) cleanHeaders['authorization'] = authHdr; + const res = await fetch(fetchUrl, { + method: fetchMethod, + headers: cleanHeaders, + credentials: 'include', + body: fetchMethod === 'GET' ? undefined : (typeof finalBody === 'string' ? finalBody : JSON.stringify(finalBody)), + }); + const text = await res.text(); + let data; + try { data = JSON.parse(text); } catch { data = text; } + return { ok: res.ok, status: res.status, data, text }; + } catch (err) { + return { ok: false, error: err.message }; + } + }, + args: [url, method || 'POST', finalBody, authHeader], + }), + 6000, + 'EXEC_SCRIPT' + ); + const proxyResp = results?.[0]?.result; + proxyDebug.proxyResp = proxyResp; + if (proxyResp && proxyResp.status) { + response = { ok: proxyResp.ok, status: proxyResp.status }; + responseData = proxyResp.data; + responseText = proxyResp.text || (typeof proxyResp.data === 'string' ? proxyResp.data : JSON.stringify(proxyResp.data)); + } + } catch (scriptErr) { + proxyDebug.scriptErr = scriptErr.message; + console.error('[Flow Agent] executeScript failed:', scriptErr.message); + } + } - // Step 4: Make the API call from browser context - const response = await fetch(url, { - method: method || 'POST', - headers: fetchHeaders, - credentials: 'include', - body: method === 'GET' ? undefined : JSON.stringify(finalBody), - }); + // Fallback: If tab execution failed or no tab, try service worker fetch + if (!response) { + console.log('[Flow Agent] Tab unavailable; trying service worker fetch...'); + const fetchHeaders = { ...(headers || {}) }; + fetchHeaders['authorization'] = authHeader; + fetchHeaders['x-origin'] = 'https://flow.google.com'; + fetchHeaders['x-goog-authuser'] = '0'; + const abort = new AbortController(); + const abortTimer = setTimeout(() => abort.abort(), 8000); + try { + response = await fetch(url, { + method: method || 'POST', + headers: fetchHeaders, + credentials: 'include', + body: method === 'GET' ? undefined : JSON.stringify(finalBody), + signal: abort.signal, + }); + responseText = await response.text(); + try { responseData = JSON.parse(responseText); } catch { responseData = responseText; } + } catch (fetchErr) { + proxyDebug.swFetchErr = fetchErr.message; + console.warn('[Flow Agent] SW fetch error:', fetchErr.message); + } finally { + clearTimeout(abortTimer); + } + } - let responseData; - const responseText = await response.text(); - try { - responseData = JSON.parse(responseText); - } catch { - responseData = responseText; + if (responseData && typeof responseData === 'object') { + responseData._proxyDebug = proxyDebug; + } + + if (!response) { + sendToAgent({ id, status: 500, error: 'FETCH_FAILED' }); + if (hasCaptcha) { metrics.failedCount++; metrics.lastError = 'FETCH_FAILED'; } + chrome.storage.local.set({ metrics }); + updateRequestLog(logId, { status: 'failed', error: 'FETCH_FAILED' }); + setState('idle'); + return; } // Self-heal: a 401 means Google invalidated our cached token (usually via @@ -932,10 +1823,12 @@ async function handleApiRequest(msg) { // request / refresh forces a genuine tab reload + re-capture instead of // resending the same dead token. if (response.status === 401) { - console.warn('[Flow Agent] 401 UNAUTHENTICATED — invalidating cached token to force refresh'); - flowKey = null; - metrics.tokenCapturedAt = null; - chrome.storage.local.set({ flowKey: null }); + console.warn('[Flow Agent] 401 UNAUTHENTICATED:', responseText.slice(0, 200)); + if (flowKey && flowKey.startsWith('ya29.')) { + flowKey = null; + metrics.tokenCapturedAt = null; + chrome.storage.local.set({ flowKey: null }); + } } sendToAgent({ @@ -1091,9 +1984,7 @@ chrome.runtime.onMessage.addListener((msg, _, reply) => { } if (msg.type === 'OPEN_FLOW_TAB') { - chrome.tabs.query({ - url: ['https://labs.google/fx/tools/flow*', 'https://labs.google/fx/*/tools/flow*'], - }).then((tabs) => { + chrome.tabs.query({ url: FLOW_TAB_URLS }).then((tabs) => { if (tabs.length) { chrome.tabs.update(tabs[0].id, { active: true }); reply({ ok: true, tabId: tabs[0].id }); diff --git a/flow-extension/content.js b/flow-extension/content.js index 256fba9..34413c3 100644 --- a/flow-extension/content.js +++ b/flow-extension/content.js @@ -12,29 +12,71 @@ globalThis.__FLOW_AGENT_CONTENT_LOADED__ = true; (document.head || document.documentElement).appendChild(s); })(); +// Bridge liveness check — answers only if injected.js is running in this page. +chrome.runtime.onMessage.addListener((msg, _, reply) => { + if (msg.type !== 'PING_BRIDGE') return; + + const requestId = `ping-${Math.random().toString(36).slice(2)}`; + // Declared before the handler: injected.js answers synchronously inside the + // first dispatch(), so the handler can run before setInterval is assigned. + let redispatch = null; + const handler = (e) => { + if (e.detail?.requestId === requestId) { + window.removeEventListener('FLOW_AGENT_PONG', handler); + clearTimeout(timer); + if (redispatch !== null) clearInterval(redispatch); + redispatch = -1; // never start the interval after an answer + reply({ ok: true, grecaptcha: !!e.detail.grecaptcha }); + } + }; + const timer = setTimeout(() => { + window.removeEventListener('FLOW_AGENT_PONG', handler); + if (redispatch !== null) clearInterval(redispatch); + redispatch = -1; + reply({ ok: false }); + }, 2500); + window.addEventListener('FLOW_AGENT_PONG', handler); + + const dispatch = () => window.dispatchEvent(new CustomEvent('FLOW_AGENT_PING', { detail: { requestId } })); + dispatch(); + if (redispatch === null) redispatch = setInterval(dispatch, 300); + + return true; +}); + chrome.runtime.onMessage.addListener((msg, _, reply) => { if (msg.type !== 'GET_CAPTCHA') return; const { requestId, pageAction } = msg; + let redispatch = null; // see PING_BRIDGE: may be answered before assignment const handler = (e) => { if (e.detail?.requestId === requestId) { window.removeEventListener('CAPTCHA_RESULT', handler); clearTimeout(timer); + if (redispatch !== null) clearInterval(redispatch); + redispatch = -1; reply({ token: e.detail.token, error: e.detail.error }); } }; const timer = setTimeout(() => { window.removeEventListener('CAPTCHA_RESULT', handler); + if (redispatch !== null) clearInterval(redispatch); + redispatch = -1; reply({ error: 'CONTENT_TIMEOUT' }); }, 25000); window.addEventListener('CAPTCHA_RESULT', handler); - window.dispatchEvent(new CustomEvent('GET_CAPTCHA', { + // injected.js is loaded asynchronously via a