From 73d700d30adbac6e12dcdc40e808d12d1f513060 Mon Sep 17 00:00:00 2001 From: Antigravity Date: Thu, 17 Sep 2026 07:39:06 +0800 Subject: [PATCH] fix(chat-swarm-runtime): drive ChatGPT composer through CDP Input --- src/chat-swarm-runtime.test.ts | 87 +++++++++++++++++++------ src/chat-swarm-runtime.ts | 115 +++++++++++++++++++++++++++------ 2 files changed, 161 insertions(+), 41 deletions(-) diff --git a/src/chat-swarm-runtime.test.ts b/src/chat-swarm-runtime.test.ts index 13ab20869..0bf022b74 100644 --- a/src/chat-swarm-runtime.test.ts +++ b/src/chat-swarm-runtime.test.ts @@ -207,26 +207,65 @@ test("CDP composer readiness ignores hidden fallback textareas and prefers visib assert.ok(expression.indexOf("[contenteditable=\"true\"]") < expression.indexOf("textarea")); }); -test("CDP prompt delivery targets the visible editable composer before textarea fallback", async () => { +test("CDP prompt delivery uses exact-target Input commands for visible editable composer", async () => { const driver = cdpDriverForSelectorTest(); - let expression = ""; + const expressions: string[] = []; + const commands: Array<{ method: string; params: Record }> = []; (driver as any).evaluate = async (_target: unknown, candidate: string) => { - expression = candidate; + expressions.push(candidate); + if (candidate.includes("const editable=")) return { ok: true, kind: "editable" }; + if (candidate.includes("return {ready:Boolean")) return { ready: true }; return { ok: true }; }; + (driver as any).pageCommand = async ( + _target: unknown, + method: string, + params: Record, + ) => { + commands.push({ method, params }); + return {}; + }; await (driver as any).sendPromptToTarget( { id: "target-1", url: "https://chatgpt.com/" }, "probe", new Date(Date.now() + 1_000).toISOString(), + "https://chatgpt.com/", ); - assert.match(expression, /getBoundingClientRect/); - assert.match(expression, /data-testid="send-button"/); - assert.ok(expression.indexOf("const editable=") < expression.indexOf("const textarea=")); - assert.match(expression, /const el=editable\|\|textarea/); - assert.match(expression, /document\.createRange\(\)/); - assert.match(expression, /selectNodeContents\(editable\)/); - assert.match(expression, /document\.execCommand\('insertText',false,prompt\)/); - assert.doesNotMatch(expression, /editable\.textContent=prompt/); + assert.match(expressions[0]!, /getBoundingClientRect/); + assert.ok(expressions[0]!.indexOf("const editable=") < expressions[0]!.indexOf("const textarea=")); + assert.match(expressions[0]!, /CHATGPT_CONVERSATION_IDENTITY_DRIFT/); + assert.doesNotMatch(expressions.join("\n"), /document\.execCommand/); + assert.doesNotMatch(expressions.join("\n"), /editable\.textContent=prompt/); + assert.deepEqual(commands, [ + { method: "Input.dispatchKeyEvent", params: { type: "keyDown", modifiers: 4, key: "a", code: "KeyA", windowsVirtualKeyCode: 65 } }, + { method: "Input.dispatchKeyEvent", params: { type: "keyUp", modifiers: 4, key: "a", code: "KeyA", windowsVirtualKeyCode: 65 } }, + { method: "Input.dispatchKeyEvent", params: { type: "keyDown", key: "Backspace", code: "Backspace", windowsVirtualKeyCode: 8 } }, + { method: "Input.dispatchKeyEvent", params: { type: "keyUp", key: "Backspace", code: "Backspace", windowsVirtualKeyCode: 8 } }, + { method: "Input.insertText", params: { text: "probe" } }, + ]); + assert.match(expressions.at(-1)!, /button\.click\(\)/); +}); + +test("CDP prompt delivery keeps textarea fallback on the DOM value/input path", async () => { + const driver = cdpDriverForSelectorTest(); + const expressions: string[] = []; + let pageCommands = 0; + (driver as any).evaluate = async (_target: unknown, candidate: string) => { + expressions.push(candidate); + if (candidate.includes("const editable=")) return { ok: true, kind: "textarea" }; + if (candidate.includes("return {ready:Boolean")) return { ready: true }; + return { ok: true }; + }; + (driver as any).pageCommand = async () => { pageCommands += 1; }; + await (driver as any).sendPromptToTarget( + { id: "target-1", url: "https://chatgpt.com/" }, + "probe", + new Date(Date.now() + 1_000).toISOString(), + ); + assert.match(expressions[0]!, /HTMLTextAreaElement\.prototype/); + assert.match(expressions[0]!, /setter\.call\(textarea,prompt\)/); + assert.match(expressions[0]!, /textarea\.dispatchEvent\(new Event\('input'/); + assert.equal(pageCommands, 0); }); function openCliDriverForTest() { @@ -795,19 +834,24 @@ function existingCdpFixture(t: test.TestContext, metadata = "43123\n/devtools/br result = { sessionId }; break; } case "Target.closeTarget": targets.delete(command.params.targetId); result = { success: true }; break; + case "Input.dispatchKeyEvent": result = {}; break; + case "Input.insertText": result = {}; break; case "Runtime.evaluate": { const url = targets.get(sessions.get(command.sessionId)!); assert.ok(url, "evaluation must be attached to one live target"); const expression: string = command.params.expression; let value: any = true; if (expression === "location.href") value = url; - else if (expression.includes("const prompt=")) { - if (disconnectAfterSend) { this.close(); return; } + else if (expression.includes("const editable=")) { if (driftBeforeSend) { targets.set(sessions.get(command.sessionId)!, "https://chatgpt.com/c/unrelated"); - // Execute the actual guard, before any DOM write could happen. - value = new Function("location", expression.replace("(() =>", "return (() =>"))({ href: "https://chatgpt.com/c/unrelated" }); - } else value = { ok: true }; + value = { ok: false, reason: "CHATGPT_CONVERSATION_IDENTITY_DRIFT" }; + } else value = { ok: true, kind: "editable" }; + } + else if (expression.includes("return {ready:Boolean")) value = { ready: true }; + else if (expression.includes("button.click()")) { + if (disconnectAfterSend) { this.close(); return; } + value = { ok: true }; } else if (expression.includes("const label=")) { const text = domState === "unknown" ? "ChatGPT" : `dev ${domState}`; @@ -968,7 +1012,8 @@ macOsOnlyTest("existing-session prompt acknowledgement loss remains unknown and assert.equal(result.delivered, false); assert.equal(result.remoteMayContinue, true); assert.match(result.blocker!, /BROWSER_CONTROL_UNAVAILABLE:DISCONNECTED/); - assert.equal(f.calls.filter(c => c.method === "Runtime.evaluate" && c.params.expression.includes("const prompt=")).length, 1); + assert.equal(f.calls.filter(c => c.method === "Input.insertText").length, 1); + assert.equal(f.calls.filter(c => c.method === "Runtime.evaluate" && c.params.expression.includes("button.click()")).length, 1); assert.equal(f.calls.filter(c => c.method === "Target.createTarget").length, 0); assert.equal(f.sockets.length, 1); }); @@ -996,7 +1041,7 @@ macOsOnlyTest("existing-session recovery after target deletion reopens only the const url = "https://chatgpt.com/c/worker-a"; assert.equal((await f.driver.recoverConversation(url, f.deadline())).ready, true); assert.deepEqual(f.calls.filter(c => c.method === "Target.createTarget").map(c => c.params.url), [url]); - assert.equal(f.calls.filter(c => c.method === "Runtime.evaluate" && c.params.expression.includes("const prompt=")).length, 0); + assert.equal(f.calls.filter(c => c.method === "Input.insertText").length, 0); assert.equal(f.targets.get("target-b"), "https://chatgpt.com/c/worker-b"); }); @@ -1085,7 +1130,8 @@ macOsOnlyTest("existing-session browser readiness surfaces unknown app binding w assert.deepEqual(await f.driver.recoverConversation(url, f.deadline()), { ready: true, blocker: "HOST_APP_BINDING_NOT_READY:UNKNOWN", }); - assert.equal(f.calls.filter(c => c.method === "Runtime.evaluate" && c.params.expression.includes("const prompt=")).length, 1); + assert.equal(f.calls.filter(c => c.method === "Input.insertText").length, 1); + assert.equal(f.calls.filter(c => c.method === "Runtime.evaluate" && c.params.expression.includes("button.click()")).length, 1); assert.ok(f.calls.filter(c => c.method === "Target.attachToTarget").every(c => c.params.targetId === "target-a")); assert.equal(f.targets.get("target-b"), "https://chatgpt.com/c/worker-b"); }); @@ -1110,7 +1156,8 @@ for (const binding of ["DISABLED", "STALE"] as const) { assert.equal(bootstrap.remoteMayContinue, false); assert.equal(bootstrap.blocker, `HOST_APP_BINDING_NOT_READY:${binding}`); } finally { registry.close(); } - assert.equal(f.calls.filter(c => c.method === "Runtime.evaluate" && c.params.expression.includes("const prompt=")).length, 0); + assert.equal(f.calls.filter(c => c.method === "Input.insertText").length, 0); + assert.equal(f.calls.filter(c => c.method === "Runtime.evaluate" && c.params.expression.includes("button.click()")).length, 0); }); } diff --git a/src/chat-swarm-runtime.ts b/src/chat-swarm-runtime.ts index 0b185a4d9..28e6abf0e 100644 --- a/src/chat-swarm-runtime.ts +++ b/src/chat-swarm-runtime.ts @@ -1827,7 +1827,11 @@ class ExistingChromeControl { }); } - async evaluate(targetId: string, expression: string): Promise { + async targetCommand( + targetId: string, + method: string, + params: Record = {}, + ): Promise { let sessionId = this.sessions.get(targetId); if (!sessionId) { const attached = await this.command<{ sessionId: string }>("Target.attachToTarget", { targetId, flatten: true }); @@ -1835,8 +1839,14 @@ class ExistingChromeControl { sessionId = attached.sessionId; this.sessions.set(targetId, sessionId); } - const result = await this.command<{ result?: { value?: T }; exceptionDetails?: unknown }>( - "Runtime.evaluate", { expression, returnByValue: true, awaitPromise: true }, sessionId, + return this.command(method, params, sessionId); + } + + async evaluate(targetId: string, expression: string): Promise { + const result = await this.targetCommand<{ result?: { value?: T }; exceptionDetails?: unknown }>( + targetId, + "Runtime.evaluate", + { expression, returnByValue: true, awaitPromise: true }, ); if (result.exceptionDetails) throw new Error("CHATGPT_EVALUATION_FAILED"); return result.result?.value as T; @@ -2111,8 +2121,12 @@ export class CdpMacWebDriver implements MacWebDriver { } } - private async evaluate(target: CdpTarget, expression: string): Promise { - if (this.control) return this.control.evaluate(target.id, expression); + private async pageCommand( + target: CdpTarget, + method: string, + params: Record = {}, + ): Promise { + if (this.control) return this.control.targetCommand(target.id, method, params); if (!target.webSocketDebuggerUrl) { target = (await this.targets( @@ -2130,16 +2144,9 @@ export class CdpMacWebDriver implements MacWebDriver { const ws = new WebSocketCtor(target.webSocketDebuggerUrl!); const timer = setTimeout(() => { try { ws.close(); } catch {} - rejectPromise(new Error("CDP evaluate timed out")); + rejectPromise(new Error(`CDP ${method} timed out`)); }, Math.min(10_000, this.config.operationTimeoutMs)); - ws.onopen = () => - ws.send( - JSON.stringify({ - id: 1, - method: "Runtime.evaluate", - params: { expression, returnByValue: true, awaitPromise: true }, - }), - ); + ws.onopen = () => ws.send(JSON.stringify({ id: 1, method, params })); ws.onerror = () => { clearTimeout(timer); rejectPromise(new Error("CDP websocket failed")); @@ -2147,21 +2154,31 @@ export class CdpMacWebDriver implements MacWebDriver { ws.onmessage = (event: { data: string }) => { const payload = JSON.parse(String(event.data)) as { id?: number; - result?: { result?: { value?: T } }; + result?: T; error?: { message?: string }; }; if (payload.id !== 1) return; clearTimeout(timer); try { ws.close(); } catch {} if (payload.error) { - rejectPromise(new Error(payload.error.message ?? "CDP evaluate failed")); + rejectPromise(new Error(payload.error.message ?? `CDP ${method} failed`)); } else { - resolvePromise(payload.result?.result?.value as T); + resolvePromise(payload.result as T); } }; }); } + private async evaluate(target: CdpTarget, expression: string): Promise { + const result = await this.pageCommand<{ result?: { value?: T }; exceptionDetails?: unknown }>( + target, + "Runtime.evaluate", + { expression, returnByValue: true, awaitPromise: true }, + ); + if (result.exceptionDetails) throw new Error("CHATGPT_EVALUATION_FAILED"); + return result.result?.value as T; + } + private async waitForComposer(target: CdpTarget, deadlineAt: string): Promise { while (Date.now() < Date.parse(deadlineAt)) { const ready = await this.evaluate( @@ -2196,12 +2213,68 @@ export class CdpMacWebDriver implements MacWebDriver { } const encoded = JSON.stringify(prompt); const expectedUrl = JSON.stringify(expectedConversationUrl ?? null); - const result = await this.evaluate<{ ok: boolean; reason?: string }>( + const composer = await this.evaluate<{ + ok: boolean; + kind?: "editable" | "textarea"; + reason?: string; + }>( + target, + `(() => { const expectedUrl=${expectedUrl}; if(expectedUrl && location.href !== expectedUrl) return {ok:false,reason:'CHATGPT_CONVERSATION_IDENTITY_DRIFT'}; const prompt=${encoded}; const visible=(el) => { const rect=el.getBoundingClientRect(); const style=getComputedStyle(el); return rect.width > 0 && rect.height > 0 && style.display !== 'none' && style.visibility !== 'hidden'; }; const editable=[...document.querySelectorAll('[contenteditable="true"]')].find(visible); const textarea=[...document.querySelectorAll('textarea')].find(visible); const el=editable||textarea; if(!el) return {ok:false,reason:'composer_missing'}; el.focus(); if(editable) return {ok:true,kind:'editable'}; const setter=Object.getOwnPropertyDescriptor(HTMLTextAreaElement.prototype,'value')?.set; if(!setter) return {ok:false,reason:'textarea_setter_missing'}; setter.call(textarea,prompt); textarea.dispatchEvent(new Event('input',{bubbles:true})); return {ok:true,kind:'textarea'}; })()`, + ); + if (!composer?.ok || !composer.kind) { + throw new Error(composer?.reason ?? "ChatGPT composer preparation failed"); + } + if (composer.kind === "editable") { + await this.pageCommand(target, "Input.dispatchKeyEvent", { + type: "keyDown", + modifiers: 4, + key: "a", + code: "KeyA", + windowsVirtualKeyCode: 65, + }); + await this.pageCommand(target, "Input.dispatchKeyEvent", { + type: "keyUp", + modifiers: 4, + key: "a", + code: "KeyA", + windowsVirtualKeyCode: 65, + }); + await this.pageCommand(target, "Input.dispatchKeyEvent", { + type: "keyDown", + key: "Backspace", + code: "Backspace", + windowsVirtualKeyCode: 8, + }); + await this.pageCommand(target, "Input.dispatchKeyEvent", { + type: "keyUp", + key: "Backspace", + code: "Backspace", + windowsVirtualKeyCode: 8, + }); + await this.pageCommand(target, "Input.insertText", { text: prompt }); + } + + let sendReady = false; + while (Date.now() < Date.parse(deadlineAt)) { + const readiness = await this.evaluate<{ ready: boolean; reason?: string }>( + target, + `(() => { const expectedUrl=${expectedUrl}; if(expectedUrl && location.href !== expectedUrl) return {ready:false,reason:'CHATGPT_CONVERSATION_IDENTITY_DRIFT'}; const button=document.querySelector('[data-testid="send-button"]') || [...document.querySelectorAll('button')].find(b => /send/i.test((b.getAttribute('aria-label')||b.textContent||''))); return {ready:Boolean(button && !button.disabled)}; })()`, + ); + if (readiness?.reason) throw new Error(readiness.reason); + if (readiness?.ready) { + sendReady = true; + break; + } + await new Promise((resolvePromise) => setTimeout(resolvePromise, 50)); + } + if (!sendReady) throw new Error("send_button_missing_or_disabled"); + + const sent = await this.evaluate<{ ok: boolean; reason?: string }>( target, - `(() => { const expectedUrl=${expectedUrl}; if(expectedUrl && location.href !== expectedUrl) return {ok:false,reason:'CHATGPT_CONVERSATION_IDENTITY_DRIFT'}; const prompt=${encoded}; const visible=(el) => { const rect=el.getBoundingClientRect(); const style=getComputedStyle(el); return rect.width > 0 && rect.height > 0 && style.display !== 'none' && style.visibility !== 'hidden'; }; const editable=[...document.querySelectorAll('[contenteditable="true"]')].find(visible); const textarea=[...document.querySelectorAll('textarea')].find(visible); const el=editable||textarea; if(!el) return {ok:false,reason:'composer_missing'}; el.focus(); if(editable){ const range=document.createRange(); range.selectNodeContents(editable); const selection=window.getSelection(); if(!selection) return {ok:false,reason:'composer_selection_unavailable'}; selection.removeAllRanges(); selection.addRange(range); const inserted=document.execCommand('insertText',false,prompt); selection.removeAllRanges(); if(!inserted) return {ok:false,reason:'composer_insert_failed'}; } else { const setter=Object.getOwnPropertyDescriptor(HTMLTextAreaElement.prototype,'value')?.set; setter?.call(textarea,prompt); textarea.dispatchEvent(new Event('input',{bubbles:true})); } const button=document.querySelector('[data-testid="send-button"]') || [...document.querySelectorAll('button')].find(b => /send/i.test((b.getAttribute('aria-label')||b.textContent||''))); if(!button || button.disabled) return {ok:false,reason:'send_button_missing_or_disabled'}; button.click(); return {ok:true}; })()`, + `(() => { const expectedUrl=${expectedUrl}; if(expectedUrl && location.href !== expectedUrl) return {ok:false,reason:'CHATGPT_CONVERSATION_IDENTITY_DRIFT'}; const button=document.querySelector('[data-testid="send-button"]') || [...document.querySelectorAll('button')].find(b => /send/i.test((b.getAttribute('aria-label')||b.textContent||''))); if(!button || button.disabled) return {ok:false,reason:'send_button_missing_or_disabled'}; button.click(); return {ok:true}; })()`, ); - if (!result?.ok) { - throw new Error(result?.reason ?? "ChatGPT prompt delivery failed"); + if (!sent?.ok) { + throw new Error(sent?.reason ?? "ChatGPT prompt delivery failed"); } if (Date.now() > Date.parse(deadlineAt)) { throw new Error("prompt delivery exceeded deadline");