diff --git a/src/chat-swarm-runtime.test.ts b/src/chat-swarm-runtime.test.ts index 7c275535..6fc963e8 100644 --- a/src/chat-swarm-runtime.test.ts +++ b/src/chat-swarm-runtime.test.ts @@ -214,7 +214,7 @@ test("CDP prompt delivery uses exact-target Input commands for visible editable (driver as any).evaluate = async (_target: unknown, candidate: string) => { expressions.push(candidate); if (candidate.includes("const editable=")) return { ok: true, kind: "editable" }; - if (candidate.includes("return {ready:Boolean")) return { ready: true }; + if (candidate.includes("const button=")) return { ready: true, x: 320, y: 48 }; return { ok: true }; }; (driver as any).pageCommand = async ( @@ -242,21 +242,31 @@ test("CDP prompt delivery uses exact-target Input commands for visible editable { method: "Input.dispatchKeyEvent", params: { type: "keyDown", key: "Backspace", code: "Backspace", windowsVirtualKeyCode: 8 } }, { method: "Input.dispatchKeyEvent", params: { type: "keyUp", key: "Backspace", code: "Backspace", windowsVirtualKeyCode: 8 } }, { method: "Input.insertText", params: { text: "probe" } }, + { method: "Input.dispatchMouseEvent", params: { type: "mousePressed", x: 320, y: 48, button: "left", clickCount: 1 } }, + { method: "Input.dispatchMouseEvent", params: { type: "mouseReleased", x: 320, y: 48, button: "left", clickCount: 1 } }, ]); - assert.match(expressions.at(-1)!, /button\.click\(\)/); + assert.doesNotMatch(expressions.join("\n"), /button\.click\(\)/); + assert.match(expressions.at(-1)!, /getBoundingClientRect/); }); test("CDP prompt delivery keeps textarea fallback on the DOM value/input path", async () => { const driver = cdpDriverForSelectorTest(); const expressions: string[] = []; - let pageCommands = 0; + const commands: Array<{ method: string; params: Record }> = []; (driver as any).evaluate = async (_target: unknown, candidate: string) => { expressions.push(candidate); if (candidate.includes("const editable=")) return { ok: true, kind: "textarea" }; - if (candidate.includes("return {ready:Boolean")) return { ready: true }; + if (candidate.includes("const button=")) return { ready: true, x: 160, y: 72 }; return { ok: true }; }; - (driver as any).pageCommand = async () => { pageCommands += 1; }; + (driver as any).pageCommand = async ( + _target: unknown, + method: string, + params: Record, + ) => { + commands.push({ method, params }); + return {}; + }; await (driver as any).sendPromptToTarget( { id: "target-1", url: "https://chatgpt.com/" }, "probe", @@ -265,7 +275,11 @@ test("CDP prompt delivery keeps textarea fallback on the DOM value/input path", assert.match(expressions[0]!, /HTMLTextAreaElement\.prototype/); assert.match(expressions[0]!, /setter\.call\(textarea,prompt\)/); assert.match(expressions[0]!, /textarea\.dispatchEvent\(new Event\('input'/); - assert.equal(pageCommands, 0); + assert.deepEqual(commands, [ + { method: "Input.dispatchMouseEvent", params: { type: "mousePressed", x: 160, y: 72, button: "left", clickCount: 1 } }, + { method: "Input.dispatchMouseEvent", params: { type: "mouseReleased", x: 160, y: 72, button: "left", clickCount: 1 } }, + ]); + assert.doesNotMatch(expressions.join("\n"), /button\.click\(\)/); }); test("CDP managed conversation waits for the initialization turn to become idle before exposing identity", async () => { diff --git a/src/chat-swarm-runtime.ts b/src/chat-swarm-runtime.ts index 3f6652bc..42bf1571 100644 --- a/src/chat-swarm-runtime.ts +++ b/src/chat-swarm-runtime.ts @@ -2255,28 +2255,46 @@ export class CdpMacWebDriver implements MacWebDriver { await this.pageCommand(target, "Input.insertText", { text: prompt }); } - let sendReady = false; + let sendPoint: { x: number; y: number } | undefined; while (Date.now() < Date.parse(deadlineAt)) { - const readiness = await this.evaluate<{ ready: boolean; reason?: string }>( + const readiness = await this.evaluate<{ + ready: boolean; + reason?: string; + x?: number; + y?: number; + }>( target, - `(() => { const expectedUrl=${expectedUrl}; if(expectedUrl && location.href !== expectedUrl) return {ready:false,reason:'CHATGPT_CONVERSATION_IDENTITY_DRIFT'}; const button=document.querySelector('[data-testid="send-button"]') || [...document.querySelectorAll('button')].find(b => /send/i.test((b.getAttribute('aria-label')||b.textContent||''))); return {ready:Boolean(button && !button.disabled)}; })()`, + `(() => { const expectedUrl=${expectedUrl}; if(expectedUrl && location.href !== expectedUrl) return {ready:false,reason:'CHATGPT_CONVERSATION_IDENTITY_DRIFT'}; const button=document.querySelector('[data-testid="send-button"]') || [...document.querySelectorAll('button')].find(b => /send/i.test((b.getAttribute('aria-label')||b.textContent||''))); if(!button || button.disabled) return {ready:false}; const rect=button.getBoundingClientRect(); const style=getComputedStyle(button); if(rect.width <= 0 || rect.height <= 0 || style.display === 'none' || style.visibility === 'hidden') return {ready:false}; return {ready:true,x:rect.left + rect.width / 2,y:rect.top + rect.height / 2}; })()`, ); if (readiness?.reason) throw new Error(readiness.reason); - if (readiness?.ready) { - sendReady = true; + if ( + readiness?.ready && + typeof readiness.x === "number" && + Number.isFinite(readiness.x) && + typeof readiness.y === "number" && + Number.isFinite(readiness.y) + ) { + sendPoint = { x: readiness.x, y: readiness.y }; break; } await new Promise((resolvePromise) => setTimeout(resolvePromise, 50)); } - if (!sendReady) throw new Error("send_button_missing_or_disabled"); + if (!sendPoint) throw new Error("send_button_missing_or_disabled"); - const sent = await this.evaluate<{ ok: boolean; reason?: string }>( - target, - `(() => { const expectedUrl=${expectedUrl}; if(expectedUrl && location.href !== expectedUrl) return {ok:false,reason:'CHATGPT_CONVERSATION_IDENTITY_DRIFT'}; const button=document.querySelector('[data-testid="send-button"]') || [...document.querySelectorAll('button')].find(b => /send/i.test((b.getAttribute('aria-label')||b.textContent||''))); if(!button || button.disabled) return {ok:false,reason:'send_button_missing_or_disabled'}; button.click(); return {ok:true}; })()`, - ); - if (!sent?.ok) { - throw new Error(sent?.reason ?? "ChatGPT prompt delivery failed"); - } + await this.pageCommand(target, "Input.dispatchMouseEvent", { + type: "mousePressed", + x: sendPoint.x, + y: sendPoint.y, + button: "left", + clickCount: 1, + }); + await this.pageCommand(target, "Input.dispatchMouseEvent", { + type: "mouseReleased", + x: sendPoint.x, + y: sendPoint.y, + button: "left", + clickCount: 1, + }); if (Date.now() > Date.parse(deadlineAt)) { throw new Error("prompt delivery exceeded deadline"); }