|
| 1 | +import type { CuaBrowserExpectation } from "@onkernel/cua-ai"; |
| 2 | +import { observedNodes, staticTextRun, type AXNode, type BrowserObservation } from "./browser-observation"; |
| 3 | +import type { BrowserExpectationEvidence, BrowserWaitForResult, BrowserWaitReason } from "./types"; |
| 4 | + |
| 5 | +/** Internal expectation truth and optional lifecycle reason. */ |
| 6 | +export interface BrowserExpectationEvaluation extends BrowserExpectationEvidence { |
| 7 | + reason?: BrowserWaitReason; |
| 8 | +} |
| 9 | + |
| 10 | +type RefExpectation = Extract<CuaBrowserExpectation, { type: "ref" }>; |
| 11 | +/** Resolve a ref expectation against one structured observation. */ |
| 12 | +export type BrowserRefResolver = (expectation: RefExpectation, observation: BrowserObservation) => BrowserExpectationEvaluation; |
| 13 | + |
| 14 | +/** Browser capabilities required by the semantic wait engine. */ |
| 15 | +export interface BrowserWaitRuntime { |
| 16 | + selectTarget(tabId?: string): Promise<string>; |
| 17 | + observeTarget(targetId: string): Promise<BrowserObservation>; |
| 18 | + dialogCount(): number; |
| 19 | + targetExists(targetId: string): Promise<boolean>; |
| 20 | + /** Reserved for runtimes that expose generation checks outside observations. */ |
| 21 | + liveGeneration?(frameId: string): number; |
| 22 | + resolveRef: BrowserRefResolver; |
| 23 | + now?(): number; |
| 24 | + delay?(ms: number): Promise<void>; |
| 25 | +} |
| 26 | + |
| 27 | +/** Timeout, polling, target, and condition options for a semantic wait. */ |
| 28 | +export interface BrowserWaitOptions { |
| 29 | + expect: CuaBrowserExpectation; |
| 30 | + timeoutMs?: number; |
| 31 | + pollMs?: number; |
| 32 | + tabId?: string; |
| 33 | +} |
| 34 | + |
| 35 | +function expectationNodes(observation: BrowserObservation): AXNode[] { |
| 36 | + const nodes = [...observedNodes(observation)].map(({ node }) => node); |
| 37 | + for (const tree of [observation.tree, ...observation.stitches.values()]) { |
| 38 | + for (const node of tree.byId.values()) { |
| 39 | + const children = node.childIds ?? []; |
| 40 | + for (let index = 0; index < children.length; index += 1) { |
| 41 | + const run = staticTextRun(tree.byId, children, index); |
| 42 | + if (run) { nodes.push(run.node); index = run.end; } |
| 43 | + } |
| 44 | + } |
| 45 | + } |
| 46 | + return nodes; |
| 47 | +} |
| 48 | + |
| 49 | +/** Evaluate a semantic condition without minting refs. Unknown means the observation cannot prove the claim. */ |
| 50 | +export function evaluateBrowserExpectation( |
| 51 | + expectation: CuaBrowserExpectation, |
| 52 | + observation: BrowserObservation, |
| 53 | + baseline: BrowserObservation, |
| 54 | + resolveRef: BrowserRefResolver, |
| 55 | +): BrowserExpectationEvaluation { |
| 56 | + if ("all" in expectation || "any" in expectation) { |
| 57 | + const all = "all" in expectation; |
| 58 | + const children = (all ? expectation.all : expectation.any).map((child) => evaluateBrowserExpectation(child, observation, baseline, resolveRef)); |
| 59 | + const truth = all |
| 60 | + ? children.some((child) => child.truth === false) ? false : children.some((child) => child.truth === undefined) ? undefined : true |
| 61 | + : children.some((child) => child.truth === true) ? true : children.some((child) => child.truth === undefined) ? undefined : false; |
| 62 | + return { |
| 63 | + truth, |
| 64 | + details: children.flatMap((child) => child.details), |
| 65 | + ...(truth === undefined ? { reason: children.find((child) => child.truth === undefined)?.reason } : {}), |
| 66 | + }; |
| 67 | + } |
| 68 | + if (expectation.type === "text" || expectation.type === "role_name") { |
| 69 | + const found = expectationNodes(observation).some((node) => !node.ignored && (expectation.type === "text" |
| 70 | + ? (node.name?.value ?? "").replace(/\s+/g, " ").toLowerCase().includes(expectation.text.replace(/\s+/g, " ").toLowerCase()) |
| 71 | + : (expectation.role === undefined || node.role?.value === expectation.role) && (expectation.name === undefined || node.name?.value === expectation.name))); |
| 72 | + const expected = expectation.exists ?? true; |
| 73 | + const complete = observation.incompleteFrames.length === 0; |
| 74 | + const truth = !found && !complete ? undefined : found === expected; |
| 75 | + return { truth, details: [`${expectation.type} ${found ? "present" : "absent"}${complete ? "" : "; observation incomplete"}`], ...(!complete && !found ? { reason: "incomplete_observation" as const } : {}) }; |
| 76 | + } |
| 77 | + if (expectation.type === "url" || expectation.type === "title") { |
| 78 | + const value = observation[expectation.type]; |
| 79 | + const initial = baseline[expectation.type]; |
| 80 | + const truth = (expectation.equals === undefined || value === expectation.equals) && |
| 81 | + (expectation.contains === undefined || value.includes(expectation.contains)) && |
| 82 | + (expectation.changed === undefined || (value !== initial) === expectation.changed); |
| 83 | + return { truth, details: [`${expectation.type}=${JSON.stringify(value)}`] }; |
| 84 | + } |
| 85 | + if (expectation.type === "ref") return resolveRef(expectation, observation); |
| 86 | + return { truth: undefined, details: ["unsupported expectation"] }; |
| 87 | +} |
| 88 | + |
| 89 | +/** Poll a semantic browser condition; in-flight browser reads settle before timeout is reported. */ |
| 90 | +export async function waitForBrowserExpectation(runtime: BrowserWaitRuntime, options: BrowserWaitOptions): Promise<BrowserWaitForResult> { |
| 91 | + const now = runtime.now ?? Date.now; |
| 92 | + const sleep = runtime.delay ?? ((ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms))); |
| 93 | + const started = now(); |
| 94 | + const timeout = options.timeoutMs ?? 2_000; |
| 95 | + const poll = options.pollMs ?? 50; |
| 96 | + const remaining = () => Math.max(0, timeout - (now() - started)); |
| 97 | + const expired = () => now() - started >= timeout; |
| 98 | + let targetId: string; |
| 99 | + let baseline: BrowserObservation; |
| 100 | + try { |
| 101 | + targetId = await beforeDeadline(() => runtime.selectTarget(options.tabId), remaining(), now); |
| 102 | + if (expired()) return timedOut(started, now()); |
| 103 | + baseline = await runtime.observeTarget(targetId); |
| 104 | + } catch (error) { |
| 105 | + if (error instanceof WaitDeadlineError) return timedOut(started, now()); |
| 106 | + return failedObservation(started, now(), error); |
| 107 | + } |
| 108 | + const initial = evaluateBrowserExpectation(options.expect, baseline, baseline, runtime.resolveRef); |
| 109 | + if (initial.reason === "stale_ref" && !("any" in options.expect)) { |
| 110 | + return terminal("interrupted", "unverifiable", initial, initial, started, now(), initial.reason); |
| 111 | + } |
| 112 | + if (expired()) return timedOut(started, now(), initial, initial); |
| 113 | + if (initial.truth === true) return terminal("satisfied", "preexisting", initial, initial, started, now()); |
| 114 | + const dialogs = runtime.dialogCount(); |
| 115 | + let final = initial; |
| 116 | + while (!expired()) { |
| 117 | + await sleep(Math.min(poll, remaining())); |
| 118 | + if (expired()) break; |
| 119 | + let exists: boolean; |
| 120 | + try { exists = await beforeDeadline(() => runtime.targetExists(targetId), remaining(), now); } |
| 121 | + catch (error) { if (error instanceof WaitDeadlineError) break; return failedObservation(started, now(), error, initial, final); } |
| 122 | + if (expired()) break; |
| 123 | + if (!exists) return terminal("interrupted", "unverifiable", initial, final, started, now(), "target_detached"); |
| 124 | + let observation: BrowserObservation; |
| 125 | + try { observation = await runtime.observeTarget(targetId); } |
| 126 | + catch (error) { return failedObservation(started, now(), error, initial, final); } |
| 127 | + final = evaluateBrowserExpectation(options.expect, observation, baseline, runtime.resolveRef); |
| 128 | + if (observation.targetId !== targetId) return terminal("interrupted", "unverifiable", initial, final, started, now(), "target_changed"); |
| 129 | + if (runtime.dialogCount() > dialogs) return terminal("interrupted", "unverifiable", initial, final, started, now(), "dialog"); |
| 130 | + const crossDocumentNavigation = observation.generations.get(targetId) !== baseline.generations.get(targetId); |
| 131 | + const sameDocumentNavigation = observation.navigationEpoch !== baseline.navigationEpoch; |
| 132 | + const locationExpectation = containsLocationExpectation(options.expect); |
| 133 | + if (crossDocumentNavigation && !locationExpectation) { |
| 134 | + return terminal("interrupted", "unverifiable", initial, final, started, now(), "navigation"); |
| 135 | + } |
| 136 | + if (final.reason === "stale_ref" && !("any" in options.expect)) { |
| 137 | + return terminal("interrupted", "unverifiable", initial, final, started, now(), final.reason); |
| 138 | + } |
| 139 | + if ((crossDocumentNavigation || sameDocumentNavigation) && locationExpectation) { |
| 140 | + if (final.truth === true && !expired()) return terminal("satisfied", "newly_verified", initial, final, started, now()); |
| 141 | + continue; |
| 142 | + } |
| 143 | + if (expired()) break; |
| 144 | + if (final.truth === true) return terminal("satisfied", "newly_verified", initial, final, started, now()); |
| 145 | + if (observation.incompleteFrames.length > 0) { |
| 146 | + final = { ...final, truth: undefined, reason: final.reason ?? "incomplete_observation" }; |
| 147 | + continue; |
| 148 | + } |
| 149 | + } |
| 150 | + if (final.truth === undefined) return terminal("unverifiable", "unverifiable", initial, final, started, now(), final.reason ?? "incomplete_observation"); |
| 151 | + return terminal("timed_out", "failed", initial, final, started, now()); |
| 152 | +} |
| 153 | + |
| 154 | +class WaitDeadlineError extends Error {} |
| 155 | + |
| 156 | +async function beforeDeadline<T>(operation: () => Promise<T>, remaining: number, now: () => number): Promise<T> { |
| 157 | + if (remaining <= 0) throw new WaitDeadlineError(); |
| 158 | + const started = now(); |
| 159 | + try { |
| 160 | + const result = await operation(); |
| 161 | + if (Number.isFinite(remaining) && now() - started >= remaining) throw new WaitDeadlineError(); |
| 162 | + return result; |
| 163 | + } catch (error) { |
| 164 | + if (error instanceof WaitDeadlineError || Number.isFinite(remaining) && now() - started >= remaining) throw new WaitDeadlineError(); |
| 165 | + throw error; |
| 166 | + } |
| 167 | +} |
| 168 | + |
| 169 | +function containsLocationExpectation(expectation: CuaBrowserExpectation): boolean { |
| 170 | + if ("all" in expectation) return expectation.all.some(containsLocationExpectation); |
| 171 | + if ("any" in expectation) return expectation.any.some(containsLocationExpectation); |
| 172 | + return expectation.type === "url" || expectation.type === "title"; |
| 173 | +} |
| 174 | + |
| 175 | +function timedOut( |
| 176 | + started: number, |
| 177 | + ended: number, |
| 178 | + initial: BrowserExpectationEvidence = { truth: undefined, details: [] }, |
| 179 | + final: BrowserExpectationEvidence = initial, |
| 180 | +): BrowserWaitForResult { |
| 181 | + return terminal("timed_out", "failed", initial, final, started, ended); |
| 182 | +} |
| 183 | + |
| 184 | +function terminal(status: BrowserWaitForResult["status"], evidence: BrowserWaitForResult["evidence"], initial: BrowserExpectationEvidence, final: BrowserExpectationEvidence, started: number, ended: number, reason?: BrowserWaitReason): BrowserWaitForResult { |
| 185 | + return { status, evidence, initial, final, elapsed_ms: Math.max(0, ended - started), ...(reason ? { reason } : {}), details: [...initial.details.map((detail) => `initial: ${detail}`), ...final.details.map((detail) => `final: ${detail}`)] }; |
| 186 | +} |
| 187 | + |
| 188 | +function failedObservation(started: number, ended: number, error: unknown, initial: BrowserExpectationEvidence = { truth: undefined, details: [] }, final = initial): BrowserWaitForResult { |
| 189 | + const message = error instanceof Error ? error.message : String(error); |
| 190 | + return { ...terminal("unverifiable", "unverifiable", initial, final, started, ended, "observation_failed"), details: [...initial.details, message] }; |
| 191 | +} |
0 commit comments