Skip to content

Commit e569e49

Browse files
waleedlatif1waleedlatif1
committed
fix(desktop): keep coordinate input on the selected window and confirm targeted focus
- Coordinate clicks and scrolls without a windowId use the snapshot's selected window through every input check; element targets keep their own window - press_key with an elementId sends the key only once that element holds focus - The service observes the selected window after a call without one - A scroll needs at least one delta, matching the worker contract
1 parent 2caf24f commit e569e49

4 files changed

Lines changed: 40 additions & 25 deletions

File tree

‎apps/desktop/native/computer-use/ComputerUse.swift‎

Lines changed: 25 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,18 @@ func requireNonSecure(_ element: AXUIElement) throws {
106106
func rectJSON(_ rect: CGRect) -> [String: Double] {
107107
["x": rect.minX, "y": rect.minY, "width": rect.width, "height": rect.height]
108108
}
109+
/// Whether keyboard focus is on `element` or an editor inside it, such as a combo box's field.
110+
func holdsFocus(_ focused: CFTypeRef, _ element: AXUIElement) -> Bool {
111+
guard CFGetTypeID(focused) == AXUIElementGetTypeID() else { return false }
112+
var current: AXUIElement? = unsafeBitCast(focused, to: AXUIElement.self)
113+
for _ in 0..<8 {
114+
guard let candidate = current else { return false }
115+
if CFEqual(candidate, element) { return true }
116+
guard let parent = attribute(candidate, kAXParentAttribute), CFGetTypeID(parent) == AXUIElementGetTypeID() else { return false }
117+
current = unsafeBitCast(parent, to: AXUIElement.self)
118+
}
119+
return false
120+
}
109121
func axActions(_ element: AXUIElement) -> [String] {
110122
var names: CFArray?
111123
return AXUIElementCopyActionNames(element, &names) == .success ? names as? [String] ?? [] : []
@@ -656,7 +668,8 @@ final class Driver {
656668
} else {
657669
try await coordinateAction { dispatch in
658670
let position = try point(p, app: app, snapshot: snapshot); let mouse: CGMouseButton = button == "right" ? .right : .left
659-
let windowID = try required(p.windowId ?? snapshot.windowFrames.first(where: { $0.value.contains(position) })?.key, "windowId")
671+
// Coordinates are window-local to the requested (or selected) window; an element target uses its own.
672+
let windowID = try required(p.windowId ?? (p.elementId == nil ? snapshot.selectedWindowId : snapshot.windowFrames.first(where: { $0.value.contains(position) })?.key), "windowId")
660673
let frame = try required(snapshot.windowFrames[windowID], "window frame")
661674
for click in 1...count {
662675
try checkCancellation()
@@ -683,18 +696,6 @@ final class Driver {
683696
try axCheck(AXUIElementSetAttributeValue(element, kAXFocusedAttribute as CFString, kCFBooleanTrue))
684697
try await Task.sleep(for: .milliseconds(50))
685698
}
686-
/// Focus may land on the element itself or on an editor inside it, such as a combo box's field.
687-
func holdsFocus(_ focused: CFTypeRef) -> Bool {
688-
guard CFGetTypeID(focused) == AXUIElementGetTypeID() else { return false }
689-
var current: AXUIElement? = unsafeBitCast(focused, to: AXUIElement.self)
690-
for _ in 0..<8 {
691-
guard let candidate = current else { return false }
692-
if CFEqual(candidate, element) { return true }
693-
guard let parent = attribute(candidate, kAXParentAttribute), CFGetTypeID(parent) == AXUIElementGetTypeID() else { return false }
694-
current = unsafeBitCast(parent, to: AXUIElement.self)
695-
}
696-
return false
697-
}
698699
let observedWindows = snapshot.observedWindows()
699700
func validateFocus() throws {
700701
try checkCancellation(); try requireNonSecure(element)
@@ -703,7 +704,7 @@ final class Driver {
703704
acceptsInput(unsafeBitCast(rawWindow, to: AXUIElement.self), observed: observedWindows) else {
704705
throw ComputerError("stale_window", "The editor's window moved or is not an observed window; observe again.")
705706
}
706-
guard let current = attribute(root, kAXFocusedUIElementAttribute), holdsFocus(current) else { throw ComputerError("focus_changed", "The exact editor is no longer focused; remaining input was not dispatched. Observe again.") }
707+
guard let current = attribute(root, kAXFocusedUIElementAttribute), holdsFocus(current, element) else { throw ComputerError("focus_changed", "The exact editor is no longer focused; remaining input was not dispatched. Observe again.") }
707708
if let focused = attribute(root, kAXFocusedUIElementAttribute), CFGetTypeID(focused) == AXUIElementGetTypeID() { try requireNonSecure(unsafeBitCast(focused, to: AXUIElement.self)) }
708709
}
709710
// AX focus requests and app activation may settle asynchronously. Wait without
@@ -756,10 +757,17 @@ final class Driver {
756757
// A targeted key first moves keyboard focus to that element, as clicking into it would.
757758
if let elementId = p.elementId {
758759
let element = try snapshot.element(elementId)
759-
if attribute(root, kAXFocusedUIElementAttribute).map({ CFEqual($0, element) }) != true {
760+
if attribute(root, kAXFocusedUIElementAttribute).map({ holdsFocus($0, element) }) != true {
760761
try axCheck(AXUIElementSetAttributeValue(element, kAXFocusedAttribute as CFString, kCFBooleanTrue))
762+
}
763+
// Focus can settle asynchronously; the key is sent only once this element holds it.
764+
var focused = false
765+
for _ in 0..<10 {
766+
try checkCancellation()
767+
if attribute(root, kAXFocusedUIElementAttribute).map({ holdsFocus($0, element) }) == true { focused = true; break }
761768
try await Task.sleep(for: .milliseconds(50))
762769
}
770+
guard focused else { throw ComputerError("focus_changed", "The element did not take keyboard focus; no key was sent. Observe again.") }
763771
}
764772
// Keys go to the app's key window: the observed window, or a dialog, panel or popover above it,
765773
// but never another document window the snapshot did not observe.
@@ -786,7 +794,8 @@ final class Driver {
786794
let position = try point(p, app: app, snapshot: snapshot); let dx = p.deltaX ?? 0; let dy = p.deltaY ?? 0
787795
guard dx.isFinite, dy.isFinite, abs(dx) <= 10000, abs(dy) <= 10000 else { throw ComputerError("invalid_arguments", "Scroll deltas exceed bounds.") }
788796
guard let event = CGEvent(scrollWheelEvent2Source: CGEventSource(stateID: .privateState), units: .pixel, wheelCount: 2, wheel1: Int32(-dy), wheel2: Int32(-dx), wheel3: 0) else { throw ComputerError("input_failed", "Could not create scroll event.") }
789-
let windowID = try required(p.windowId ?? snapshot.windowFrames.first(where: { $0.value.contains(position) })?.key, "windowId")
797+
// Coordinates are window-local to the requested (or selected) window; an element target uses its own.
798+
let windowID = try required(p.windowId ?? (p.elementId == nil ? snapshot.selectedWindowId : snapshot.windowFrames.first(where: { $0.value.contains(position) })?.key), "windowId")
790799
let frame = try required(snapshot.windowFrames[windowID], "window frame")
791800
try requireForegroundTarget(pid: pid, windowID: windowID, window: try required(snapshot.windowElements[windowID], "observed window"), point: position, frame: frame)
792801
event.location = position; dispatch.willDispatch(); event.post(tap: .cghidEventTap)

‎apps/desktop/src/main/computer-use/service.ts‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -228,12 +228,12 @@ export class ComputerUseService {
228228
}
229229
this.check(work)
230230
const owner = 'snapshotId' in input ? this.snapshots.get(input.snapshotId) : undefined
231+
const elementId = 'elementId' in input ? input.elementId : undefined
232+
/** Calls without a window act on the snapshot's selected window, or the element's own. */
231233
const windowId =
232-
'windowId' in input
233-
? input.windowId
234-
: 'elementId' in input
235-
? (owner?.elementWindows.get(input.elementId) ?? owner?.windowId)
236-
: undefined
234+
('windowId' in input ? input.windowId : undefined) ??
235+
(elementId ? owner?.elementWindows.get(elementId) : undefined) ??
236+
owner?.windowId
237237
if ('snapshotId' in input) this.snapshots.delete(input.snapshotId)
238238
/** Mutations accept includeScreenshot as shorthand for observeAfter.includeScreenshot. */
239239
const isMutation = input.action !== 'get_app_state'

‎apps/sim/lib/mothership/generated/computer-use.ts‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,9 @@ const ScrollOffsets = {
6161
.describe("Vertical scroll distance in points; positive scrolls down, negative up. Defaults to 0."),
6262
};
6363

64+
const hasScrollDistance = (input: { deltaX?: number; deltaY?: number }) =>
65+
input.deltaX !== undefined || input.deltaY !== undefined;
66+
6467
/** Strict branches require an observed, unambiguous target before native input. */
6568
export const ComputerUseSchema = z.union([
6669
z.strictObject({ action: z.literal("status") }),
@@ -94,8 +97,8 @@ export const ComputerUseSchema = z.union([
9497
elementId: ElementId.optional().describe("Focus this element from the snapshot before pressing the key."),
9598
key: z.string().min(1).max(128).describe("One key or chord, such as Enter, Tab, Escape, or Cmd+A."),
9699
}),
97-
z.strictObject({ action: z.literal("scroll"), ...ElementTarget, ...ObserveAfter, ...ScrollOffsets }),
98-
z.strictObject({ action: z.literal("scroll"), ...PointTarget, ...ObserveAfter, ...ScrollOffsets }),
100+
z.strictObject({ action: z.literal("scroll"), ...ElementTarget, ...ObserveAfter, ...ScrollOffsets }).refine(hasScrollDistance, "Scroll needs deltaX or deltaY."),
101+
z.strictObject({ action: z.literal("scroll"), ...PointTarget, ...ObserveAfter, ...ScrollOffsets }).refine(hasScrollDistance, "Scroll needs deltaX or deltaY."),
99102
z.strictObject({
100103
action: z.literal("drag"),
101104
...PointTarget,

‎packages/desktop-bridge/src/computer-use.generated.ts‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,9 @@ const ScrollOffsets = {
6161
.describe("Vertical scroll distance in points; positive scrolls down, negative up. Defaults to 0."),
6262
};
6363

64+
const hasScrollDistance = (input: { deltaX?: number; deltaY?: number }) =>
65+
input.deltaX !== undefined || input.deltaY !== undefined;
66+
6467
/** Strict branches require an observed, unambiguous target before native input. */
6568
export const ComputerUseSchema = z.union([
6669
z.strictObject({ action: z.literal("status") }),
@@ -94,8 +97,8 @@ export const ComputerUseSchema = z.union([
9497
elementId: ElementId.optional().describe("Focus this element from the snapshot before pressing the key."),
9598
key: z.string().min(1).max(128).describe("One key or chord, such as Enter, Tab, Escape, or Cmd+A."),
9699
}),
97-
z.strictObject({ action: z.literal("scroll"), ...ElementTarget, ...ObserveAfter, ...ScrollOffsets }),
98-
z.strictObject({ action: z.literal("scroll"), ...PointTarget, ...ObserveAfter, ...ScrollOffsets }),
100+
z.strictObject({ action: z.literal("scroll"), ...ElementTarget, ...ObserveAfter, ...ScrollOffsets }).refine(hasScrollDistance, "Scroll needs deltaX or deltaY."),
101+
z.strictObject({ action: z.literal("scroll"), ...PointTarget, ...ObserveAfter, ...ScrollOffsets }).refine(hasScrollDistance, "Scroll needs deltaX or deltaY."),
99102
z.strictObject({
100103
action: z.literal("drag"),
101104
...PointTarget,

0 commit comments

Comments
 (0)