Skip to content

Commit 72050e1

Browse files
committed
fix(desktop): refuse unverifiable clicks, stop menu completion on cancel, scope press_key
- Refuse coordinate input when Accessibility cannot say which app owns the point - Check cancellation before the menu-completion Return - press_key goes to the observed window or a dialog, panel or popover above it, never another document window - Only presses treat an Accessibility timeout as delivered; a value or selection change that times out fails as uncertain
1 parent fd0fa73 commit 72050e1

1 file changed

Lines changed: 18 additions & 18 deletions

File tree

‎apps/desktop/native/computer-use/ComputerUse.swift‎

Lines changed: 18 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -47,11 +47,13 @@ func axCheck(_ result: AXError) throws {
4747
guard result == .success else { throw ComputerError("accessibility_error", "Accessibility operation failed (\(result.rawValue)); observe the app again before retrying.") }
4848
}
4949
/// A press that opens a sheet, alert or modal panel holds the app's Accessibility reply until that UI
50-
/// is up, so `cannotComplete` after the messaging timeout means the request was delivered, not refused.
51-
func dispatchAX(_ element: AXUIElement, _ operation: (AXUIElement) -> AXError) throws {
50+
/// is up, so for a press `cannotComplete` after the messaging timeout means the request was delivered.
51+
/// For a value or selection change the outcome is unknown: it counts as dispatched but still fails.
52+
func dispatchAX(_ element: AXUIElement, press: Bool = false, _ operation: (AXUIElement) -> AXError) throws {
5253
AXUIElementSetMessagingTimeout(element, 2)
5354
let result = operation(element)
54-
if result == .success || result == .cannotComplete { inputDispatched = true; return }
55+
if result == .cannotComplete { inputDispatched = true; if press { return } }
56+
if result == .success { inputDispatched = true; return }
5557
try axCheck(result)
5658
}
5759
/// Some apps (Calendar on macOS 26) answer AXPress on a menu item by opening its menu with the item
@@ -64,7 +66,9 @@ func completeMenuPress(_ item: AXUIElement, pid: pid_t) async throws {
6466
let menuLevel = Int(CGWindowLevelForKey(.popUpMenuWindow))
6567
guard let entries = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]],
6668
entries.contains(where: { ($0[kCGWindowOwnerPID as String] as? NSNumber)?.int32Value == pid && ($0[kCGWindowLayer as String] as? NSNumber)?.intValue == menuLevel }) else { return }
69+
try checkCancellation()
6770
_ = AXUIElementSetAttributeValue(item, kAXSelectedAttribute as CFString, kCFBooleanTrue)
71+
try checkCancellation()
6872
try postKey(pid: pid, code: 36)
6973
}
7074
/// A menu item's keyboard shortcut in macOS notation, such as ⌥⌘N, so the agent can use real shortcuts.
@@ -405,22 +409,15 @@ func requireForegroundTarget(pid: pid_t, windowID: String, point: CGPoint, frame
405409
/// The process that receives a click at a global point. Notification Center and the Dock keep
406410
/// full-screen click-through windows above every app, so the topmost window in the window list is not
407411
/// the click target; Accessibility hit-testing follows the window server's routing. The app's own
408-
/// sheets, popovers and menus above the observed window resolve to the app itself.
412+
/// sheets, popovers and menus above the observed window resolve to the app itself. Without an answer
413+
/// the point's owner is unknown, and the caller refuses the input.
409414
func pointOwner(_ point: CGPoint) -> pid_t? {
410415
let system = AXUIElementCreateSystemWide()
411416
AXUIElementSetMessagingTimeout(system, 0.25)
412417
var hit: AXUIElement?
413418
var owner: pid_t = 0
414-
if AXUIElementCopyElementAtPosition(system, Float(point.x), Float(point.y), &hit) == .success, let hit, AXUIElementGetPid(hit, &owner) == .success { return owner }
415-
// Without a hit-test answer, take the topmost window an app could own below the Dock's level.
416-
guard let entries = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]] else { return nil }
417-
let dockLevel = Int(CGWindowLevelForKey(.dockWindow))
418-
let top = entries.first { entry in
419-
guard (entry[kCGWindowAlpha as String] as? Double ?? 0) > 0, let layer = entry[kCGWindowLayer as String] as? Int, layer < dockLevel,
420-
let bounds = entry[kCGWindowBounds as String] as? [String: Any], let rect = CGRect(dictionaryRepresentation: bounds as CFDictionary) else { return false }
421-
return rect.contains(point)
422-
}
423-
return (top?[kCGWindowOwnerPID as String] as? NSNumber)?.int32Value
419+
guard AXUIElementCopyElementAtPosition(system, Float(point.x), Float(point.y), &hit) == .success, let hit, AXUIElementGetPid(hit, &owner) == .success else { return nil }
420+
return owner
424421
}
425422
final class CoordinateDispatch {
426423
private(set) var hasDispatched = false
@@ -584,7 +581,7 @@ final class Driver {
584581
let element = try snapshot.element(p.elementId); let action = try required(p.accessibilityAction, "accessibilityAction")
585582
var names: CFArray?; try axCheck(AXUIElementCopyActionNames(element, &names))
586583
guard (names as? [String] ?? []).contains(action) else { throw ComputerError("unsupported_action", "Action was not advertised by the selected element.") }
587-
try dispatchAX(element) { AXUIElementPerformAction($0, action as CFString) }
584+
try dispatchAX(element, press: true) { AXUIElementPerformAction($0, action as CFString) }
588585
if action == kAXPressAction || action == kAXPickAction { try await completeMenuPress(element, pid: pid) }
589586
case "set_value":
590587
let element = try snapshot.element(p.elementId); let value = try required(p.value, "value")
@@ -596,7 +593,7 @@ final class Driver {
596593
let button = p.button ?? "left"; let count = p.clickCount ?? 1
597594
guard ["left", "right"].contains(button), (1...3).contains(count) else { throw ComputerError("invalid_arguments", "Invalid button or clickCount.") }
598595
if p.elementId != nil && button == "left" && count == 1, let element = try? snapshot.element(p.elementId), axActions(element).contains(kAXPressAction) {
599-
try dispatchAX(element) { AXUIElementPerformAction($0, kAXPressAction as CFString) }
596+
try dispatchAX(element, press: true) { AXUIElementPerformAction($0, kAXPressAction as CFString) }
600597
try await completeMenuPress(element, pid: pid)
601598
} else {
602599
try await coordinateAction { dispatch in
@@ -687,8 +684,11 @@ final class Driver {
687684
let windowId = try required(p.windowId, "windowId"); let frame = try windowRect(windowId, pid: pid)
688685
guard snapshot.windowFrames[windowId] == frame else { throw ComputerError("stale_window", "Observe the target window again.") }
689686
let root = AXUIElementCreateApplication(pid)
690-
// Keys go to the app's key window, which may be a sheet, panel or popover above the observed one.
691-
guard let focusedWindow = attribute(root, kAXFocusedWindowAttribute), CFGetTypeID(focusedWindow) == AXUIElementGetTypeID() else { throw ComputerError("focus_failed", "The app has no focused window to receive keys; click a window or call activate_app, then observe again.") }
687+
// Keys go to the app's key window: the observed window, or a dialog, panel or popover above it,
688+
// but never another document window the snapshot did not observe.
689+
guard let rawWindow = attribute(root, kAXFocusedWindowAttribute), CFGetTypeID(rawWindow) == AXUIElementGetTypeID() else { throw ComputerError("focus_failed", "The app has no focused window to receive keys; click a window or call activate_app, then observe again.") }
690+
let focusedWindow = unsafeBitCast(rawWindow, to: AXUIElement.self)
691+
guard elementRect(focusedWindow) == frame || attribute(focusedWindow, kAXSubroleAttribute) as? String != kAXStandardWindowSubrole else { throw ComputerError("focus_failed", "Another document window has keyboard focus; observe that window and target it instead.") }
692692
if let focused = attribute(root, kAXFocusedUIElementAttribute), CFGetTypeID(focused) == AXUIElementGetTypeID() { try requireNonSecure(unsafeBitCast(focused, to: AXUIElement.self)) }
693693
let (code, flags) = try parseKey(required(p.key, "key"))
694694
var selectedAll = false

0 commit comments

Comments
 (0)