Skip to content

Commit bc55309

Browse files
committed
fix(desktop): compare the observed window's identity, not only its bounds
Each observed windowId keeps its Accessibility window, and click, key and editor checks require that exact window at its observed frame (or a dialog or floating panel), so a second document window with identical bounds is refused.
1 parent 299812f commit bc55309

1 file changed

Lines changed: 30 additions & 16 deletions

File tree

‎apps/desktop/native/computer-use/ComputerUse.swift‎

Lines changed: 30 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -221,6 +221,9 @@ final class Snapshot {
221221
let created = Date()
222222
var elements: [String: AXUIElement] = [:]
223223
var windowFrames: [String: CGRect] = [:]
224+
/// The Accessibility window behind each observed windowId, so input checks compare identity, not
225+
/// just bounds that two windows can share.
226+
var windowElements: [String: AXUIElement] = [:]
224227
var nodes: [[String: Any]] = []
225228
var truncated = false
226229
init(app: NSRunningApplication) { pid = app.processIdentifier; launchDate = app.launchDate }
@@ -306,6 +309,9 @@ final class Snapshot {
306309
}
307310
if queue.hasPending || queue.truncated { truncated = true }
308311
}
312+
func observedWindows() -> [(element: AXUIElement, frame: CGRect)] {
313+
windowFrames.compactMap { id, frame in windowElements[id].map { ($0, frame) } }
314+
}
309315
func element(_ id: String?) throws -> AXUIElement {
310316
let id = try required(id, "elementId")
311317
guard let element = elements[id], attribute(element, kAXRoleAttribute) != nil else { throw ComputerError("stale_element", "Element is stale; observe the app again.") }
@@ -403,7 +409,7 @@ func mapPoint(frame: CGRect, x: Double, y: Double) throws -> CGPoint {
403409
guard x.isFinite, y.isFinite, x >= 0, y >= 0, x < frame.width, y < frame.height else { throw ComputerError("invalid_coordinates", "Coordinates must lie within the selected window in points.") }
404410
return CGPoint(x: frame.minX + x, y: frame.minY + y)
405411
}
406-
func requireForegroundTarget(pid: pid_t, windowID: String, point: CGPoint, frame: CGRect) throws {
412+
func requireForegroundTarget(pid: pid_t, windowID: String, window: AXUIElement, point: CGPoint, frame: CGRect) throws {
407413
guard NSWorkspace.shared.frontmostApplication?.processIdentifier == pid else { throw ComputerError("foreground_required", "Coordinate mouse input requires the target app in front. Call activate_app, then observe again.") }
408414
guard try windowRect(windowID, pid: pid) == frame else { throw ComputerError("stale_window", "Window moved during input; observe it again.") }
409415
guard let hit = elementAtPoint(point) else { throw ComputerError("window_occluded", "Cannot confirm which window receives this point; observe again before retrying.") }
@@ -413,7 +419,7 @@ func requireForegroundTarget(pid: pid_t, windowID: String, point: CGPoint, frame
413419
// to one of the app's dialogs or floating panels above it, or to an open menu, which has no window.
414420
// Anything unconfirmed is refused.
415421
if let rawWindow = attribute(hit, kAXWindowAttribute), CFGetTypeID(rawWindow) == AXUIElementGetTypeID() {
416-
guard acceptsInput(unsafeBitCast(rawWindow, to: AXUIElement.self), observed: [frame]) else {
422+
guard acceptsInput(unsafeBitCast(rawWindow, to: AXUIElement.self), observed: [(window, frame)]) else {
417423
throw ComputerError("window_occluded", "Another of this app's windows covers the target point; observe that window and target it instead.")
418424
}
419425
} else if ![kAXMenuRole, kAXMenuItemRole, kAXMenuBarRole, kAXMenuBarItemRole].contains(attribute(hit, kAXRoleAttribute) as? String ?? "") {
@@ -423,8 +429,8 @@ func requireForegroundTarget(pid: pid_t, windowID: String, point: CGPoint, frame
423429
/// Whether input aimed at an observed window may go to `window`: the observed window itself, or one of
424430
/// the app's dialogs or floating panels above it. Another document window, or one whose kind cannot be
425431
/// read, is refused.
426-
func acceptsInput(_ window: AXUIElement, observed: [CGRect]) -> Bool {
427-
if let frame = elementRect(window), observed.contains(frame) { return true }
432+
func acceptsInput(_ window: AXUIElement, observed: [(element: AXUIElement, frame: CGRect)]) -> Bool {
433+
if observed.contains(where: { CFEqual($0.element, window) && elementRect(window) == $0.frame }) { return true }
428434
let auxiliary: Set<String> = [kAXDialogSubrole, kAXSystemDialogSubrole, kAXFloatingWindowSubrole, kAXSystemFloatingWindowSubrole]
429435
return (attribute(window, kAXSubroleAttribute) as? String).map(auxiliary.contains) ?? false
430436
}
@@ -452,9 +458,12 @@ func coordinateAction(_ operation: (CoordinateDispatch) async throws -> Void) as
452458
do { try await operation(dispatch) }
453459
catch let error as ComputerError { throw dispatch.classify(error) }
454460
}
455-
func postMouse(pid: pid_t, type: CGEventType, point: CGPoint, button: CGMouseButton, count: Int = 1, windowID: String? = nil, frame: CGRect? = nil, release: Bool = false, dispatch: CoordinateDispatch) throws {
461+
func postMouse(pid: pid_t, type: CGEventType, point: CGPoint, button: CGMouseButton, count: Int = 1, windowID: String? = nil, window: AXUIElement? = nil, frame: CGRect? = nil, release: Bool = false, dispatch: CoordinateDispatch) throws {
456462
guard let windowID, let frame else { throw ComputerError("invalid_window", "Mouse input requires an observed app window.") }
457-
if !release { try requireForegroundTarget(pid: pid, windowID: windowID, point: point, frame: frame) }
463+
if !release {
464+
guard let window else { throw ComputerError("invalid_window", "Mouse input requires an observed app window.") }
465+
try requireForegroundTarget(pid: pid, windowID: windowID, window: window, point: point, frame: frame)
466+
}
458467
guard let source = CGEventSource(stateID: .privateState), let event = CGEvent(mouseEventSource: source, mouseType: type, mouseCursorPosition: point, mouseButton: button) else { throw ComputerError("input_failed", "Could not create mouse event.") }
459468
event.setIntegerValueField(.mouseEventClickState, value: Int64(count))
460469
dispatch.willDispatch()
@@ -544,10 +553,12 @@ final class Driver {
544553
let app = try await appFor(p.bundleId, launch: true); let snapshot = Snapshot(app: app)
545554
let preferredFrame = try p.windowId.map { try windowRect($0, pid: app.processIdentifier) }
546555
try await snapshot.read(preferredWindowFrame: preferredFrame)
547-
let axWindowFrames = snapshot.elements.values.filter { attribute($0, kAXRoleAttribute) as? String == kAXWindowRole }.compactMap(elementRect)
556+
let axWindows = snapshot.elements.values.filter { attribute($0, kAXRoleAttribute) as? String == kAXWindowRole }
557+
let axWindowFrames = axWindows.compactMap(elementRect)
548558
let windows = windowsFor(app.processIdentifier).prefix(100).compactMap { entry -> [String: Any]? in
549559
guard let id = entry[kCGWindowNumber as String] as? NSNumber, let bounds = entry[kCGWindowBounds as String] as? [String: Any], let rect = CGRect(dictionaryRepresentation: bounds as CFDictionary), rect.width > 0, rect.height > 0, axWindowFrames.contains(rect) else { return nil }
550560
snapshot.windowFrames[id.stringValue] = rect
561+
snapshot.windowElements[id.stringValue] = axWindows.first { elementRect($0) == rect }
551562
var result: [String: Any] = ["windowId": id.stringValue, "title": entry[kCGWindowName as String] as? String ?? ""]
552563
if let title = result["title"] as? String { result["title"] = String(title.prefix(4096)) }
553564
result.merge(rectJSON(rect)) { _, new in new }; return result
@@ -622,8 +633,8 @@ final class Driver {
622633
let frame = try required(snapshot.windowFrames[windowID], "window frame")
623634
for click in 1...count {
624635
try checkCancellation()
625-
try postMouse(pid: pid, type: button == "right" ? .rightMouseDown : .leftMouseDown, point: position, button: mouse, count: click, windowID: windowID, frame: frame, dispatch: dispatch)
626-
try postMouse(pid: pid, type: button == "right" ? .rightMouseUp : .leftMouseUp, point: position, button: mouse, count: click, windowID: windowID, frame: frame, release: true, dispatch: dispatch)
636+
try postMouse(pid: pid, type: button == "right" ? .rightMouseDown : .leftMouseDown, point: position, button: mouse, count: click, windowID: windowID, window: snapshot.windowElements[windowID], frame: frame, dispatch: dispatch)
637+
try postMouse(pid: pid, type: button == "right" ? .rightMouseUp : .leftMouseUp, point: position, button: mouse, count: click, windowID: windowID, window: snapshot.windowElements[windowID], frame: frame, release: true, dispatch: dispatch)
627638
}
628639
}
629640
}
@@ -656,12 +667,12 @@ final class Driver {
656667
}
657668
return false
658669
}
659-
let observedFrames = Array(snapshot.windowFrames.values)
670+
let observedWindows = snapshot.observedWindows()
660671
func validateFocus() throws {
661672
try checkCancellation(); try requireNonSecure(element)
662673
// The editor must still sit in an observed window (or a dialog above it) at the observed frame.
663674
guard let rawWindow = attribute(element, kAXWindowAttribute), CFGetTypeID(rawWindow) == AXUIElementGetTypeID(),
664-
acceptsInput(unsafeBitCast(rawWindow, to: AXUIElement.self), observed: observedFrames) else {
675+
acceptsInput(unsafeBitCast(rawWindow, to: AXUIElement.self), observed: observedWindows) else {
665676
throw ComputerError("stale_window", "The editor's window moved or is not an observed window; observe again.")
666677
}
667678
guard let current = attribute(root, kAXFocusedUIElementAttribute), holdsFocus(current) else { throw ComputerError("focus_changed", "The exact editor is no longer focused; remaining input was not dispatched. Observe again.") }
@@ -718,7 +729,10 @@ final class Driver {
718729
// but never another document window the snapshot did not observe.
719730
guard let rawWindow = attribute(root, kAXFocusedWindowAttribute), CFGetTypeID(rawWindow) == AXUIElementGetTypeID() else { throw ComputerError("focus_failed", "The app has no focused window to receive keys; click a window or call activate_app, then observe again.") }
720731
let focusedWindow = unsafeBitCast(rawWindow, to: AXUIElement.self)
721-
guard acceptsInput(focusedWindow, observed: frame.map { [$0] } ?? Array(snapshot.windowFrames.values)) else { throw ComputerError("focus_failed", "Another document window has keyboard focus; observe that window and target it instead.") }
732+
let targets: [(element: AXUIElement, frame: CGRect)]
733+
if let windowId = p.windowId, let frame { targets = snapshot.windowElements[windowId].map { [($0, frame)] } ?? [] }
734+
else { targets = snapshot.observedWindows() }
735+
guard acceptsInput(focusedWindow, observed: targets) else { throw ComputerError("focus_failed", "Another document window has keyboard focus; observe that window and target it instead.") }
722736
if let focused = attribute(root, kAXFocusedUIElementAttribute), CFGetTypeID(focused) == AXUIElementGetTypeID() { try requireNonSecure(unsafeBitCast(focused, to: AXUIElement.self)) }
723737
let (code, flags) = try parseKey(required(p.key, "key"))
724738
var selectedAll = false
@@ -738,22 +752,22 @@ final class Driver {
738752
guard let event = CGEvent(scrollWheelEvent2Source: CGEventSource(stateID: .privateState), units: .pixel, wheelCount: 2, wheel1: Int32(-dy), wheel2: Int32(-dx), wheel3: 0) else { throw ComputerError("input_failed", "Could not create scroll event.") }
739753
let windowID = try required(p.windowId ?? snapshot.windowFrames.first(where: { $0.value.contains(position) })?.key, "windowId")
740754
let frame = try required(snapshot.windowFrames[windowID], "window frame")
741-
try requireForegroundTarget(pid: pid, windowID: windowID, point: position, frame: frame)
755+
try requireForegroundTarget(pid: pid, windowID: windowID, window: try required(snapshot.windowElements[windowID], "observed window"), point: position, frame: frame)
742756
event.location = position; dispatch.willDispatch(); event.post(tap: .cghidEventTap)
743757
}
744758
case "drag":
745759
try await coordinateAction { dispatch in
746760
let start = try point(p, app: app, snapshot: snapshot); let end = try point(p, app: app, snapshot: snapshot, end: true)
747761
let windowID = try required(p.windowId, "windowId"); let frame = try required(snapshot.windowFrames[windowID], "window frame")
748-
try postMouse(pid: pid, type: .leftMouseDown, point: start, button: .left, windowID: windowID, frame: frame, dispatch: dispatch)
762+
try postMouse(pid: pid, type: .leftMouseDown, point: start, button: .left, windowID: windowID, window: snapshot.windowElements[windowID], frame: frame, dispatch: dispatch)
749763
var lastPoint = start
750-
defer { try? postMouse(pid: pid, type: .leftMouseUp, point: lastPoint, button: .left, windowID: windowID, frame: frame, release: true, dispatch: dispatch) }
764+
defer { try? postMouse(pid: pid, type: .leftMouseUp, point: lastPoint, button: .left, windowID: windowID, window: snapshot.windowElements[windowID], frame: frame, release: true, dispatch: dispatch) }
751765
try await Task.sleep(for: .milliseconds(10))
752766
for step in 1...10 {
753767
try checkCancellation()
754768
let amount = CGFloat(step) / 10
755769
let position = CGPoint(x: start.x + (end.x - start.x) * amount, y: start.y + (end.y - start.y) * amount)
756-
try postMouse(pid: pid, type: .leftMouseDragged, point: position, button: .left, windowID: windowID, frame: frame, dispatch: dispatch)
770+
try postMouse(pid: pid, type: .leftMouseDragged, point: position, button: .left, windowID: windowID, window: snapshot.windowElements[windowID], frame: frame, dispatch: dispatch)
757771
lastPoint = position
758772
try await Task.sleep(for: .milliseconds(10))
759773
}

0 commit comments

Comments
 (0)