@@ -221,6 +221,9 @@ final class Snapshot {
221221 let created = Date ( )
222222 var elements : [ String : AXUIElement ] = [ : ]
223223 var windowFrames : [ String : CGRect ] = [ : ]
224+ /// The Accessibility window behind each observed windowId, so input checks compare identity, not
225+ /// just bounds that two windows can share.
226+ var windowElements : [ String : AXUIElement ] = [ : ]
224227 var nodes : [ [ String : Any ] ] = [ ]
225228 var truncated = false
226229 init ( app : NSRunningApplication ) { pid = app . processIdentifier ; launchDate = app . launchDate }
@@ -306,6 +309,9 @@ final class Snapshot {
306309 }
307310 if queue . hasPending || queue . truncated { truncated = true }
308311 }
312+ func observedWindows ( ) -> [ ( element : AXUIElement , frame : CGRect ) ] {
313+ windowFrames . compactMap { id , frame in windowElements [ id ] . map { ( $0 , frame ) } }
314+ }
309315 func element ( _ id : String ? ) throws -> AXUIElement {
310316 let id = try required ( id , " elementId " )
311317 guard let element = elements [ id ] , attribute ( element , kAXRoleAttribute ) != nil else { throw ComputerError ( " stale_element " , " Element is stale; observe the app again. " ) }
@@ -403,7 +409,7 @@ func mapPoint(frame: CGRect, x: Double, y: Double) throws -> CGPoint {
403409 guard x . isFinite , y . isFinite , x >= 0 , y >= 0 , x < frame . width , y < frame . height else { throw ComputerError ( " invalid_coordinates " , " Coordinates must lie within the selected window in points. " ) }
404410 return CGPoint ( x : frame . minX + x , y : frame . minY + y )
405411}
406- func requireForegroundTarget ( pid : pid_t , windowID : String , point : CGPoint , frame : CGRect ) throws {
412+ func requireForegroundTarget ( pid : pid_t , windowID : String , window : AXUIElement , point : CGPoint , frame : CGRect ) throws {
407413 guard NSWorkspace . shared . frontmostApplication ? . processIdentifier == pid else { throw ComputerError ( " foreground_required " , " Coordinate mouse input requires the target app in front. Call activate_app, then observe again. " ) }
408414 guard try windowRect ( windowID , pid : pid ) == frame else { throw ComputerError ( " stale_window " , " Window moved during input; observe it again. " ) }
409415 guard let hit = elementAtPoint ( point ) else { throw ComputerError ( " window_occluded " , " Cannot confirm which window receives this point; observe again before retrying. " ) }
@@ -413,7 +419,7 @@ func requireForegroundTarget(pid: pid_t, windowID: String, point: CGPoint, frame
413419 // to one of the app's dialogs or floating panels above it, or to an open menu, which has no window.
414420 // Anything unconfirmed is refused.
415421 if let rawWindow = attribute ( hit , kAXWindowAttribute ) , CFGetTypeID ( rawWindow ) == AXUIElementGetTypeID ( ) {
416- guard acceptsInput ( unsafeBitCast ( rawWindow , to : AXUIElement . self ) , observed : [ frame ] ) else {
422+ guard acceptsInput ( unsafeBitCast ( rawWindow , to : AXUIElement . self ) , observed : [ ( window , frame ) ] ) else {
417423 throw ComputerError ( " window_occluded " , " Another of this app's windows covers the target point; observe that window and target it instead. " )
418424 }
419425 } else if ! [ kAXMenuRole , kAXMenuItemRole , kAXMenuBarRole , kAXMenuBarItemRole ] . contains ( attribute ( hit , kAXRoleAttribute ) as? String ?? " " ) {
@@ -423,8 +429,8 @@ func requireForegroundTarget(pid: pid_t, windowID: String, point: CGPoint, frame
423429/// Whether input aimed at an observed window may go to `window`: the observed window itself, or one of
424430/// the app's dialogs or floating panels above it. Another document window, or one whose kind cannot be
425431/// read, is refused.
426- func acceptsInput ( _ window : AXUIElement , observed : [ CGRect ] ) -> Bool {
427- if let frame = elementRect ( window ) , observed . contains ( frame ) { return true }
432+ func acceptsInput ( _ window : AXUIElement , observed : [ ( element : AXUIElement , frame : CGRect ) ] ) -> Bool {
433+ if observed . contains ( where : { CFEqual ( $0 . element , window ) && elementRect ( window ) == $0 . frame } ) { return true }
428434 let auxiliary : Set < String > = [ kAXDialogSubrole , kAXSystemDialogSubrole , kAXFloatingWindowSubrole , kAXSystemFloatingWindowSubrole ]
429435 return ( attribute ( window , kAXSubroleAttribute ) as? String ) . map ( auxiliary . contains ) ?? false
430436}
@@ -452,9 +458,12 @@ func coordinateAction(_ operation: (CoordinateDispatch) async throws -> Void) as
452458 do { try await operation ( dispatch ) }
453459 catch let error as ComputerError { throw dispatch . classify ( error ) }
454460}
455- func postMouse ( pid : pid_t , type : CGEventType , point : CGPoint , button : CGMouseButton , count : Int = 1 , windowID : String ? = nil , frame : CGRect ? = nil , release : Bool = false , dispatch : CoordinateDispatch ) throws {
461+ func postMouse ( pid : pid_t , type : CGEventType , point : CGPoint , button : CGMouseButton , count : Int = 1 , windowID : String ? = nil , window : AXUIElement ? = nil , frame : CGRect ? = nil , release : Bool = false , dispatch : CoordinateDispatch ) throws {
456462 guard let windowID , let frame else { throw ComputerError ( " invalid_window " , " Mouse input requires an observed app window. " ) }
457- if ! release { try requireForegroundTarget ( pid : pid , windowID : windowID , point : point , frame : frame ) }
463+ if ! release {
464+ guard let window else { throw ComputerError ( " invalid_window " , " Mouse input requires an observed app window. " ) }
465+ try requireForegroundTarget ( pid : pid , windowID : windowID , window : window , point : point , frame : frame )
466+ }
458467 guard let source = CGEventSource ( stateID : . privateState ) , let event = CGEvent ( mouseEventSource : source , mouseType : type , mouseCursorPosition : point , mouseButton : button ) else { throw ComputerError ( " input_failed " , " Could not create mouse event. " ) }
459468 event . setIntegerValueField ( . mouseEventClickState , value : Int64 ( count ) )
460469 dispatch . willDispatch ( )
@@ -544,10 +553,12 @@ final class Driver {
544553 let app = try await appFor ( p . bundleId , launch : true ) ; let snapshot = Snapshot ( app : app )
545554 let preferredFrame = try p . windowId . map { try windowRect ( $0 , pid : app . processIdentifier ) }
546555 try await snapshot . read ( preferredWindowFrame : preferredFrame )
547- let axWindowFrames = snapshot . elements . values . filter { attribute ( $0 , kAXRoleAttribute ) as? String == kAXWindowRole } . compactMap ( elementRect )
556+ let axWindows = snapshot . elements . values . filter { attribute ( $0 , kAXRoleAttribute ) as? String == kAXWindowRole }
557+ let axWindowFrames = axWindows . compactMap ( elementRect )
548558 let windows = windowsFor ( app . processIdentifier ) . prefix ( 100 ) . compactMap { entry -> [ String : Any ] ? in
549559 guard let id = entry [ kCGWindowNumber as String ] as? NSNumber , let bounds = entry [ kCGWindowBounds as String ] as? [ String : Any ] , let rect = CGRect ( dictionaryRepresentation : bounds as CFDictionary ) , rect . width > 0 , rect . height > 0 , axWindowFrames . contains ( rect ) else { return nil }
550560 snapshot . windowFrames [ id . stringValue ] = rect
561+ snapshot . windowElements [ id . stringValue ] = axWindows . first { elementRect ( $0 ) == rect }
551562 var result : [ String : Any ] = [ " windowId " : id . stringValue , " title " : entry [ kCGWindowName as String ] as? String ?? " " ]
552563 if let title = result [ " title " ] as? String { result [ " title " ] = String ( title . prefix ( 4096 ) ) }
553564 result . merge ( rectJSON ( rect ) ) { _ , new in new } ; return result
@@ -622,8 +633,8 @@ final class Driver {
622633 let frame = try required ( snapshot . windowFrames [ windowID ] , " window frame " )
623634 for click in 1 ... count {
624635 try checkCancellation ( )
625- try postMouse ( pid : pid , type : button == " right " ? . rightMouseDown : . leftMouseDown , point : position , button : mouse , count : click , windowID : windowID , frame : frame , dispatch : dispatch )
626- try postMouse ( pid : pid , type : button == " right " ? . rightMouseUp : . leftMouseUp , point : position , button : mouse , count : click , windowID : windowID , frame : frame , release : true , dispatch : dispatch )
636+ try postMouse ( pid : pid , type : button == " right " ? . rightMouseDown : . leftMouseDown , point : position , button : mouse , count : click , windowID : windowID , window : snapshot . windowElements [ windowID ] , frame : frame , dispatch : dispatch )
637+ try postMouse ( pid : pid , type : button == " right " ? . rightMouseUp : . leftMouseUp , point : position , button : mouse , count : click , windowID : windowID , window : snapshot . windowElements [ windowID ] , frame : frame , release : true , dispatch : dispatch )
627638 }
628639 }
629640 }
@@ -656,12 +667,12 @@ final class Driver {
656667 }
657668 return false
658669 }
659- let observedFrames = Array ( snapshot . windowFrames . values )
670+ let observedWindows = snapshot . observedWindows ( )
660671 func validateFocus ( ) throws {
661672 try checkCancellation ( ) ; try requireNonSecure ( element )
662673 // The editor must still sit in an observed window (or a dialog above it) at the observed frame.
663674 guard let rawWindow = attribute ( element , kAXWindowAttribute ) , CFGetTypeID ( rawWindow ) == AXUIElementGetTypeID ( ) ,
664- acceptsInput ( unsafeBitCast ( rawWindow , to : AXUIElement . self ) , observed : observedFrames ) else {
675+ acceptsInput ( unsafeBitCast ( rawWindow , to : AXUIElement . self ) , observed : observedWindows ) else {
665676 throw ComputerError ( " stale_window " , " The editor's window moved or is not an observed window; observe again. " )
666677 }
667678 guard let current = attribute ( root , kAXFocusedUIElementAttribute ) , holdsFocus ( current ) else { throw ComputerError ( " focus_changed " , " The exact editor is no longer focused; remaining input was not dispatched. Observe again. " ) }
@@ -718,7 +729,10 @@ final class Driver {
718729 // but never another document window the snapshot did not observe.
719730 guard let rawWindow = attribute ( root , kAXFocusedWindowAttribute ) , CFGetTypeID ( rawWindow ) == AXUIElementGetTypeID ( ) else { throw ComputerError ( " focus_failed " , " The app has no focused window to receive keys; click a window or call activate_app, then observe again. " ) }
720731 let focusedWindow = unsafeBitCast ( rawWindow , to : AXUIElement . self )
721- guard acceptsInput ( focusedWindow , observed : frame . map { [ $0 ] } ?? Array ( snapshot . windowFrames . values ) ) else { throw ComputerError ( " focus_failed " , " Another document window has keyboard focus; observe that window and target it instead. " ) }
732+ let targets : [ ( element : AXUIElement , frame : CGRect ) ]
733+ if let windowId = p . windowId , let frame { targets = snapshot . windowElements [ windowId ] . map { [ ( $0 , frame ) ] } ?? [ ] }
734+ else { targets = snapshot . observedWindows ( ) }
735+ guard acceptsInput ( focusedWindow , observed : targets ) else { throw ComputerError ( " focus_failed " , " Another document window has keyboard focus; observe that window and target it instead. " ) }
722736 if let focused = attribute ( root , kAXFocusedUIElementAttribute ) , CFGetTypeID ( focused ) == AXUIElementGetTypeID ( ) { try requireNonSecure ( unsafeBitCast ( focused , to : AXUIElement . self ) ) }
723737 let ( code , flags ) = try parseKey ( required ( p . key , " key " ) )
724738 var selectedAll = false
@@ -738,22 +752,22 @@ final class Driver {
738752 guard let event = CGEvent ( scrollWheelEvent2Source : CGEventSource ( stateID : . privateState ) , units : . pixel , wheelCount : 2 , wheel1 : Int32 ( - dy ) , wheel2 : Int32 ( - dx ) , wheel3 : 0 ) else { throw ComputerError ( " input_failed " , " Could not create scroll event. " ) }
739753 let windowID = try required ( p . windowId ?? snapshot . windowFrames . first ( where : { $0 . value . contains ( position ) } ) ? . key , " windowId " )
740754 let frame = try required ( snapshot . windowFrames [ windowID ] , " window frame " )
741- try requireForegroundTarget ( pid : pid , windowID : windowID , point : position , frame : frame )
755+ try requireForegroundTarget ( pid : pid , windowID : windowID , window : try required ( snapshot . windowElements [ windowID ] , " observed window " ) , point : position , frame : frame )
742756 event . location = position ; dispatch . willDispatch ( ) ; event . post ( tap : . cghidEventTap )
743757 }
744758 case " drag " :
745759 try await coordinateAction { dispatch in
746760 let start = try point ( p , app : app , snapshot : snapshot ) ; let end = try point ( p , app : app , snapshot : snapshot , end : true )
747761 let windowID = try required ( p . windowId , " windowId " ) ; let frame = try required ( snapshot . windowFrames [ windowID ] , " window frame " )
748- try postMouse ( pid : pid , type : . leftMouseDown , point : start , button : . left , windowID : windowID , frame : frame , dispatch : dispatch )
762+ try postMouse ( pid : pid , type : . leftMouseDown , point : start , button : . left , windowID : windowID , window : snapshot . windowElements [ windowID ] , frame : frame , dispatch : dispatch )
749763 var lastPoint = start
750- defer { try? postMouse ( pid : pid , type : . leftMouseUp , point : lastPoint , button : . left , windowID : windowID , frame : frame , release : true , dispatch : dispatch ) }
764+ defer { try? postMouse ( pid : pid , type : . leftMouseUp , point : lastPoint , button : . left , windowID : windowID , window : snapshot . windowElements [ windowID ] , frame : frame , release : true , dispatch : dispatch ) }
751765 try await Task . sleep ( for : . milliseconds ( 10 ) )
752766 for step in 1 ... 10 {
753767 try checkCancellation ( )
754768 let amount = CGFloat ( step ) / 10
755769 let position = CGPoint ( x : start . x + ( end . x - start . x ) * amount , y : start . y + ( end . y - start . y ) * amount )
756- try postMouse ( pid : pid , type : . leftMouseDragged , point : position , button : . left , windowID : windowID , frame : frame , dispatch : dispatch )
770+ try postMouse ( pid : pid , type : . leftMouseDragged , point : position , button : . left , windowID : windowID , window : snapshot . windowElements [ windowID ] , frame : frame , dispatch : dispatch )
757771 lastPoint = position
758772 try await Task . sleep ( for : . milliseconds ( 10 ) )
759773 }
0 commit comments