Skip to content

Commit b22eecd

Browse files
committed
fix(desktop): keep hidden tabs out of the window list and pin typed text to its first editor
1 parent 9f29b11 commit b22eecd

2 files changed

Lines changed: 16 additions & 7 deletions

File tree

‎apps/desktop/native/computer-use/ComputerUse.swift‎

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -612,15 +612,20 @@ final class Driver {
612612
try await snapshot.read(preferredWindowFrame: preferredFrame)
613613
let axWindows = snapshot.elements.values.filter { attribute($0, kAXRoleAttribute) as? String == kAXWindowRole }
614614
let axWindowFrames = axWindows.compactMap(elementRect)
615-
let windows = windowsFor(app.processIdentifier).prefix(100).compactMap { entry -> [String: Any]? in
616-
guard let id = entry[kCGWindowNumber as String] as? NSNumber, let bounds = entry[kCGWindowBounds as String] as? [String: Any], let rect = CGRect(dictionaryRepresentation: bounds as CFDictionary), rect.width > 0, rect.height > 0, axWindowFrames.contains(rect) else { return nil }
615+
let entries = windowsFor(app.processIdentifier).prefix(100)
616+
func bounds(_ entry: [String: Any]) -> CGRect? { (entry[kCGWindowBounds as String] as? [String: Any]).flatMap { CGRect(dictionaryRepresentation: $0 as CFDictionary) } }
617+
func onScreen(_ entry: [String: Any]) -> Bool { entry[kCGWindowIsOnscreen as String] as? Bool == true }
618+
// A background tab is an off-screen window with the bounds of the tab in front, which is the
619+
// only one Accessibility shows, so it can be neither observed nor targeted.
620+
let onScreenRects = entries.filter(onScreen).compactMap(bounds)
621+
let windows = entries.compactMap { entry -> [String: Any]? in
622+
guard let id = entry[kCGWindowNumber as String] as? NSNumber, let rect = bounds(entry), rect.width > 0, rect.height > 0, axWindowFrames.contains(rect), onScreen(entry) || !onScreenRects.contains(rect) else { return nil }
617623
snapshot.windowFrames[id.stringValue] = rect
618624
// Public APIs match a window-list entry to its Accessibility window only by bounds; when
619625
// two windows share bounds that match is ambiguous, so neither gets an identity and
620-
// input aimed at either is refused. A background tab or minimized window is off screen
621-
// and shares bounds with the window it hides behind, so it gets none either.
626+
// input aimed at either is refused. An off-screen (minimized) window takes no input either.
622627
let matches = axWindows.filter { elementRect($0) == rect }
623-
if matches.count == 1, entry[kCGWindowIsOnscreen as String] as? Bool == true { snapshot.windowElements[id.stringValue] = matches[0] }
628+
if matches.count == 1, onScreen(entry) { snapshot.windowElements[id.stringValue] = matches[0] }
624629
var result: [String: Any] = ["windowId": id.stringValue, "title": entry[kCGWindowName as String] as? String ?? ""]
625630
if let title = result["title"] as? String { result["title"] = String(title.prefix(4096)) }
626631
result.merge(rectJSON(rect)) { _, new in new }; return result
@@ -723,17 +728,20 @@ final class Driver {
723728
try await Task.sleep(for: .milliseconds(50))
724729
}
725730
let observedWindows = snapshot.observedWindows()
731+
// The first editor found holding focus receives the whole input, so text never follows focus elsewhere.
732+
var editor: AXUIElement?
726733
func validateFocus() throws {
727734
try checkCancellation(); try requireNonSecure(element)
728735
// The editor must still sit in an observed window (or a dialog above it) at the observed frame.
729736
guard let window = containingWindow(element), acceptsInput(window, observed: observedWindows) else {
730737
throw ComputerError("stale_window", "The editor's window moved or is not an observed window; observe again.")
731738
}
732-
guard let current = attribute(root, kAXFocusedUIElementAttribute), holdsFocus(current, element) else { throw ComputerError("focus_changed", "The exact editor is no longer focused; remaining input was not dispatched. Observe again.") }
739+
guard let current = attribute(root, kAXFocusedUIElementAttribute), holdsFocus(current, element), editor.map({ CFEqual($0, current) }) ?? true else { throw ComputerError("focus_changed", "The exact editor is no longer focused; remaining input was not dispatched. Observe again.") }
733740
// Keystrokes land on the focused element, so it (or the text control targeted) must take text.
734741
let receiver = unsafeBitCast(current, to: AXUIElement.self)
735742
try requireNonSecure(receiver)
736743
guard targetTakesText || takesText(receiver) else { throw ComputerError("unsupported_action", "Keyboard focus is not in a text input; no text was sent. Click into the text field, or use press_key for shortcuts.") }
744+
editor = receiver
737745
}
738746
// AX focus requests and app activation may settle asynchronously. Wait without
739747
// posting input or requesting focus again; every eventual input keeps the same guard.

‎apps/desktop/native/computer-use/tests/live.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,8 @@ def coordinate(snapshot, node, fx=0.5, fy=0.5):
118118
error = refused('type_text', snapshot, elementId=slider['elementId'], text=' ')
119119
assert error['code'] == 'unsupported_action' and error['dispatchState'] == 'not_started', error
120120
assert find(state(), role='AXSlider')['value'] == slider['value'], 'Text reached a control that does not take text'
121-
snapshot = mutate('click', state(), elementId=find(snapshot, role='AXTextField', label='Fixture editable text')['elementId'])
121+
snapshot = state()
122+
snapshot = mutate('click', snapshot, elementId=find(snapshot, role='AXTextField', label='Fixture editable text')['elementId'])
122123
snapshot = mutate('type_text', snapshot, elementId=find(snapshot, role='AXWindow')['elementId'], text=' window')
123124
assert find(snapshot, role='AXTextField', label='Fixture editable text')['value'].endswith(' window'), 'Window target did not type into its focused editor'
124125
window = next(w for w in snapshot['windows'] if w['windowId'] == snapshot['windowId'])

0 commit comments

Comments
 (0)