From 142638120825648c40a5f318e073966c18ac1ed9 Mon Sep 17 00:00:00 2001 From: hawai-i <32040032+hawai-i@users.noreply.github.com> Date: Sun, 30 Aug 2026 17:39:31 +0300 Subject: [PATCH] fix(macos): preserve foreground events when context capture fails --- Makefile | 11 +- aw_watcher_window/macos.swift | 421 ++++++++++++++++------------ aw_watcher_window/macos_state.swift | 24 ++ tests/macos_state_tests.swift | 39 +++ 4 files changed, 314 insertions(+), 181 deletions(-) create mode 100644 aw_watcher_window/macos_state.swift create mode 100644 tests/macos_state_tests.swift diff --git a/Makefile b/Makefile index 9e3e70d..ab7d516 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build test package clean +.PHONY: build test test-swift package clean MACOSX_DEPLOYMENT_TARGET ?= 12.0 @@ -11,12 +11,19 @@ build: build-swift: aw_watcher_window/aw-watcher-window-macos -aw_watcher_window/aw-watcher-window-macos: aw_watcher_window/macos.swift +aw_watcher_window/aw-watcher-window-macos: aw_watcher_window/macos.swift aw_watcher_window/macos_state.swift swiftc -target "$(shell uname -m)-apple-macosx$(MACOSX_DEPLOYMENT_TARGET)" $^ -o $@ test: poetry run aw-watcher-window --help # Ensures that it at least starts make typecheck + if [ "$(shell uname)" = "Darwin" ]; then \ + make test-swift; \ + fi + +test-swift: + swiftc aw_watcher_window/macos_state.swift tests/macos_state_tests.swift -o /tmp/aw-watcher-window-macos-state-tests + /tmp/aw-watcher-window-macos-state-tests typecheck: poetry run mypy aw_watcher_window/ --ignore-missing-imports diff --git a/aw_watcher_window/macos.swift b/aw_watcher_window/macos.swift index e8e0136..9cd1b9b 100644 --- a/aw_watcher_window/macos.swift +++ b/aw_watcher_window/macos.swift @@ -161,18 +161,29 @@ let researchBrowserApps = Set([ let main = MainThing() var oldHeartbeat: Heartbeat? -let encoder = JSONEncoder() -let formatter = ISO8601DateFormatter() -formatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds] - -encoder.dateEncodingStrategy = .custom({ date, encoder in - var container = encoder.singleValueContainer() - let dateString = formatter.string(from: date) - try container.encode(dateString) -}) - -start() -RunLoop.main.run() +let formatter: ISO8601DateFormatter = { + let formatter = ISO8601DateFormatter() + formatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds] + return formatter +}() + +let encoder: JSONEncoder = { + let encoder = JSONEncoder() + encoder.dateEncodingStrategy = .custom({ date, encoder in + var container = encoder.singleValueContainer() + let dateString = formatter.string(from: date) + try container.encode(dateString) + }) + return encoder +}() + +@main +struct ActivityWatchMacOSWatcher { + static func main() { + start() + RunLoop.main.run() + } +} func compileExcludeTitlePattern(_ pattern: String) -> NSRegularExpression { do { @@ -336,12 +347,12 @@ func start() { // listen for changes in focused application NSWorkspace.shared.notificationCenter.addObserver( main, - selector: #selector(main.focusedAppChanged), + selector: #selector(main.focusedAppChanged(_:)), name: NSWorkspace.didActivateApplicationNotification, object: nil ) - main.focusedAppChanged() + main.reconcileCurrentForegroundApp(source: "startup") // Start the polling timer main.pollingTimer = Timer.scheduledTimer(timeInterval: 10.0, target: main, selector: #selector(main.pollActiveWindow), userInfo: nil, repeats: true) @@ -434,9 +445,14 @@ func sendHeartbeatSingle(_ heartbeat: Heartbeat, pulsetime: Double) async throws class MainThing { var observer: AXObserver? + var foregroundApplication: NSRunningApplication? var oldWindow: AXUIElement? var pollingTimer: Timer? + var trackedPID: pid_t? { + return foregroundApplication?.processIdentifier + } + // list of chrome equivalent browsers let CHROME_BROWSERS = [ "Google Chrome", @@ -498,128 +514,250 @@ class MainThing { return nil } + func elementPID(_ element: AXUIElement) -> pid_t? { + var pid: pid_t = 0 + return AXUIElementGetPid(element, &pid) == .success ? pid : nil + } + @objc func pollActiveWindow() { - debug("Polling active window") + reconcileCurrentForegroundApp(source: "poll") + } - guard let frontmost = NSWorkspace.shared.frontmostApplication else { - log("Failed to get frontmost application from polling") + @objc func focusedAppChanged(_ notification: Notification) { + let notificationApplication = notification.userInfo?[NSWorkspace.applicationUserInfoKey] as? NSRunningApplication + if let application = notificationApplication, !application.isTerminated { + reconcileForegroundApp(application: application, source: "activation") + } else { + log("Activation notification lacked a live application; falling back to current foreground app") + reconcileCurrentForegroundApp(source: "activation-fallback") + } + } + + func reconcileCurrentForegroundApp(source: String) { + guard let application = NSWorkspace.shared.frontmostApplication, !application.isTerminated else { + log("Failed to get a live foreground application from \(source)") return } + reconcileForegroundApp(application: application, source: source) + } + + func reconcileForegroundApp(application: NSRunningApplication, source: String) { + let pid = application.processIdentifier + let action = foregroundReconciliationAction( + trackedPID: trackedPID, + observerAvailable: observer != nil, + candidatePID: pid + ) + + if action == .rebuildObserver { + debug("Rebuilding AX observer for pid \(pid) from \(source)") + rebuildObserver(for: application) + } else { + foregroundApplication = application + } + + refreshFocusedWindow(for: application) + } + + func tearDownObserver() { + if let observer = observer { + if let oldWindow = oldWindow { + AXObserverRemoveNotification(observer, oldWindow, kAXTitleChangedNotification as CFString) + } + CFRunLoopRemoveSource( + RunLoop.current.getCFRunLoop(), + AXObserverGetRunLoopSource(observer), + CFRunLoopMode.defaultMode + ) + } + observer = nil + oldWindow = nil + } - let pid = frontmost.processIdentifier + func rebuildObserver(for application: NSRunningApplication) { + tearDownObserver() + foregroundApplication = application + + let pid = application.processIdentifier let focusedApp = AXUIElementCreateApplication(pid) + var newObserver: AXObserver? + let createResult = AXObserverCreate( + pid, + { + ( + axObserver: AXObserver, + axElement: AXUIElement, + notification: CFString, + userData: UnsafeMutableRawPointer? + ) -> Void in + guard let userData = userData else { + log("Missing AX observer userData") + return + } + let watcher = Unmanaged.fromOpaque(userData).takeUnretainedValue() + watcher.handleAXNotification( + observer: axObserver, + element: axElement, + notification: notification + ) + }, + &newObserver + ) - var focusedWindow: AnyObject? - AXUIElementCopyAttributeValue(focusedApp, kAXFocusedWindowAttribute as CFString, &focusedWindow) + guard createResult == .success, let newObserver = newObserver else { + log("Failed to create AX observer for pid \(pid): \(createResult.rawValue)") + return + } - if focusedWindow != nil { - focusedWindowChanged(observer!, window: focusedWindow as! AXUIElement) + let selfPtr = UnsafeMutableRawPointer(Unmanaged.passUnretained(self).toOpaque()) + let addResult = AXObserverAddNotification( + newObserver, + focusedApp, + kAXFocusedWindowChangedNotification as CFString, + selfPtr + ) + guard addResult == .success || addResult == .notificationAlreadyRegistered else { + log("Failed to observe focused-window changes for pid \(pid): \(addResult.rawValue)") + return } + + observer = newObserver + CFRunLoopAddSource( + RunLoop.current.getCFRunLoop(), + AXObserverGetRunLoopSource(newObserver), + CFRunLoopMode.defaultMode + ) } - deinit { - pollingTimer?.invalidate() + func refreshFocusedWindow(for application: NSRunningApplication) { + guard trackedPID == application.processIdentifier else { + debug("Ignoring focused-window refresh for stale pid \(application.processIdentifier)") + return + } + + let focusedApp = AXUIElementCreateApplication(application.processIdentifier) + var focusedWindowValue: AnyObject? + let result = AXUIElementCopyAttributeValue( + focusedApp, + kAXFocusedWindowAttribute as CFString, + &focusedWindowValue + ) + var focusedWindow: AXUIElement? + if result == .success, let focusedWindowValue = focusedWindowValue { + focusedWindow = (focusedWindowValue as! AXUIElement) + } + updateFocusedWindow(focusedWindow, for: application) } - func windowTitleChanged( - _ axObserver: AXObserver, - axElement: AXUIElement, + func updateFocusedWindow(_ window: AXUIElement?, for application: NSRunningApplication) { + guard trackedPID == application.processIdentifier else { + debug("Ignoring focused-window update for stale pid \(application.processIdentifier)") + return + } + + var windowChanged = oldWindow == nil || window == nil + if let oldWindow = oldWindow, let window = window { + windowChanged = !CFEqual(oldWindow, window) + } else if oldWindow == nil && window == nil { + windowChanged = false + } + + if windowChanged, let observer = observer { + if let oldWindow = oldWindow { + AXObserverRemoveNotification(observer, oldWindow, kAXTitleChangedNotification as CFString) + } + if let window = window { + let selfPtr = UnsafeMutableRawPointer(Unmanaged.passUnretained(self).toOpaque()) + let addResult = AXObserverAddNotification( + observer, + window, + kAXTitleChangedNotification as CFString, + selfPtr + ) + if addResult != .success && addResult != .notificationAlreadyRegistered { + log("Failed to observe title changes for pid \(application.processIdentifier): \(addResult.rawValue)") + } + } + } + + oldWindow = window + emitHeartbeat(application: application, window: window) + } + + func handleAXNotification( + observer callbackObserver: AXObserver, + element: AXUIElement, notification: CFString ) { - guard let frontmost = NSWorkspace.shared.frontmostApplication else { - log("Failed to get frontmost application from window title notification") + guard let application = foregroundApplication, + let currentObserver = observer, + CFEqual(callbackObserver, currentObserver), + axCallbackBelongsToForeground(trackedPID: trackedPID, elementPID: elementPID(element)) else { + debug("Ignoring stale AX callback") return } - // calculate now before executing any scripting since that can take some time - let nowTime = Date.now + if notification == kAXFocusedWindowChangedNotification as CFString { + refreshFocusedWindow(for: application) + } else if notification == kAXTitleChangedNotification as CFString { + emitHeartbeat(application: application, window: element) + } + } + + func emitHeartbeat(application: NSRunningApplication, window: AXUIElement?) { + guard trackedPID == application.processIdentifier else { + debug("Ignoring heartbeat for stale pid \(application.processIdentifier)") + return + } + // Calculate now before optional browser scripting, which may take time. + let nowTime = Date.now var windowTitle: AnyObject? - AXUIElementCopyAttributeValue(axElement, kAXTitleAttribute as CFString, &windowTitle) + if let window = window { + AXUIElementCopyAttributeValue(window, kAXTitleAttribute as CFString, &windowTitle) + } - let applicationName = frontmost.localizedName ?? frontmost.bundleIdentifier ?? "" + let applicationName = application.localizedName ?? application.bundleIdentifier ?? "" var data = NetworkMessage(app: applicationName, title: windowTitle as? String ?? "") if CHROME_BROWSERS.contains(applicationName) { debug("Chrome browser detected, extracting URL and title") - - guard let bundleIdentifier = frontmost.bundleIdentifier else { - log("Failed to get bundle identifier from frontmost application, which was recognized to be Chrome") - return - } - let chromeObject: ChromeProtocol = SBApplication.init(bundleIdentifier: bundleIdentifier)! - - guard let windows = chromeObject.windows, - let frontWindow = windows().first else { - log("Failed to get chrome front window") - return - } - guard let activeTab = frontWindow.activeTab else { - log("Failed to get chrome active tab") - return - } - - if frontWindow.mode == "incognito" { - data = NetworkMessage(app: "", title: "") - } else { - data.url = activeTab.URL - - // the tab title is more accurate and often different than the window title - // however, in some cases the binary does not have the right permissions to read - // the title properly and will return a blank string - - if let tabTitle = activeTab.title { - if(tabTitle != "" && data.title != tabTitle) { - error("tab title diff: \(tabTitle), window title: \(data.title)") + if let bundleIdentifier = application.bundleIdentifier, + let chromeObject: ChromeProtocol = SBApplication.init(bundleIdentifier: bundleIdentifier), + let windows = chromeObject.windows, + let frontWindow = windows().first, + let activeTab = frontWindow.activeTab { + if frontWindow.mode == "incognito" { + data = NetworkMessage(app: "", title: "") + } else { + data.url = activeTab.URL + if let tabTitle = activeTab.title, tabTitle != "", data.title != tabTitle { data.title = tabTitle } } + } else { + log("Failed to read Chrome context; emitting foreground heartbeat without URL") } - } else if frontmost.localizedName == "Safari" { + } else if applicationName == "Safari" { debug("Safari browser detected, extracting URL and title") - - guard let bundleIdentifier = frontmost.bundleIdentifier else { - log("Failed to get bundle identifier from frontmost application, which was recognized to be Safari") - return - } - let safariObject: SafariApplication = SBApplication.init(bundleIdentifier: bundleIdentifier)! - - guard let windows = safariObject.windows, - let frontWindow = windows().first else { - log("Failed to get safari front window") - return - } - guard let activeTab = frontWindow.currentTab else { - log("Failed to get safari active tab") - return - } - - // Safari doesn't allow incognito mode to be inspected, so we do not know if we should hide the url - data.url = activeTab.URL - - // comment above applies here as well - if let tabTitle = activeTab.name { - if tabTitle != "" && data.title != tabTitle { - error("tab title diff: \(tabTitle), window title: \(data.title)") + if let bundleIdentifier = application.bundleIdentifier, + let safariObject: SafariApplication = SBApplication.init(bundleIdentifier: bundleIdentifier), + let windows = safariObject.windows, + let frontWindow = windows().first, + let activeTab = frontWindow.currentTab { + data.url = activeTab.URL + if let tabTitle = activeTab.name, tabTitle != "", data.title != tabTitle { data.title = tabTitle } + } else { + log("Failed to read Safari context; emitting foreground heartbeat without URL") } - } else if FIREFOX_BROWSERS.contains(applicationName) { + } else if FIREFOX_BROWSERS.contains(applicationName), let window = window { debug("Firefox-based browser detected, extracting URL from accessibility tree") - - // note: private windows are not hidden here (unlike the Chrome incognito - // branch) — Gecko does not mark them in the accessibility tree, and their - // window titles carry a "Private Browsing" suffix for rules to match - data.url = geckoURL(window: axElement) + data.url = geckoURL(window: window) if data.url == nil { - // Newer Gecko builds instantiate their accessibility engine lazily and - // no longer treat plain tree walks as an assistive client, leaving the - // window's AX tree without any web content. Requesting - // AXEnhancedUserInterface (as VoiceOver does) turns the engine on; the - // call may report an error while the engine spins up, but the tree is - // populated for subsequent polls and stays on for the browser session. - let axApp = AXUIElementCreateApplication(frontmost.processIdentifier) + let axApp = AXUIElementCreateApplication(application.processIdentifier) AXUIElementSetAttributeValue(axApp, "AXEnhancedUserInterface" as CFString, kCFBooleanTrue) } } @@ -628,90 +766,15 @@ class MainThing { data = applyResearchFilter(data) } else if excludeTitle || titleShouldBeExcluded(data.title ?? "") { data.title = "excluded" - // the URL identifies the page at least as precisely as the title does, - // so an excluded window must not report it either data.url = nil } - let heartbeat = Heartbeat(timestamp: nowTime, data: data) - sendHeartbeat(heartbeat) - } - - @objc func focusedWindowChanged(_ observer: AXObserver, window: AXUIElement) { - debug("Focused window changed") - - if oldWindow != nil { - AXObserverRemoveNotification(observer, oldWindow!, kAXFocusedWindowChangedNotification as CFString) - } - - let selfPtr = UnsafeMutableRawPointer(Unmanaged.passUnretained(self).toOpaque()) - AXObserverAddNotification(observer, window, kAXTitleChangedNotification as CFString, selfPtr) - - windowTitleChanged( - observer, axElement: window, notification: kAXTitleChangedNotification as CFString) - - oldWindow = window + sendHeartbeat(Heartbeat(timestamp: nowTime, data: data)) } - @objc func focusedAppChanged() { - debug("Focused app changed") - - if observer != nil { - CFRunLoopRemoveSource( - RunLoop.current.getCFRunLoop(), - AXObserverGetRunLoopSource(observer!), - CFRunLoopMode.defaultMode - ) - } - - guard let frontmost = NSWorkspace.shared.frontmostApplication else { - log("Failed to get frontmost application from app change notification") - return - } - - let pid = frontmost.processIdentifier - let focusedApp = AXUIElementCreateApplication(pid) - - AXObserverCreate( - pid, - { - ( - _ axObserver: AXObserver, - axElement: AXUIElement, - notification: CFString, - userData: UnsafeMutableRawPointer? - ) -> Void in - guard let userData = userData else { - log("Missing userData") - return - } - let application = Unmanaged.fromOpaque(userData).takeUnretainedValue() - if notification == kAXFocusedWindowChangedNotification as CFString { - application.focusedWindowChanged(axObserver, window: axElement) - } else { - application.windowTitleChanged( - axObserver, - axElement: axElement, - notification: notification - ) - } - }, &observer) - - let selfPtr = UnsafeMutableRawPointer(Unmanaged.passUnretained(self).toOpaque()) - AXObserverAddNotification(observer!, focusedApp, kAXFocusedWindowChangedNotification as CFString, selfPtr) - - CFRunLoopAddSource( - RunLoop.current.getCFRunLoop(), - AXObserverGetRunLoopSource(observer!), - CFRunLoopMode.defaultMode - ) - - var focusedWindow: AnyObject? - AXUIElementCopyAttributeValue(focusedApp, kAXFocusedWindowAttribute as CFString, &focusedWindow) - - if focusedWindow != nil { - focusedWindowChanged(observer!, window: focusedWindow as! AXUIElement) - } + deinit { + pollingTimer?.invalidate() + tearDownObserver() } } diff --git a/aw_watcher_window/macos_state.swift b/aw_watcher_window/macos_state.swift new file mode 100644 index 0000000..cf3e3b3 --- /dev/null +++ b/aw_watcher_window/macos_state.swift @@ -0,0 +1,24 @@ +import Foundation + +enum ForegroundReconciliationAction: Equatable { + case rebuildObserver + case refreshWindow +} + +func foregroundReconciliationAction( + trackedPID: pid_t?, + observerAvailable: Bool, + candidatePID: pid_t +) -> ForegroundReconciliationAction { + if trackedPID != candidatePID || !observerAvailable { + return .rebuildObserver + } + return .refreshWindow +} + +func axCallbackBelongsToForeground(trackedPID: pid_t?, elementPID: pid_t?) -> Bool { + guard let trackedPID = trackedPID, let elementPID = elementPID else { + return false + } + return trackedPID == elementPID +} diff --git a/tests/macos_state_tests.swift b/tests/macos_state_tests.swift new file mode 100644 index 0000000..cf3f4a4 --- /dev/null +++ b/tests/macos_state_tests.swift @@ -0,0 +1,39 @@ +import Foundation + +@main +struct MacOSStateTests { + static func expect(_ condition: @autoclosure () -> Bool, _ message: String) { + if !condition() { + fputs("FAIL: \(message)\n", stderr) + exit(1) + } + } + + static func main() { + expect( + foregroundReconciliationAction(trackedPID: 100, observerAvailable: true, candidatePID: 200) == .rebuildObserver, + "activation/poll PID change must rebuild the observer" + ) + expect( + foregroundReconciliationAction(trackedPID: 200, observerAvailable: true, candidatePID: 200) == .refreshWindow, + "same PID with a live observer must only refresh the focused window" + ) + expect( + foregroundReconciliationAction(trackedPID: 200, observerAvailable: false, candidatePID: 200) == .rebuildObserver, + "a missing observer must be repaired even when the PID is unchanged" + ) + expect( + !axCallbackBelongsToForeground(trackedPID: 200, elementPID: 100), + "a stale AX callback must not overwrite the current foreground app" + ) + expect( + axCallbackBelongsToForeground(trackedPID: 200, elementPID: 200), + "an AX callback for the tracked PID must be accepted" + ) + expect( + !axCallbackBelongsToForeground(trackedPID: nil, elementPID: 200), + "an AX callback must be rejected when no foreground PID is tracked" + ) + print("macOS foreground state tests passed") + } +}