From 993948ffcfa1eaaa899a70d871616dd395e9c6fd Mon Sep 17 00:00:00 2001 From: timo <44401485+Timo972@users.noreply.github.com> Date: Thu, 22 Oct 2026 19:25:18 +0200 Subject: [PATCH] fix(mobilerun): report keyboard focus, not focus-engine focus, in /mobilerun/state /mobilerun/state set every node's isFocused, and picked phone_state.focusedElement/isEditable, from the snapshot's hasFocus. That is focus-engine focus: WebKit sets it on every of a page, so all web fields read as focused with nothing focused, and UIKit text fields never set it on a touch-driven iPhone, so a Settings sheet with the keyboard up had no focusedElement and isEditable false (DRO-3220). Pick the one focused node up front from the same snapshot: the deepest node with hasKeyboardFocus, else the deepest with the IsEditing trait (1 << 21). Only that node reports isFocused, and phone_state describes it. isEditable also accepts the TextEntry/IsEditing traits besides the four text input types. tvOS keeps hasFocus as its first signal. Co-Authored-By: Claude Opus 5.5 --- WebDriverAgent.xcodeproj/project.pbxproj | 4 + .../Categories/XCUIApplication+FBHelpers.h | 13 ++ .../Categories/XCUIApplication+FBHelpers.m | 87 +++++++- .../UnitTests/FBMobilerunFocusTests.m | 195 ++++++++++++++++++ 4 files changed, 295 insertions(+), 4 deletions(-) create mode 100644 WebDriverAgentTests/UnitTests/FBMobilerunFocusTests.m diff --git a/WebDriverAgent.xcodeproj/project.pbxproj b/WebDriverAgent.xcodeproj/project.pbxproj index 1fa113b44..d52463352 100644 --- a/WebDriverAgent.xcodeproj/project.pbxproj +++ b/WebDriverAgent.xcodeproj/project.pbxproj @@ -1334,6 +1334,7 @@ FBCAFE000000000000006051 /* FBAudioStreamTests.m in Sources */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000006050 /* FBAudioStreamTests.m */; }; FBCAFE000000000000008051 /* FBBroadcastLabelsTests.m in Sources */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000008050 /* FBBroadcastLabelsTests.m */; }; FBCAFE000000000000008053 /* FBMobilerunA11yCommandsTests.m in Sources */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000008052 /* FBMobilerunA11yCommandsTests.m */; }; + FBCAFE000000000000008055 /* FBMobilerunFocusTests.m in Sources */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000008054 /* FBMobilerunFocusTests.m */; }; FBCAFE000000000000007002 /* FBSocks5TunnelProtocol.h in Headers */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000007000 /* FBSocks5TunnelProtocol.h */; }; FBCAFE000000000000007003 /* FBSocks5TunnelProtocol.h in Headers */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000007000 /* FBSocks5TunnelProtocol.h */; }; FBCAFE000000000000007004 /* FBSocks5TunnelProtocol.m in Sources */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000007001 /* FBSocks5TunnelProtocol.m */; }; @@ -2113,6 +2114,7 @@ FBCAFE000000000000006050 /* FBAudioStreamTests.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = FBAudioStreamTests.m; sourceTree = ""; }; FBCAFE000000000000008050 /* FBBroadcastLabelsTests.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = FBBroadcastLabelsTests.m; sourceTree = ""; }; FBCAFE000000000000008052 /* FBMobilerunA11yCommandsTests.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = FBMobilerunA11yCommandsTests.m; sourceTree = ""; }; + FBCAFE000000000000008054 /* FBMobilerunFocusTests.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = FBMobilerunFocusTests.m; sourceTree = ""; }; FBCAFE000000000000007000 /* FBSocks5TunnelProtocol.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = FBSocks5TunnelProtocol.h; sourceTree = ""; }; FBCAFE000000000000007001 /* FBSocks5TunnelProtocol.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = FBSocks5TunnelProtocol.m; sourceTree = ""; }; FBCAFE000000000000007010 /* FBSocks5URI.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = FBSocks5URI.h; sourceTree = ""; }; @@ -2961,6 +2963,7 @@ FBCAFE000000000000006050 /* FBAudioStreamTests.m */, FBCAFE000000000000008050 /* FBBroadcastLabelsTests.m */, FBCAFE000000000000008052 /* FBMobilerunA11yCommandsTests.m */, + FBCAFE000000000000008054 /* FBMobilerunFocusTests.m */, FBCAFE000000000000002001 /* FBImageUtilsTests.m */, ); path = UnitTests; @@ -5280,6 +5283,7 @@ FBCAFE000000000000006051 /* FBAudioStreamTests.m in Sources */, FBCAFE000000000000008051 /* FBBroadcastLabelsTests.m in Sources */, FBCAFE000000000000008053 /* FBMobilerunA11yCommandsTests.m in Sources */, + FBCAFE000000000000008055 /* FBMobilerunFocusTests.m in Sources */, FBCAFE000000000000002002 /* FBImageUtilsTests.m in Sources */, ); runOnlyForDeploymentPostprocessing = 0; diff --git a/WebDriverAgentLib/Categories/XCUIApplication+FBHelpers.h b/WebDriverAgentLib/Categories/XCUIApplication+FBHelpers.h index 0f1ba0569..7157ae63b 100644 --- a/WebDriverAgentLib/Categories/XCUIApplication+FBHelpers.h +++ b/WebDriverAgentLib/Categories/XCUIApplication+FBHelpers.h @@ -57,6 +57,19 @@ NS_ASSUME_NONNULL_BEGIN */ - (NSDictionary *)fb_mobilerunA11yStateWithScale:(CGFloat)scale; +/** + Picks the one node of a snapshot tree that holds keyboard focus, as reported by + `fb_mobilerunA11yStateWithScale:` (the only node with `isFocused`, and the source of + `phone_state.focusedElement`). The deepest node reporting `hasKeyboardFocus` wins; + failing that, the deepest node carrying the IsEditing accessibility trait. + `hasFocus` is ignored on iOS (WebKit sets it on every web input), and only used + first on tvOS, where focus-engine focus is the real focus. + + @param root The root of the snapshot tree, normally the application snapshot. + @return The focused node, or nil when nothing has keyboard focus. + */ ++ (nullable id)fb_mobilerunFocusedSnapshotInTree:(nullable id)root; + /** Return application elements tree in a form of xml string with default options. diff --git a/WebDriverAgentLib/Categories/XCUIApplication+FBHelpers.m b/WebDriverAgentLib/Categories/XCUIApplication+FBHelpers.m index 6abae1cba..82c5e5bbd 100644 --- a/WebDriverAgentLib/Categories/XCUIApplication+FBHelpers.m +++ b/WebDriverAgentLib/Categories/XCUIApplication+FBHelpers.m @@ -411,14 +411,92 @@ static BOOL FBMobilerunIsCheckableType(XCUIElementType type) } } +// Private UIAccessibility trait bits (absent from the public SDK headers). They are the +// same bits the native RemoteXPC tree in mobilerun-ios reads: TextEntry marks every text +// input (secure ones too), IsEditing marks the one input currently being edited. +static const unsigned long long FBMobilerunTraitTextEntry = 1ULL << 18; +static const unsigned long long FBMobilerunTraitIsEditing = 1ULL << 21; + +// Whether a node takes typed text: the XCTest text input types, plus anything the app +// marks as a text input by trait (web inputs and custom views need not map onto one of +// those element types). +static BOOL FBMobilerunIsEditable(XCUIElementType type, unsigned long long traits) +{ + return FBDoesElementSupportInnerText(type) + || (traits & (FBMobilerunTraitTextEntry | FBMobilerunTraitIsEditing)) != 0; +} + +// Walks the snapshot tree in pre-order and keeps the deepest node accepted by 'match' +// (the first one on a tie), so an ancestor that reports the same flag as its focused +// descendant never wins over it. +static void FBMobilerunFindDeepest(id snapshot, + NSUInteger depth, + BOOL (^match)(id), + id __strong *best, + NSInteger *bestDepth) +{ + if (match(snapshot) && (NSInteger)depth > *bestDepth) { + *best = snapshot; + *bestDepth = (NSInteger)depth; + } + for (id child in snapshot.children) { + FBMobilerunFindDeepest(child, depth + 1, match, best, bestDepth); + } +} + +static id FBMobilerunDeepestMatch(id root, + BOOL (^match)(id)) +{ + id best = nil; + NSInteger bestDepth = -1; + FBMobilerunFindDeepest(root, 0, match, &best, &bestDepth); + return best; +} + ++ (nullable id)fb_mobilerunFocusedSnapshotInTree:(nullable id)root +{ + if (nil == root) { + return nil; + } +#if TARGET_OS_TV + // tvOS has no first responder to speak of; the focus engine's focus is the real one. + id engineFocused = FBMobilerunDeepestMatch(root, ^BOOL(id s) { + return s.hasFocus; + }); + if (nil != engineFocused) { + return engineFocused; + } +#endif + // Keyboard focus belongs to the single first responder. 'hasFocus' is deliberately not + // consulted on iOS: it is focus-engine focus, which UIKit text fields never report on a + // touch-driven iPhone, while WebKit reports it on every of a page whether or not + // one is focused. + id focused = FBMobilerunDeepestMatch(root, ^BOOL(id s) { + return s.hasKeyboardFocus; + }); + if (nil != focused) { + return focused; + } + // Fall back to the IsEditing trait, which marks the input being edited. + return FBMobilerunDeepestMatch(root, ^BOOL(id s) { + return (s.traits & FBMobilerunTraitIsEditing) != 0; + }); +} + - (NSDictionary *)fb_mobilerunA11yStateWithScale:(CGFloat)scale { id snapshot = [self fb_standardSnapshot]; NSString *bundleId = self.bundleID ?: @""; // Side-channel populated during the single tree walk so `phone_state` needs no - // extra snapshots or attribute queries. + // extra snapshots or attribute queries. The focused node is picked up front, from the + // same snapshot, so exactly one node (or none) reports isFocused and phone_state + // describes that same node. NSMutableDictionary *context = [NSMutableDictionary dictionary]; + id focusedSnapshot = [self.class fb_mobilerunFocusedSnapshotInTree:snapshot]; + if (nil != focusedSnapshot) { + context[@"focusedSnapshot"] = focusedSnapshot; + } NSDictionary *tree = [self.class fb_a11yNodeForSnapshot:snapshot scale:scale packageName:bundleId @@ -479,7 +557,8 @@ + (NSDictionary *)fb_a11yNodeForSnapshot:(id)snapshot node[@"isClickable"] = @(isClickable); node[@"isEnabled"] = @(snapshot.isEnabled); node[@"isFocusable"] = @(FBDoesElementSupportInnerText(elementType)); - node[@"isFocused"] = @(snapshot.hasFocus); + BOOL isFocused = (snapshot == context[@"focusedSnapshot"]); + node[@"isFocused"] = @(isFocused); node[@"isScrollable"] = @(FBMobilerunIsScrollableType(elementType)); node[@"isLongClickable"] = @(NO); node[@"isPassword"] = @(isPassword); @@ -489,13 +568,13 @@ + (NSDictionary *)fb_a11yNodeForSnapshot:(id)snapshot if (elementType == XCUIElementTypeKeyboard) { context[@"keyboardVisible"] = @YES; } - if (snapshot.hasFocus && nil == context[@"focusedElement"]) { + if (isFocused) { context[@"focusedElement"] = @{ @"text": wrappedSnapshot.wdValue ?: @"", @"className": className ?: @"", @"resourceId": identifier, }; - context[@"isEditable"] = @(FBDoesElementSupportInnerText(elementType)); + context[@"isEditable"] = @(FBMobilerunIsEditable(elementType, traits)); } NSArray *childElements = snapshot.children; diff --git a/WebDriverAgentTests/UnitTests/FBMobilerunFocusTests.m b/WebDriverAgentTests/UnitTests/FBMobilerunFocusTests.m new file mode 100644 index 000000000..045037aca --- /dev/null +++ b/WebDriverAgentTests/UnitTests/FBMobilerunFocusTests.m @@ -0,0 +1,195 @@ +/** + * Copyright (c) 2026-present, Droidrun. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#import + +#import "FBXCElementSnapshot.h" +#import "XCElementSnapshotDouble.h" +#import "XCUIApplication+FBHelpers.h" + +static const unsigned long long FBTestTraitIsEditing = 1ULL << 21; + +// A snapshot node with settable focus flags, traits, type and children. +@interface FBFocusSnapshotDouble : XCElementSnapshotDouble +@property (nonatomic, assign) BOOL fakeHasFocus; +@property (nonatomic, assign) BOOL fakeHasKeyboardFocus; +@property (nonatomic, assign) unsigned long long fakeTraits; +@property (nonatomic, assign) XCUIElementType fakeElementType; +@property (nonatomic, copy) NSString *fakeIdentifier; +@property (nonatomic, copy) NSArray *fakeChildren; +@end + +@implementation FBFocusSnapshotDouble + +- (BOOL)hasFocus { return self.fakeHasFocus; } +- (BOOL)hasKeyboardFocus { return self.fakeHasKeyboardFocus; } +- (UIAccessibilityTraits)traits { return (UIAccessibilityTraits)self.fakeTraits; } +- (XCUIElementType)elementType { return self.fakeElementType; } +- (NSString *)identifier { return self.fakeIdentifier ?: @""; } +- (NSArray *)children { return self.fakeChildren ?: @[]; } +- (BOOL)isSelected { return NO; } + +@end + +// fb_a11yNodeForSnapshot:... is private to the category; the test drives the whole walk. +@interface XCUIApplication (FBMobilerunFocusTesting) ++ (NSDictionary *)fb_a11yNodeForSnapshot:(id)snapshot + scale:(CGFloat)scale + packageName:(NSString *)bundleId + context:(NSMutableDictionary *)context; +@end + +@interface FBMobilerunFocusTests : XCTestCase +@end + +@implementation FBMobilerunFocusTests + +static FBFocusSnapshotDouble *FBNode(XCUIElementType type, NSString *identifier, NSArray *children) +{ + FBFocusSnapshotDouble *node = [FBFocusSnapshotDouble new]; + node.fakeElementType = type; + node.fakeIdentifier = identifier; + node.fakeChildren = children; + return node; +} + +static void FBCollectFocused(NSDictionary *node, NSMutableArray *focused) +{ + if ([node[@"isFocused"] boolValue]) { + [focused addObject:node[@"resourceId"]]; + } + for (NSDictionary *child in node[@"children"]) { + FBCollectFocused(child, focused); + } +} + +// Runs the same walk fb_mobilerunA11yStateWithScale: runs, minus taking the snapshot. +static NSDictionary *FBWalk(id root, NSMutableDictionary *context) +{ + id focused = [XCUIApplication fb_mobilerunFocusedSnapshotInTree:root]; + if (nil != focused) { + context[@"focusedSnapshot"] = focused; + } + return [XCUIApplication fb_a11yNodeForSnapshot:root scale:1 packageName:@"com.test" context:context]; +} + +- (void)testWebInputsReportingHasFocusAreNotFocused +{ + // WebKit sets hasFocus on every of a page, focused or not. + FBFocusSnapshotDouble *first = FBNode(XCUIElementTypeTextField, @"first", nil); + FBFocusSnapshotDouble *second = FBNode(XCUIElementTypeTextField, @"second", nil); + first.fakeHasFocus = YES; + second.fakeHasFocus = YES; + FBFocusSnapshotDouble *web = FBNode(XCUIElementTypeWebView, @"web", @[first, second]); + FBFocusSnapshotDouble *app = FBNode(XCUIElementTypeApplication, @"app", @[web]); + + NSMutableDictionary *context = [NSMutableDictionary dictionary]; + NSDictionary *tree = FBWalk((id)app, context); + + NSMutableArray *focused = [NSMutableArray array]; + FBCollectFocused(tree, focused); + XCTAssertEqualObjects(focused, @[]); + XCTAssertNil(context[@"focusedElement"]); + XCTAssertFalse([context[@"isEditable"] boolValue]); +} + +- (void)testOnlyTheKeyboardFocusedWebInputIsFocused +{ + FBFocusSnapshotDouble *first = FBNode(XCUIElementTypeTextField, @"first", nil); + FBFocusSnapshotDouble *second = FBNode(XCUIElementTypeTextField, @"second", nil); + first.fakeHasFocus = YES; + second.fakeHasFocus = YES; + second.fakeHasKeyboardFocus = YES; + FBFocusSnapshotDouble *app = FBNode(XCUIElementTypeApplication, @"app", + @[FBNode(XCUIElementTypeWebView, @"web", @[first, second])]); + + NSMutableDictionary *context = [NSMutableDictionary dictionary]; + NSDictionary *tree = FBWalk((id)app, context); + + NSMutableArray *focused = [NSMutableArray array]; + FBCollectFocused(tree, focused); + XCTAssertEqualObjects(focused, @[@"second"]); + XCTAssertEqualObjects(context[@"focusedElement"][@"resourceId"], @"second"); + XCTAssertTrue([context[@"isEditable"] boolValue]); +} + +- (void)testKeyboardFocusedFieldInASheetIsReported +{ + // A UIKit field never reports hasFocus on iPhone; only keyboard focus marks it. + FBFocusSnapshotDouble *name = FBNode(XCUIElementTypeTextField, @"name", nil); + name.fakeHasKeyboardFocus = YES; + FBFocusSnapshotDouble *sheet = FBNode(XCUIElementTypeOther, @"sheet", + @[FBNode(XCUIElementTypeCell, @"cell", @[name])]); + FBFocusSnapshotDouble *keyboard = FBNode(XCUIElementTypeKeyboard, @"keyboard", nil); + FBFocusSnapshotDouble *app = FBNode(XCUIElementTypeApplication, @"app", + @[FBNode(XCUIElementTypeWindow, @"main", nil), + FBNode(XCUIElementTypeWindow, @"sheetWindow", @[sheet]), + keyboard]); + + NSMutableDictionary *context = [NSMutableDictionary dictionary]; + NSDictionary *tree = FBWalk((id)app, context); + + NSMutableArray *focused = [NSMutableArray array]; + FBCollectFocused(tree, focused); + XCTAssertEqualObjects(focused, @[@"name"]); + XCTAssertEqualObjects(context[@"focusedElement"][@"resourceId"], @"name"); + XCTAssertEqualObjects(context[@"focusedElement"][@"className"], @"XCUIElementTypeTextField"); + XCTAssertTrue([context[@"isEditable"] boolValue]); + XCTAssertTrue([context[@"keyboardVisible"] boolValue]); +} + +- (void)testDeepestKeyboardFocusWins +{ + // An ancestor reporting keyboard focus must not shadow the field that has it. + FBFocusSnapshotDouble *field = FBNode(XCUIElementTypeSecureTextField, @"password", nil); + field.fakeHasKeyboardFocus = YES; + FBFocusSnapshotDouble *container = FBNode(XCUIElementTypeOther, @"container", @[field]); + container.fakeHasKeyboardFocus = YES; + FBFocusSnapshotDouble *app = FBNode(XCUIElementTypeApplication, @"app", @[container]); + + id focused = + [XCUIApplication fb_mobilerunFocusedSnapshotInTree:(id)app]; + XCTAssertEqual((id)focused, (id)field); +} + +- (void)testIsEditingTraitIsTheFallback +{ + FBFocusSnapshotDouble *plain = FBNode(XCUIElementTypeTextField, @"plain", nil); + FBFocusSnapshotDouble *editing = FBNode(XCUIElementTypeOther, @"editing", nil); + editing.fakeTraits = FBTestTraitIsEditing; + FBFocusSnapshotDouble *app = FBNode(XCUIElementTypeApplication, @"app", @[plain, editing]); + + NSMutableDictionary *context = [NSMutableDictionary dictionary]; + NSDictionary *tree = FBWalk((id)app, context); + + NSMutableArray *focused = [NSMutableArray array]; + FBCollectFocused(tree, focused); + XCTAssertEqualObjects(focused, @[@"editing"]); + // Not a text input type, but the IsEditing trait says it takes text. + XCTAssertTrue([context[@"isEditable"] boolValue]); +} + +- (void)testKeyboardFocusOutranksIsEditingTrait +{ + FBFocusSnapshotDouble *editing = FBNode(XCUIElementTypeTextField, @"editing", nil); + editing.fakeTraits = FBTestTraitIsEditing; + FBFocusSnapshotDouble *keyboardFocused = FBNode(XCUIElementTypeTextView, @"keyboardFocused", nil); + keyboardFocused.fakeHasKeyboardFocus = YES; + FBFocusSnapshotDouble *app = FBNode(XCUIElementTypeApplication, @"app", @[editing, keyboardFocused]); + + id focused = + [XCUIApplication fb_mobilerunFocusedSnapshotInTree:(id)app]; + XCTAssertEqual((id)focused, (id)keyboardFocused); +} + +- (void)testNothingFocusedWithoutATree +{ + XCTAssertNil([XCUIApplication fb_mobilerunFocusedSnapshotInTree:nil]); +} + +@end