diff --git a/WebDriverAgent.xcodeproj/project.pbxproj b/WebDriverAgent.xcodeproj/project.pbxproj index 1141c4545..7e69d4c9f 100644 --- a/WebDriverAgent.xcodeproj/project.pbxproj +++ b/WebDriverAgent.xcodeproj/project.pbxproj @@ -1335,6 +1335,7 @@ FBCAFE000000000000006051 /* FBAudioStreamTests.m in Sources */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000006050 /* FBAudioStreamTests.m */; }; FBCAFE000000000000008051 /* FBBroadcastLabelsTests.m in Sources */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000008050 /* FBBroadcastLabelsTests.m */; }; FBCAFE000000000000008053 /* FBMobilerunA11yCommandsTests.m in Sources */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000008052 /* FBMobilerunA11yCommandsTests.m */; }; + FBCAFE000000000000008055 /* FBMobilerunFocusTests.m in Sources */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000008054 /* FBMobilerunFocusTests.m */; }; FBCAFE000000000000007002 /* FBSocks5TunnelProtocol.h in Headers */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000007000 /* FBSocks5TunnelProtocol.h */; }; FBCAFE000000000000007003 /* FBSocks5TunnelProtocol.h in Headers */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000007000 /* FBSocks5TunnelProtocol.h */; }; FBCAFE000000000000007004 /* FBSocks5TunnelProtocol.m in Sources */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000007001 /* FBSocks5TunnelProtocol.m */; }; @@ -2115,6 +2116,7 @@ FBCAFE000000000000006050 /* FBAudioStreamTests.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = FBAudioStreamTests.m; sourceTree = ""; }; FBCAFE000000000000008050 /* FBBroadcastLabelsTests.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = FBBroadcastLabelsTests.m; sourceTree = ""; }; FBCAFE000000000000008052 /* FBMobilerunA11yCommandsTests.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = FBMobilerunA11yCommandsTests.m; sourceTree = ""; }; + FBCAFE000000000000008054 /* FBMobilerunFocusTests.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = FBMobilerunFocusTests.m; sourceTree = ""; }; FBCAFE000000000000007000 /* FBSocks5TunnelProtocol.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = FBSocks5TunnelProtocol.h; sourceTree = ""; }; FBCAFE000000000000007001 /* FBSocks5TunnelProtocol.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = FBSocks5TunnelProtocol.m; sourceTree = ""; }; FBCAFE000000000000007010 /* FBSocks5URI.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = FBSocks5URI.h; sourceTree = ""; }; @@ -2964,6 +2966,7 @@ FBCAFE000000000000006050 /* FBAudioStreamTests.m */, FBCAFE000000000000008050 /* FBBroadcastLabelsTests.m */, FBCAFE000000000000008052 /* FBMobilerunA11yCommandsTests.m */, + FBCAFE000000000000008054 /* FBMobilerunFocusTests.m */, FBCAFE000000000000002001 /* FBImageUtilsTests.m */, ); path = UnitTests; @@ -5284,6 +5287,7 @@ FBCAFE000000000000006051 /* FBAudioStreamTests.m in Sources */, FBCAFE000000000000008051 /* FBBroadcastLabelsTests.m in Sources */, FBCAFE000000000000008053 /* FBMobilerunA11yCommandsTests.m in Sources */, + FBCAFE000000000000008055 /* FBMobilerunFocusTests.m in Sources */, FBCAFE000000000000002002 /* FBImageUtilsTests.m in Sources */, ); runOnlyForDeploymentPostprocessing = 0; diff --git a/WebDriverAgentLib/Categories/XCUIApplication+FBHelpers.h b/WebDriverAgentLib/Categories/XCUIApplication+FBHelpers.h index b9059d4d4..88921e7b4 100644 --- a/WebDriverAgentLib/Categories/XCUIApplication+FBHelpers.h +++ b/WebDriverAgentLib/Categories/XCUIApplication+FBHelpers.h @@ -77,6 +77,19 @@ NS_ASSUME_NONNULL_BEGIN scale:(CGFloat)scale truncated:(BOOL)truncated; +/** + Picks the one node of a snapshot tree that holds keyboard focus, as reported by + `fb_mobilerunA11yStateWithSnapshot:scale:truncated:` (the only node with `isFocused`, and the source of + `phone_state.focusedElement`). The deepest node reporting `hasKeyboardFocus` wins; + failing that, the deepest node carrying the IsEditing accessibility trait. + `hasFocus` is ignored on iOS (WebKit sets it on every web input), and only used + first on tvOS, where focus-engine focus is the real focus. + + @param root The root of the snapshot tree, normally the application snapshot. + @return The focused node, or nil when nothing has keyboard focus. + */ ++ (nullable id)fb_mobilerunFocusedSnapshotInTree:(nullable id)root; + /** Return application elements tree in a form of xml string with default options. diff --git a/WebDriverAgentLib/Categories/XCUIApplication+FBHelpers.m b/WebDriverAgentLib/Categories/XCUIApplication+FBHelpers.m index 797980afc..364a81925 100644 --- a/WebDriverAgentLib/Categories/XCUIApplication+FBHelpers.m +++ b/WebDriverAgentLib/Categories/XCUIApplication+FBHelpers.m @@ -415,6 +415,78 @@ static BOOL FBMobilerunIsCheckableType(XCUIElementType type) } } +// Private UIAccessibility trait bits (absent from the public SDK headers). They are the +// same bits the native RemoteXPC tree in mobilerun-ios reads: TextEntry marks every text +// input (secure ones too), IsEditing marks the one input currently being edited. +static const unsigned long long FBMobilerunTraitTextEntry = 1ULL << 18; +static const unsigned long long FBMobilerunTraitIsEditing = 1ULL << 21; + +// Whether a node takes typed text: the XCTest text input types, plus anything the app +// marks as a text input by trait (web inputs and custom views need not map onto one of +// those element types). +static BOOL FBMobilerunIsEditable(XCUIElementType type, unsigned long long traits) +{ + return FBDoesElementSupportInnerText(type) + || (traits & (FBMobilerunTraitTextEntry | FBMobilerunTraitIsEditing)) != 0; +} + +// Walks the snapshot tree in pre-order and keeps the deepest node accepted by 'match' +// (the first one on a tie), so an ancestor that reports the same flag as its focused +// descendant never wins over it. +static void FBMobilerunFindDeepest(id snapshot, + NSUInteger depth, + BOOL (^match)(id), + id __strong *best, + NSInteger *bestDepth) +{ + if (match(snapshot) && (NSInteger)depth > *bestDepth) { + *best = snapshot; + *bestDepth = (NSInteger)depth; + } + for (id child in snapshot.children) { + FBMobilerunFindDeepest(child, depth + 1, match, best, bestDepth); + } +} + +static id FBMobilerunDeepestMatch(id root, + BOOL (^match)(id)) +{ + id best = nil; + NSInteger bestDepth = -1; + FBMobilerunFindDeepest(root, 0, match, &best, &bestDepth); + return best; +} + ++ (nullable id)fb_mobilerunFocusedSnapshotInTree:(nullable id)root +{ + if (nil == root) { + return nil; + } +#if TARGET_OS_TV + // tvOS has no first responder to speak of; the focus engine's focus is the real one. + id engineFocused = FBMobilerunDeepestMatch(root, ^BOOL(id s) { + return s.hasFocus; + }); + if (nil != engineFocused) { + return engineFocused; + } +#endif + // Keyboard focus belongs to the single first responder. 'hasFocus' is deliberately not + // consulted on iOS: it is focus-engine focus, which UIKit text fields never report on a + // touch-driven iPhone, while WebKit reports it on every of a page whether or not + // one is focused. + id focused = FBMobilerunDeepestMatch(root, ^BOOL(id s) { + return s.hasKeyboardFocus; + }); + if (nil != focused) { + return focused; + } + // Fall back to the IsEditing trait, which marks the input being edited. + return FBMobilerunDeepestMatch(root, ^BOOL(id s) { + return (s.traits & FBMobilerunTraitIsEditing) != 0; + }); +} + - (id)fb_snapshotWithTimeout:(NSTimeInterval)timeout parameters:(NSDictionary *)parameters error:(NSError **)error @@ -444,8 +516,14 @@ - (NSDictionary *)fb_mobilerunA11yStateWithSnapshot:(id)sna NSString *bundleId = self.bundleID ?: @""; // Side-channel populated during the single tree walk so `phone_state` needs no - // extra snapshots or attribute queries. + // extra snapshots or attribute queries. The focused node is picked up front, from the + // same snapshot, so exactly one node (or none) reports isFocused and phone_state + // describes that same node. NSMutableDictionary *context = [NSMutableDictionary dictionary]; + id focusedSnapshot = [self.class fb_mobilerunFocusedSnapshotInTree:snapshot]; + if (nil != focusedSnapshot) { + context[@"focusedSnapshot"] = focusedSnapshot; + } NSDictionary *tree = nil == snapshot ? [self.class fb_a11yRootNodeWithScale:scale packageName:bundleId] : [self.class fb_a11yNodeForSnapshot:snapshot @@ -545,7 +623,8 @@ + (NSDictionary *)fb_a11yNodeForSnapshot:(id)snapshot node[@"isClickable"] = @(isClickable); node[@"isEnabled"] = @(snapshot.isEnabled); node[@"isFocusable"] = @(FBDoesElementSupportInnerText(elementType)); - node[@"isFocused"] = @(snapshot.hasFocus); + BOOL isFocused = (snapshot == context[@"focusedSnapshot"]); + node[@"isFocused"] = @(isFocused); node[@"isScrollable"] = @(FBMobilerunIsScrollableType(elementType)); node[@"isLongClickable"] = @(NO); node[@"isPassword"] = @(isPassword); @@ -555,13 +634,13 @@ + (NSDictionary *)fb_a11yNodeForSnapshot:(id)snapshot if (elementType == XCUIElementTypeKeyboard) { context[@"keyboardVisible"] = @YES; } - if (snapshot.hasFocus && nil == context[@"focusedElement"]) { + if (isFocused) { context[@"focusedElement"] = @{ @"text": wrappedSnapshot.wdValue ?: @"", @"className": className ?: @"", @"resourceId": identifier, }; - context[@"isEditable"] = @(FBDoesElementSupportInnerText(elementType)); + context[@"isEditable"] = @(FBMobilerunIsEditable(elementType, traits)); } NSArray *childElements = snapshot.children; diff --git a/WebDriverAgentTests/UnitTests/FBMobilerunFocusTests.m b/WebDriverAgentTests/UnitTests/FBMobilerunFocusTests.m new file mode 100644 index 000000000..814030061 --- /dev/null +++ b/WebDriverAgentTests/UnitTests/FBMobilerunFocusTests.m @@ -0,0 +1,195 @@ +/** + * Copyright (c) 2026-present, Droidrun. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#import + +#import "FBXCElementSnapshot.h" +#import "XCElementSnapshotDouble.h" +#import "XCUIApplication+FBHelpers.h" + +static const unsigned long long FBTestTraitIsEditing = 1ULL << 21; + +// A snapshot node with settable focus flags, traits, type and children. +@interface FBFocusSnapshotDouble : XCElementSnapshotDouble +@property (nonatomic, assign) BOOL fakeHasFocus; +@property (nonatomic, assign) BOOL fakeHasKeyboardFocus; +@property (nonatomic, assign) unsigned long long fakeTraits; +@property (nonatomic, assign) XCUIElementType fakeElementType; +@property (nonatomic, copy) NSString *fakeIdentifier; +@property (nonatomic, copy) NSArray *fakeChildren; +@end + +@implementation FBFocusSnapshotDouble + +- (BOOL)hasFocus { return self.fakeHasFocus; } +- (BOOL)hasKeyboardFocus { return self.fakeHasKeyboardFocus; } +- (UIAccessibilityTraits)traits { return (UIAccessibilityTraits)self.fakeTraits; } +- (XCUIElementType)elementType { return self.fakeElementType; } +- (NSString *)identifier { return self.fakeIdentifier ?: @""; } +- (NSArray *)children { return self.fakeChildren ?: @[]; } +- (BOOL)isSelected { return NO; } + +@end + +// fb_a11yNodeForSnapshot:... is private to the category; the test drives the whole walk. +@interface XCUIApplication (FBMobilerunFocusTesting) ++ (NSDictionary *)fb_a11yNodeForSnapshot:(id)snapshot + scale:(CGFloat)scale + packageName:(NSString *)bundleId + context:(NSMutableDictionary *)context; +@end + +@interface FBMobilerunFocusTests : XCTestCase +@end + +@implementation FBMobilerunFocusTests + +static FBFocusSnapshotDouble *FBNode(XCUIElementType type, NSString *identifier, NSArray *children) +{ + FBFocusSnapshotDouble *node = [FBFocusSnapshotDouble new]; + node.fakeElementType = type; + node.fakeIdentifier = identifier; + node.fakeChildren = children; + return node; +} + +static void FBCollectFocused(NSDictionary *node, NSMutableArray *focused) +{ + if ([node[@"isFocused"] boolValue]) { + [focused addObject:node[@"resourceId"]]; + } + for (NSDictionary *child in node[@"children"]) { + FBCollectFocused(child, focused); + } +} + +// Runs the same walk fb_mobilerunA11yStateWithSnapshot:scale:truncated: runs. +static NSDictionary *FBWalk(id root, NSMutableDictionary *context) +{ + id focused = [XCUIApplication fb_mobilerunFocusedSnapshotInTree:root]; + if (nil != focused) { + context[@"focusedSnapshot"] = focused; + } + return [XCUIApplication fb_a11yNodeForSnapshot:root scale:1 packageName:@"com.test" context:context]; +} + +- (void)testWebInputsReportingHasFocusAreNotFocused +{ + // WebKit sets hasFocus on every of a page, focused or not. + FBFocusSnapshotDouble *first = FBNode(XCUIElementTypeTextField, @"first", nil); + FBFocusSnapshotDouble *second = FBNode(XCUIElementTypeTextField, @"second", nil); + first.fakeHasFocus = YES; + second.fakeHasFocus = YES; + FBFocusSnapshotDouble *web = FBNode(XCUIElementTypeWebView, @"web", @[first, second]); + FBFocusSnapshotDouble *app = FBNode(XCUIElementTypeApplication, @"app", @[web]); + + NSMutableDictionary *context = [NSMutableDictionary dictionary]; + NSDictionary *tree = FBWalk((id)app, context); + + NSMutableArray *focused = [NSMutableArray array]; + FBCollectFocused(tree, focused); + XCTAssertEqualObjects(focused, @[]); + XCTAssertNil(context[@"focusedElement"]); + XCTAssertFalse([context[@"isEditable"] boolValue]); +} + +- (void)testOnlyTheKeyboardFocusedWebInputIsFocused +{ + FBFocusSnapshotDouble *first = FBNode(XCUIElementTypeTextField, @"first", nil); + FBFocusSnapshotDouble *second = FBNode(XCUIElementTypeTextField, @"second", nil); + first.fakeHasFocus = YES; + second.fakeHasFocus = YES; + second.fakeHasKeyboardFocus = YES; + FBFocusSnapshotDouble *app = FBNode(XCUIElementTypeApplication, @"app", + @[FBNode(XCUIElementTypeWebView, @"web", @[first, second])]); + + NSMutableDictionary *context = [NSMutableDictionary dictionary]; + NSDictionary *tree = FBWalk((id)app, context); + + NSMutableArray *focused = [NSMutableArray array]; + FBCollectFocused(tree, focused); + XCTAssertEqualObjects(focused, @[@"second"]); + XCTAssertEqualObjects(context[@"focusedElement"][@"resourceId"], @"second"); + XCTAssertTrue([context[@"isEditable"] boolValue]); +} + +- (void)testKeyboardFocusedFieldInASheetIsReported +{ + // A UIKit field never reports hasFocus on iPhone; only keyboard focus marks it. + FBFocusSnapshotDouble *name = FBNode(XCUIElementTypeTextField, @"name", nil); + name.fakeHasKeyboardFocus = YES; + FBFocusSnapshotDouble *sheet = FBNode(XCUIElementTypeOther, @"sheet", + @[FBNode(XCUIElementTypeCell, @"cell", @[name])]); + FBFocusSnapshotDouble *keyboard = FBNode(XCUIElementTypeKeyboard, @"keyboard", nil); + FBFocusSnapshotDouble *app = FBNode(XCUIElementTypeApplication, @"app", + @[FBNode(XCUIElementTypeWindow, @"main", nil), + FBNode(XCUIElementTypeWindow, @"sheetWindow", @[sheet]), + keyboard]); + + NSMutableDictionary *context = [NSMutableDictionary dictionary]; + NSDictionary *tree = FBWalk((id)app, context); + + NSMutableArray *focused = [NSMutableArray array]; + FBCollectFocused(tree, focused); + XCTAssertEqualObjects(focused, @[@"name"]); + XCTAssertEqualObjects(context[@"focusedElement"][@"resourceId"], @"name"); + XCTAssertEqualObjects(context[@"focusedElement"][@"className"], @"XCUIElementTypeTextField"); + XCTAssertTrue([context[@"isEditable"] boolValue]); + XCTAssertTrue([context[@"keyboardVisible"] boolValue]); +} + +- (void)testDeepestKeyboardFocusWins +{ + // An ancestor reporting keyboard focus must not shadow the field that has it. + FBFocusSnapshotDouble *field = FBNode(XCUIElementTypeSecureTextField, @"password", nil); + field.fakeHasKeyboardFocus = YES; + FBFocusSnapshotDouble *container = FBNode(XCUIElementTypeOther, @"container", @[field]); + container.fakeHasKeyboardFocus = YES; + FBFocusSnapshotDouble *app = FBNode(XCUIElementTypeApplication, @"app", @[container]); + + id focused = + [XCUIApplication fb_mobilerunFocusedSnapshotInTree:(id)app]; + XCTAssertEqual((id)focused, (id)field); +} + +- (void)testIsEditingTraitIsTheFallback +{ + FBFocusSnapshotDouble *plain = FBNode(XCUIElementTypeTextField, @"plain", nil); + FBFocusSnapshotDouble *editing = FBNode(XCUIElementTypeOther, @"editing", nil); + editing.fakeTraits = FBTestTraitIsEditing; + FBFocusSnapshotDouble *app = FBNode(XCUIElementTypeApplication, @"app", @[plain, editing]); + + NSMutableDictionary *context = [NSMutableDictionary dictionary]; + NSDictionary *tree = FBWalk((id)app, context); + + NSMutableArray *focused = [NSMutableArray array]; + FBCollectFocused(tree, focused); + XCTAssertEqualObjects(focused, @[@"editing"]); + // Not a text input type, but the IsEditing trait says it takes text. + XCTAssertTrue([context[@"isEditable"] boolValue]); +} + +- (void)testKeyboardFocusOutranksIsEditingTrait +{ + FBFocusSnapshotDouble *editing = FBNode(XCUIElementTypeTextField, @"editing", nil); + editing.fakeTraits = FBTestTraitIsEditing; + FBFocusSnapshotDouble *keyboardFocused = FBNode(XCUIElementTypeTextView, @"keyboardFocused", nil); + keyboardFocused.fakeHasKeyboardFocus = YES; + FBFocusSnapshotDouble *app = FBNode(XCUIElementTypeApplication, @"app", @[editing, keyboardFocused]); + + id focused = + [XCUIApplication fb_mobilerunFocusedSnapshotInTree:(id)app]; + XCTAssertEqual((id)focused, (id)keyboardFocused); +} + +- (void)testNothingFocusedWithoutATree +{ + XCTAssertNil([XCUIApplication fb_mobilerunFocusedSnapshotInTree:nil]); +} + +@end