From 775effb4111d289748d8ccaeb3b8d35cc47fa7a7 Mon Sep 17 00:00:00 2001 From: Ilia Kharebashvili Date: Sun, 25 May 2025 17:11:38 +0300 Subject: [PATCH] draft: experimental implementation of isHittable via snapshot-to-element resolution --- .../Categories/XCUIApplication+FBHelpers.m | 81 ++++++++++++------- .../XCUIElement+FBWebDriverAttributes.m | 34 +------- WebDriverAgentLib/Routing/FBElement.h | 3 - .../FBElementAttributeTests.m | 11 --- 4 files changed, 55 insertions(+), 74 deletions(-) diff --git a/WebDriverAgentLib/Categories/XCUIApplication+FBHelpers.m b/WebDriverAgentLib/Categories/XCUIApplication+FBHelpers.m index 29f86c781..b81459395 100644 --- a/WebDriverAgentLib/Categories/XCUIApplication+FBHelpers.m +++ b/WebDriverAgentLib/Categories/XCUIApplication+FBHelpers.m @@ -38,6 +38,7 @@ #import "XCUIElement+FBWebDriverAttributes.h" #import "XCUIElementQuery.h" #import "FBElementHelpers.h" +#import "XCUIElement+FBUID.h" static NSString* const FBUnknownBundleId = @"unknown"; @@ -180,7 +181,7 @@ - (NSDictionary *)fb_tree - (NSDictionary *)fb_tree:(nullable NSSet *)excludedAttributes { id snapshot = [self fb_standardSnapshot]; - return [self.class dictionaryForElement:snapshot + return [self dictionaryForElement:snapshot recursive:YES excludedAttributes:excludedAttributes]; } @@ -191,9 +192,9 @@ - (NSDictionary *)fb_accessibilityTree return [self.class accessibilityInfoForElement:snapshot]; } -+ (NSDictionary *)dictionaryForElement:(id)snapshot - recursive:(BOOL)recursive - excludedAttributes:(nullable NSSet *)excludedAttributes +- (NSDictionary *)dictionaryForElement:(id)snapshot + recursive:(BOOL)recursive + excludedAttributes:(nullable NSSet *)excludedAttributes { NSMutableDictionary *info = [[NSMutableDictionary alloc] init]; info[@"type"] = [FBElementTypeTransformer shortStringWithElementType:snapshot.elementType]; @@ -203,10 +204,25 @@ + (NSDictionary *)dictionaryForElement:(id)snapshot info[@"value"] = FBValueOrNull(wrappedSnapshot.wdValue); info[@"label"] = FBValueOrNull(wrappedSnapshot.wdLabel); info[@"rect"] = wrappedSnapshot.wdRect; - - NSDictionary *attributeBlocks = [self fb_attributeBlockMapForWrappedSnapshot:wrappedSnapshot - excludedAttributes:excludedAttributes]; + // Flatten the tree to get all snapshots + NSArray> *allSnapshots = [self.class fb_flattenTree:snapshot]; + NSArray *resolvedElements = [self fb_filterDescendantsWithSnapshots:allSnapshots onlyChildren:NO]; + + NSMutableDictionary *uidToElement = [NSMutableDictionary dictionary]; + for (XCUIElement *element in resolvedElements) { + NSString *uid = element.fb_uid; + if (uid != nil) { + uidToElement[uid] = element; + } + } + + NSString *uid = [FBXCElementSnapshotWrapper wdUIDWithSnapshot:wrappedSnapshot.snapshot]; + XCUIElement *resolvedElement = uid != nil ? uidToElement[uid] : nil; + + NSDictionary *attributeBlocks = [self.class fb_attributeBlockMapForWrappedSnapshot:wrappedSnapshot + resolvedElement:resolvedElement + excludedAttributes:excludedAttributes]; NSSet *nonPrefixedKeys = [NSSet setWithObjects: FBExclusionAttributeFrame, @@ -215,14 +231,14 @@ + (NSDictionary *)dictionaryForElement:(id)snapshot nil]; for (NSString *key in attributeBlocks) { - if (excludedAttributes == nil || ![excludedAttributes containsObject:key]) { - NSString *value = ((NSString * (^)(void))attributeBlocks[key])(); - if ([nonPrefixedKeys containsObject:key]) { - info[key] = value; - } else { - info[[NSString stringWithFormat:@"is%@", [key capitalizedString]]] = value; - } + if (excludedAttributes == nil || ![excludedAttributes containsObject:key]) { + NSString *value = ((NSString * (^)(void))attributeBlocks[key])(); + if ([nonPrefixedKeys containsObject:key]) { + info[key] = value; + } else { + info[[NSString stringWithFormat:@"is%@", [key capitalizedString]]] = value; } + } } if (!recursive) { @@ -235,20 +251,26 @@ + (NSDictionary *)dictionaryForElement:(id)snapshot for (id childSnapshot in childElements) { @autoreleasepool { [info[@"children"] addObject:[self dictionaryForElement:childSnapshot - recursive:YES - excludedAttributes:excludedAttributes]]; + recursive:YES + excludedAttributes:excludedAttributes]]; } } } return info; } -// Helper used by `dictionaryForElement:` to assemble attribute value blocks, -// including both common attributes and conditionally included ones like placeholderValue. -+ (NSDictionary *)fb_attributeBlockMapForWrappedSnapshot:(FBXCElementSnapshotWrapper *)wrappedSnapshot - excludedAttributes:(nullable NSSet *)excludedAttributes - ++ (NSArray> *)fb_flattenTree:(id)snapshot +{ + NSMutableArray *result = [NSMutableArray arrayWithObject:snapshot]; + for (id child in snapshot.children) { + [result addObjectsFromArray:[self fb_flattenTree:child]]; + } + return result; +} ++ (NSDictionary *)fb_attributeBlockMapForWrappedSnapshot:(FBXCElementSnapshotWrapper *)wrappedSnapshot + resolvedElement:(nullable XCUIElement *)resolvedElement + excludedAttributes:(nullable NSSet *)excludedAttributes { // Base attributes common to every element NSMutableDictionary *blocks = @@ -281,14 +303,15 @@ + (NSDictionary *)dictionaryForElement:(id)snapshot return (NSString *)FBValueOrNull(wrappedSnapshot.wdPlaceholderValue); }; } - - // Adding isHittable, only if not excluded - if (excludedAttributes == nil || ![excludedAttributes containsObject:FBExclusionAttributeHittable]) { - blocks[FBExclusionAttributeHittable] = ^{ - return [@([wrappedSnapshot isWDNativeHittable]) stringValue]; - }; - } - + + // Add isHittable only if resolvedElement is available and attribute not excluded + // TODO: gate behind explicit config flag + if (resolvedElement && (excludedAttributes == nil || ![excludedAttributes containsObject:FBExclusionAttributeHittable])) { + blocks[FBExclusionAttributeHittable] = ^{ + return [@(resolvedElement.hittable) stringValue]; + }; + } + return [blocks copy]; } diff --git a/WebDriverAgentLib/Categories/XCUIElement+FBWebDriverAttributes.m b/WebDriverAgentLib/Categories/XCUIElement+FBWebDriverAttributes.m index e191ed39c..387057300 100644 --- a/WebDriverAgentLib/Categories/XCUIElement+FBWebDriverAttributes.m +++ b/WebDriverAgentLib/Categories/XCUIElement+FBWebDriverAttributes.m @@ -45,6 +45,9 @@ @implementation XCUIElement (WebDriverAttributesForwarding) - (id)fb_valueForWDAttributeName:(NSString *)name { NSString *wdAttributeName = [FBElementUtils wdAttributeNameForAttributeName:name]; + if ([wdAttributeName isEqualToString:@"isHittable"]) { + return @(self.hittable); + } id snapshot = [self fb_snapshotForAttributeName:wdAttributeName]; return [[FBXCElementSnapshotWrapper ensureWrapped:snapshot] fb_valueForWDAttributeName:name]; } @@ -239,37 +242,6 @@ - (BOOL)isWDHittable return nil == result ? NO : result.hittable; } -/*! Whether the element is truly hittable based on XCUIElement.hittable */ -- (BOOL)isWDNativeHittable -{ - XCUIApplication *app = [XCUIApplication fb_activeApplication]; - XCUIElementQuery *query = [app descendantsMatchingType:self.elementType]; - XCUIElement *matchedElement = nil; - - NSString *identifier = self.identifier; - NSString *label = self.label; - XCUIElementType type = self.elementType; - - // Attempt to match by accessibilityIdentifier first - if (identifier.length > 0) { - XCUIElement *candidate = [query matchingIdentifier:identifier].element; - if (candidate.exists && candidate.elementType == type) { - matchedElement = candidate; - } - } - - // If no match by identifier, try matching by label and type - if (!matchedElement.exists && label.length > 0) { - NSPredicate *predicate = [NSPredicate predicateWithBlock:^BOOL(XCUIElement *el, NSDictionary *_) { - return [el.label isEqualToString:label] && el.elementType == type; - }]; - matchedElement = [[query matchingPredicate:predicate] element]; - } - - // Return hittable status if element is found, otherwise NO - return matchedElement.exists ? matchedElement.hittable : NO; -} - - (NSDictionary *)wdRect { CGRect frame = self.wdFrame; diff --git a/WebDriverAgentLib/Routing/FBElement.h b/WebDriverAgentLib/Routing/FBElement.h index 77624673b..a24d4c2bd 100644 --- a/WebDriverAgentLib/Routing/FBElement.h +++ b/WebDriverAgentLib/Routing/FBElement.h @@ -62,9 +62,6 @@ NS_ASSUME_NONNULL_BEGIN /*! Whether the element is considered hittable based on snapshot hit point */ @property (nonatomic, readonly, getter = isWDHittable) BOOL wdHittable; -/*! Whether the element is truly hittable based on XCUIElement.hittable */ -@property (nonatomic, readonly, getter = isWDNativeHittable) BOOL wdNativeHittable; - /*! Element's index relatively to its parent. Starts from zero */ @property (nonatomic, readonly) NSUInteger wdIndex; diff --git a/WebDriverAgentTests/IntegrationTests/FBElementAttributeTests.m b/WebDriverAgentTests/IntegrationTests/FBElementAttributeTests.m index 584f1f3e8..db956eeab 100644 --- a/WebDriverAgentTests/IntegrationTests/FBElementAttributeTests.m +++ b/WebDriverAgentTests/IntegrationTests/FBElementAttributeTests.m @@ -193,15 +193,4 @@ - (void)testTextViewAttributes XCTAssertEqualObjects(element.wdValue, @"1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901"); } -- (void)testNativeHittableAttribute -{ - XCUIElement *button = self.testedApplication.buttons[@"Button"]; - XCTAssertTrue(button.exists); - - id snapshot = [button fb_standardSnapshot]; - FBXCElementSnapshotWrapper *wrapped = [FBXCElementSnapshotWrapper ensureWrapped:snapshot]; - - XCTAssertEqual([wrapped isWDNativeHittable], button.hittable); -} - @end