From 4f34388903ea1a6cf325ffd469ac66cae9377091 Mon Sep 17 00:00:00 2001 From: Ilia Kharebashvili Date: Thu, 22 May 2025 17:55:44 +0300 Subject: [PATCH 1/2] feat: add isHittable attribute to iOS page source --- .../Categories/XCUIApplication+FBHelpers.m | 14 +++++++- .../XCUIElement+FBWebDriverAttributes.m | 32 +++++++++++++++++++ WebDriverAgentLib/Routing/FBElement.h | 5 ++- .../FBElementAttributeTests.m | 11 +++++++ 4 files changed, 60 insertions(+), 2 deletions(-) diff --git a/WebDriverAgentLib/Categories/XCUIApplication+FBHelpers.m b/WebDriverAgentLib/Categories/XCUIApplication+FBHelpers.m index 1737ff7af..29f86c781 100644 --- a/WebDriverAgentLib/Categories/XCUIApplication+FBHelpers.m +++ b/WebDriverAgentLib/Categories/XCUIApplication+FBHelpers.m @@ -48,6 +48,7 @@ static NSString* const FBExclusionAttributeFocused = @"focused"; static NSString* const FBExclusionAttributePlaceholderValue = @"placeholderValue"; static NSString* const FBExclusionAttributeNativeFrame = @"nativeFrame"; +static NSString* const FBExclusionAttributeHittable = @"hittable"; _Nullable id extractIssueProperty(id issue, NSString *propertyName) { SEL selector = NSSelectorFromString(propertyName); @@ -203,7 +204,9 @@ + (NSDictionary *)dictionaryForElement:(id)snapshot info[@"label"] = FBValueOrNull(wrappedSnapshot.wdLabel); info[@"rect"] = wrappedSnapshot.wdRect; - NSDictionary *attributeBlocks = [self fb_attributeBlockMapForWrappedSnapshot:wrappedSnapshot]; + NSDictionary *attributeBlocks = [self fb_attributeBlockMapForWrappedSnapshot:wrappedSnapshot + excludedAttributes:excludedAttributes]; + NSSet *nonPrefixedKeys = [NSSet setWithObjects: FBExclusionAttributeFrame, @@ -243,6 +246,8 @@ + (NSDictionary *)dictionaryForElement:(id)snapshot // Helper used by `dictionaryForElement:` to assemble attribute value blocks, // including both common attributes and conditionally included ones like placeholderValue. + (NSDictionary *)fb_attributeBlockMapForWrappedSnapshot:(FBXCElementSnapshotWrapper *)wrappedSnapshot + excludedAttributes:(nullable NSSet *)excludedAttributes + { // Base attributes common to every element @@ -277,6 +282,13 @@ + (NSDictionary *)dictionaryForElement:(id)snapshot }; } + // Adding isHittable, only if not excluded + if (excludedAttributes == nil || ![excludedAttributes containsObject:FBExclusionAttributeHittable]) { + blocks[FBExclusionAttributeHittable] = ^{ + return [@([wrappedSnapshot isWDNativeHittable]) stringValue]; + }; + } + return [blocks copy]; } diff --git a/WebDriverAgentLib/Categories/XCUIElement+FBWebDriverAttributes.m b/WebDriverAgentLib/Categories/XCUIElement+FBWebDriverAttributes.m index 8bb068709..e191ed39c 100644 --- a/WebDriverAgentLib/Categories/XCUIElement+FBWebDriverAttributes.m +++ b/WebDriverAgentLib/Categories/XCUIElement+FBWebDriverAttributes.m @@ -22,6 +22,7 @@ #import "FBElementUtils.h" #import "XCTestPrivateSymbols.h" #import "XCUIHitPointResult.h" +#import "XCUIApplication+FBHelpers.h" #define BROKEN_RECT CGRectMake(-1, -1, 0, 0) @@ -238,6 +239,37 @@ - (BOOL)isWDHittable return nil == result ? NO : result.hittable; } +/*! Whether the element is truly hittable based on XCUIElement.hittable */ +- (BOOL)isWDNativeHittable +{ + XCUIApplication *app = [XCUIApplication fb_activeApplication]; + XCUIElementQuery *query = [app descendantsMatchingType:self.elementType]; + XCUIElement *matchedElement = nil; + + NSString *identifier = self.identifier; + NSString *label = self.label; + XCUIElementType type = self.elementType; + + // Attempt to match by accessibilityIdentifier first + if (identifier.length > 0) { + XCUIElement *candidate = [query matchingIdentifier:identifier].element; + if (candidate.exists && candidate.elementType == type) { + matchedElement = candidate; + } + } + + // If no match by identifier, try matching by label and type + if (!matchedElement.exists && label.length > 0) { + NSPredicate *predicate = [NSPredicate predicateWithBlock:^BOOL(XCUIElement *el, NSDictionary *_) { + return [el.label isEqualToString:label] && el.elementType == type; + }]; + matchedElement = [[query matchingPredicate:predicate] element]; + } + + // Return hittable status if element is found, otherwise NO + return matchedElement.exists ? matchedElement.hittable : NO; +} + - (NSDictionary *)wdRect { CGRect frame = self.wdFrame; diff --git a/WebDriverAgentLib/Routing/FBElement.h b/WebDriverAgentLib/Routing/FBElement.h index eed55b8b7..77624673b 100644 --- a/WebDriverAgentLib/Routing/FBElement.h +++ b/WebDriverAgentLib/Routing/FBElement.h @@ -59,9 +59,12 @@ NS_ASSUME_NONNULL_BEGIN /*! Whether element is focused */ @property (nonatomic, readonly, getter = isWDFocused) BOOL wdFocused; -/*! Whether element is hittable */ +/*! Whether the element is considered hittable based on snapshot hit point */ @property (nonatomic, readonly, getter = isWDHittable) BOOL wdHittable; +/*! Whether the element is truly hittable based on XCUIElement.hittable */ +@property (nonatomic, readonly, getter = isWDNativeHittable) BOOL wdNativeHittable; + /*! Element's index relatively to its parent. Starts from zero */ @property (nonatomic, readonly) NSUInteger wdIndex; diff --git a/WebDriverAgentTests/IntegrationTests/FBElementAttributeTests.m b/WebDriverAgentTests/IntegrationTests/FBElementAttributeTests.m index db956eeab..584f1f3e8 100644 --- a/WebDriverAgentTests/IntegrationTests/FBElementAttributeTests.m +++ b/WebDriverAgentTests/IntegrationTests/FBElementAttributeTests.m @@ -193,4 +193,15 @@ - (void)testTextViewAttributes XCTAssertEqualObjects(element.wdValue, @"1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901"); } +- (void)testNativeHittableAttribute +{ + XCUIElement *button = self.testedApplication.buttons[@"Button"]; + XCTAssertTrue(button.exists); + + id snapshot = [button fb_standardSnapshot]; + FBXCElementSnapshotWrapper *wrapped = [FBXCElementSnapshotWrapper ensureWrapped:snapshot]; + + XCTAssertEqual([wrapped isWDNativeHittable], button.hittable); +} + @end From b3f413d3df0bb46ff8baf37f53e4fa72d0cb1a99 Mon Sep 17 00:00:00 2001 From: Ilia Kharebashvili Date: Mon, 26 May 2025 17:50:00 +0300 Subject: [PATCH 2/2] refactor: replace isHittable resolution with snapshot-based heuristic Replaced XCUIElement-based hittability with a lightweight estimation: isWDAccessible && isWDHittable && isWDVisible. This approach avoids performance overhead of resolving live elements, while producing results consistent with vanilla `.hittable` in testing. --- .../Categories/XCUIApplication+FBHelpers.m | 15 +++------ .../XCUIElement+FBWebDriverAttributes.m | 33 +++---------------- WebDriverAgentLib/Routing/FBElement.h | 7 ++-- .../FBElementAttributeTests.m | 2 +- 4 files changed, 15 insertions(+), 42 deletions(-) diff --git a/WebDriverAgentLib/Categories/XCUIApplication+FBHelpers.m b/WebDriverAgentLib/Categories/XCUIApplication+FBHelpers.m index 29f86c781..677178964 100644 --- a/WebDriverAgentLib/Categories/XCUIApplication+FBHelpers.m +++ b/WebDriverAgentLib/Categories/XCUIApplication+FBHelpers.m @@ -204,8 +204,7 @@ + (NSDictionary *)dictionaryForElement:(id)snapshot info[@"label"] = FBValueOrNull(wrappedSnapshot.wdLabel); info[@"rect"] = wrappedSnapshot.wdRect; - NSDictionary *attributeBlocks = [self fb_attributeBlockMapForWrappedSnapshot:wrappedSnapshot - excludedAttributes:excludedAttributes]; + NSDictionary *attributeBlocks = [self fb_attributeBlockMapForWrappedSnapshot:wrappedSnapshot]; NSSet *nonPrefixedKeys = [NSSet setWithObjects: @@ -246,9 +245,6 @@ + (NSDictionary *)dictionaryForElement:(id)snapshot // Helper used by `dictionaryForElement:` to assemble attribute value blocks, // including both common attributes and conditionally included ones like placeholderValue. + (NSDictionary *)fb_attributeBlockMapForWrappedSnapshot:(FBXCElementSnapshotWrapper *)wrappedSnapshot - excludedAttributes:(nullable NSSet *)excludedAttributes - - { // Base attributes common to every element NSMutableDictionary *blocks = @@ -282,12 +278,9 @@ + (NSDictionary *)dictionaryForElement:(id)snapshot }; } - // Adding isHittable, only if not excluded - if (excludedAttributes == nil || ![excludedAttributes containsObject:FBExclusionAttributeHittable]) { - blocks[FBExclusionAttributeHittable] = ^{ - return [@([wrappedSnapshot isWDNativeHittable]) stringValue]; - }; - } + blocks[FBExclusionAttributeHittable] = ^{ + return [@([wrappedSnapshot isWDResolvedHittable]) stringValue]; + }; return [blocks copy]; } diff --git a/WebDriverAgentLib/Categories/XCUIElement+FBWebDriverAttributes.m b/WebDriverAgentLib/Categories/XCUIElement+FBWebDriverAttributes.m index e191ed39c..579d36885 100644 --- a/WebDriverAgentLib/Categories/XCUIElement+FBWebDriverAttributes.m +++ b/WebDriverAgentLib/Categories/XCUIElement+FBWebDriverAttributes.m @@ -239,35 +239,12 @@ - (BOOL)isWDHittable return nil == result ? NO : result.hittable; } -/*! Whether the element is truly hittable based on XCUIElement.hittable */ -- (BOOL)isWDNativeHittable +- (BOOL)isWDResolvedHittable { - XCUIApplication *app = [XCUIApplication fb_activeApplication]; - XCUIElementQuery *query = [app descendantsMatchingType:self.elementType]; - XCUIElement *matchedElement = nil; - - NSString *identifier = self.identifier; - NSString *label = self.label; - XCUIElementType type = self.elementType; - - // Attempt to match by accessibilityIdentifier first - if (identifier.length > 0) { - XCUIElement *candidate = [query matchingIdentifier:identifier].element; - if (candidate.exists && candidate.elementType == type) { - matchedElement = candidate; - } - } - - // If no match by identifier, try matching by label and type - if (!matchedElement.exists && label.length > 0) { - NSPredicate *predicate = [NSPredicate predicateWithBlock:^BOOL(XCUIElement *el, NSDictionary *_) { - return [el.label isEqualToString:label] && el.elementType == type; - }]; - matchedElement = [[query matchingPredicate:predicate] element]; - } - - // Return hittable status if element is found, otherwise NO - return matchedElement.exists ? matchedElement.hittable : NO; + // Snapshot-based estimation of XCUIElement.hittable using accessibility, visibility, and hit point. + // Does not rely on live XCUIElement resolution. + // See discussion: https://github.com/appium/WebDriverAgent/pull/1021 + return self.isWDAccessible && self.isWDHittable && self.isWDVisible; } - (NSDictionary *)wdRect diff --git a/WebDriverAgentLib/Routing/FBElement.h b/WebDriverAgentLib/Routing/FBElement.h index 77624673b..a418cac86 100644 --- a/WebDriverAgentLib/Routing/FBElement.h +++ b/WebDriverAgentLib/Routing/FBElement.h @@ -62,8 +62,11 @@ NS_ASSUME_NONNULL_BEGIN /*! Whether the element is considered hittable based on snapshot hit point */ @property (nonatomic, readonly, getter = isWDHittable) BOOL wdHittable; -/*! Whether the element is truly hittable based on XCUIElement.hittable */ -@property (nonatomic, readonly, getter = isWDNativeHittable) BOOL wdNativeHittable; +/*! + * Returns a snapshot-based estimation of hittability + * using accessibility, visibility, and hit point heuristics. + */ +@property (nonatomic, readonly, getter = isWDResolvedHittable) BOOL wdResolvedHittable; /*! Element's index relatively to its parent. Starts from zero */ @property (nonatomic, readonly) NSUInteger wdIndex; diff --git a/WebDriverAgentTests/IntegrationTests/FBElementAttributeTests.m b/WebDriverAgentTests/IntegrationTests/FBElementAttributeTests.m index 584f1f3e8..4235675cf 100644 --- a/WebDriverAgentTests/IntegrationTests/FBElementAttributeTests.m +++ b/WebDriverAgentTests/IntegrationTests/FBElementAttributeTests.m @@ -201,7 +201,7 @@ - (void)testNativeHittableAttribute id snapshot = [button fb_standardSnapshot]; FBXCElementSnapshotWrapper *wrapped = [FBXCElementSnapshotWrapper ensureWrapped:snapshot]; - XCTAssertEqual([wrapped isWDNativeHittable], button.hittable); + XCTAssertEqual([wrapped isWDResolvedHittable], button.hittable); } @end