Problem
I am developing a WebDriver agent for automation and using dictionaryRepresentation to retrieve the coordinates of the iOS app hierarchy. However, I am encountering an issue with the accuracy of the x and y coordinates.
Approach Tried
I tested the setup on:
iPhone 12 Pro Max (iOS 16.2): Accuracy issues with the coordinates were observed.
iPhone SE (3rd Generation) (iOS 16.2): Coordinates were accurate for tap actions, with no issues identified.
Observation
It appears that devices with fingerprint biometric authentication provide accurate coordinates.
Can anyone help here to understand is there anything wrong in the code. Are do we have to adjust frame of the element for different devices?
Sample Code
- (NSDictionary *)json_tree
{
NSDictionary<XCUIElementAttributeName, id>
*dictionaryRepresentation = [[self snapshotWithError:nil] dictionaryRepresentation];
return [self.class dictionaryForElementAttributes:dictionaryRepresentation recursive:YES];
}
// This method converts the dictionary to CGRect, handling any broken frame values (e.g., Infinity)
+ (CGRect)handleBrokenFrameFromDict:(id)frameDict {
if ([frameDict isKindOfClass:[NSDictionary class]]) {
CGFloat originX = [frameDict[@"X"] floatValue];
CGFloat originY = [frameDict[@"Y"] floatValue];
CGFloat sizeWidth = [frameDict[@"Width"] floatValue];
CGFloat sizeHeight = [frameDict[@"Height"] floatValue];
CGRect frame = CGRectMake(originX, originY, sizeWidth, sizeHeight);
// Replace Infinity values with CGRectZero
return (isinf(frame.size.width) || isinf(frame.size.height)
|| isinf(frame.origin.x) || isinf(frame.origin.y))
? CGRectZero // or another predefined constant like BROKEN_RECT
: CGRectIntegral(frame);
}
return CGRectZero; // If frameDict is not a valid dictionary, return CGRectZero
}
// This method converts CGRect into a dictionary representation for "rect"
+ (NSDictionary *)rectDictionaryFromCGRect:(CGRect)rect {
return @{
@"x": @(rect.origin.x),
@"y": @(rect.origin.y),
@"width": @(rect.size.width),
@"height": @(rect.size.height)
};
}
+ (NSString *)label:(NSDictionary<XCUIElementAttributeName, id> *)dict
{
XCUIElementType elementType = [dict[XCUIElementAttributeNameElementType] intValue];
NSString *label = dict[XCUIElementAttributeNameLabel];
if (elementType == XCUIElementTypeTextField || elementType == XCUIElementTypeSecureTextField) {
return label;
}
return FBTransferEmptyStringToNil(label);
}
+ (NSString *)name:(NSDictionary<XCUIElementAttributeName, id> *)dict
{
NSString *identifier = dict[XCUIElementAttributeNameIdentifier];
if (nil != identifier && identifier.length != 0) {
return identifier;
}
NSString *label = dict[XCUIElementAttributeNameLabel];
return FBTransferEmptyStringToNil(label);
}
+ (NSString *)value:(NSDictionary<XCUIElementAttributeName, id> *)dict
{
id value = dict[XCUIElementAttributeNameValue];
XCUIElementType elementType = [dict[XCUIElementAttributeNameElementType] intValue];
if (elementType == XCUIElementTypeStaticText) {
NSString *label = [self label:dict];
value = FBFirstNonEmptyValue(value, label);
} else if (elementType == XCUIElementTypeButton) {
NSNumber *isSelected = [dict[XCUIElementAttributeNameSelected] boolValue] ? @YES : nil;
value = FBFirstNonEmptyValue(value, isSelected);
} else if (elementType == XCUIElementTypeSwitch) {
value = @([value boolValue]);
} else if (elementType == XCUIElementTypeTextView ||
elementType == XCUIElementTypeTextField ||
elementType == XCUIElementTypeSecureTextField) {
NSString *placeholderValue = dict[XCUIElementAttributeNamePlaceholderValue];
value = FBFirstNonEmptyValue(value, placeholderValue);
}
value = FBTransferEmptyStringToNil(value);
if (value) {
value = [NSString stringWithFormat:@"%@", value];
}
return value;
}
+ (NSDictionary *)dictionaryForElementAttributes:(NSDictionary<XCUIElementAttributeName, id> *)dict recursive:(BOOL)recursive
{
NSMutableDictionary *info = [[NSMutableDictionary alloc] init];
info[@"type"] = [FBElementTypeTransformer shortStringWithElementType:[dict[XCUIElementAttributeNameElementType] intValue]];
info[@"rawIdentifier"] = FBValueOrNull([dict[XCUIElementAttributeNameIdentifier] isEqual:@""] ? nil : dict[XCUIElementAttributeNameIdentifier]);
info[@"name"] = FBValueOrNull([self name:dict]);
info[@"value"] = FBValueOrNull([self value:dict]);
info[@"label"] = FBValueOrNull([self label:dict]);
// Handle the frame value
CGRect frame = [self handleBrokenFrameFromDict:dict[XCUIElementAttributeNameFrame]];
info[@"frame"] = NSStringFromCGRect(frame);
// Add the rect value
info[@"rect"] = [self rectDictionaryFromCGRect:frame];
info[@"isEnabled"] = [@([dict[XCUIElementAttributeNameEnabled] boolValue]) stringValue];
// visible
// accessible
info[@"isFocused"] = [@([dict[XCUIElementAttributeNameHasFocus] boolValue]) stringValue];
if (!recursive) {
return info.copy;
}
NSArray<NSDictionary<XCUIElementAttributeName, id> *> *childElements = [dict[XCUIElementAttributeNameChildren] isKindOfClass:[NSArray class]] ? dict[XCUIElementAttributeNameChildren] : @[];
if ([childElements count]) {
info[@"children"] = [[NSMutableArray alloc] init];
for (NSDictionary<XCUIElementAttributeName, id> * childSnapshot in childElements) {
[info[@"children"] addObject:[self dictionaryForElementAttributes:childSnapshot recursive:YES]];
}
}
return info;
}
