From 2821c3a0cc448f73ad79b7ccf12d9e07ba371c80 Mon Sep 17 00:00:00 2001 From: gmegidish Date: Mon, 21 Sep 2026 13:19:11 +0200 Subject: [PATCH 1/2] feat(ios): keep slider, picker, alert, list and navigation elements in ui dump --- devices/devicekit/source.go | 11 ++++-- devices/devicekit/source_test.go | 58 ++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/devices/devicekit/source.go b/devices/devicekit/source.go index d1e0fb0..d050b6d 100644 --- a/devices/devicekit/source.go +++ b/devices/devicekit/source.go @@ -45,7 +45,14 @@ func filterSourceElements(source sourceTreeElement) []types.ScreenElement { childElements = append(childElements, filterSourceElements(child)...) } - acceptedTypes := []string{"TextField", "TextView", "Button", "Switch", "Icon", "SearchField", "StaticText", "Image", "SecureTextField", "WebView"} + acceptedTypes := []string{ + "TextField", "TextView", "Button", "Switch", "Icon", "SearchField", "StaticText", "Image", "SecureTextField", "WebView", + // types that clients map to semantic roles (slider, progressbar, + // combobox, link, tab, header, alert, list, listitem) + "Slider", "ProgressIndicator", "ActivityIndicator", "Picker", "PickerWheel", + "Link", "Tab", "TabBar", "NavigationBar", "Toolbar", "Alert", "Sheet", + "Cell", "Table", "CollectionView", "ScrollView", + } // strip XCUIElementType prefix if present elementType := strings.TrimPrefix(source.Type, "XCUIElementType") @@ -69,7 +76,7 @@ func filterSourceElements(source sourceTreeElement) []types.ScreenElement { } hasIdentifier := source.Label != nil || source.Name != nil || source.RawIdentifier != nil || source.PlaceholderValue != nil - alwaysInclude := elementType == "TextField" || elementType == "TextView" || elementType == "SecureTextField" || elementType == "Button" || elementType == "Switch" || elementType == "SearchField" || elementType == "WebView" + alwaysInclude := elementType == "TextField" || elementType == "TextView" || elementType == "SecureTextField" || elementType == "Button" || elementType == "Switch" || elementType == "SearchField" || elementType == "WebView" || elementType == "Slider" || elementType == "Picker" || elementType == "PickerWheel" if !hasIdentifier && !alwaysInclude { return childElements } diff --git a/devices/devicekit/source_test.go b/devices/devicekit/source_test.go index 8ca6fb2..3df2dc2 100644 --- a/devices/devicekit/source_test.go +++ b/devices/devicekit/source_test.go @@ -408,3 +408,61 @@ func TestFilterSourceElementsOmitsChildrenFromJsonWhenEmpty(t *testing.T) { t.Errorf("expected leaf element to have nil Children, got %+v", output[0].Children) } } + +func TestFilterSourceElementsIncludesTypesBehindSemanticRoles(t *testing.T) { + // Clients map these types to semantic roles (slider, progressbar, alert, + // combobox, list, listitem, tab, link, header). A labeled element of any + // of them must survive filtering, or the role can never match on iOS. + roleTypes := []string{ + "Slider", "ProgressIndicator", "ActivityIndicator", "Picker", "PickerWheel", + "Link", "Tab", "TabBar", "NavigationBar", "Toolbar", "Alert", "Sheet", + "Cell", "Table", "CollectionView", "ScrollView", + } + + for _, roleType := range roleTypes { + output := filterSourceElements(sourceTreeElement{ + Type: "XCUIElementType" + roleType, + Label: strPtr("labeled"), + Rect: visibleRect(0, 100, 300, 44), + }) + + if len(output) != 1 || output[0].Type != roleType { + t.Errorf("expected a labeled %s to be included, got %+v", roleType, output) + } + } +} + +func TestFilterSourceElementsIncludesUnlabeledAdjustableControls(t *testing.T) { + // Sliders and pickers are interactive yet often carry no label, so they + // are always included, like TextField and Switch. + for _, controlType := range []string{"Slider", "Picker", "PickerWheel"} { + output := filterSourceElements(sourceTreeElement{ + Type: "XCUIElementType" + controlType, + Rect: visibleRect(0, 100, 300, 44), + }) + + if len(output) != 1 || output[0].Type != controlType { + t.Errorf("expected an unlabeled %s to be included, got %+v", controlType, output) + } + } +} + +func TestFilterSourceElementsStillDropsUnlabeledContainers(t *testing.T) { + // Unlabeled containers stay out of the dump to keep it small; their + // children are hoisted as before. + output := filterSourceElements(sourceTreeElement{ + Type: "XCUIElementTypeTable", + Rect: visibleRect(0, 0, 402, 874), + Children: []sourceTreeElement{ + { + Type: "XCUIElementTypeButton", + Label: strPtr("Join"), + Rect: visibleRect(338, 194, 31, 20), + }, + }, + }) + + if len(output) != 1 || output[0].Type != "Button" { + t.Fatalf("expected only the hoisted Button, got %+v", output) + } +} From 99c678c08381ea19b2d604d48956b7fcb648a020 Mon Sep 17 00:00:00 2001 From: gmegidish Date: Mon, 21 Sep 2026 13:25:24 +0200 Subject: [PATCH 2/2] fix(ios): treat empty label, name and placeholder as absent when filtering ui dump --- devices/devicekit/source.go | 10 ++++++++-- devices/devicekit/source_test.go | 23 +++++++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/devices/devicekit/source.go b/devices/devicekit/source.go index d050b6d..10d98f9 100644 --- a/devices/devicekit/source.go +++ b/devices/devicekit/source.go @@ -35,6 +35,12 @@ func isVisible(rect sourceTreeElementRect) bool { return rect.X >= 0 && rect.Y >= 0 && rect.Width > 0 && rect.Height > 0 } +// hasText reports whether an optional string attribute carries a value; an +// empty string identifies an element no better than a missing one. +func hasText(value *string) bool { + return value != nil && *value != "" +} + // filterSourceElements converts a WDA source tree into ScreenElements, // preserving hierarchy: filtered descendants of an accepted element become its // Children, while descendants of rejected elements are hoisted to the nearest @@ -67,7 +73,7 @@ func filterSourceElements(source sourceTreeElement) []types.ScreenElement { // elements explicitly tagged with accessibilityIdentifier are always // included, regardless of type, see https://github.com/mobile-next/mobilecli/issues/341 - if source.RawIdentifier != nil && *source.RawIdentifier != "" { + if hasText(source.RawIdentifier) { typeAccepted = true } @@ -75,7 +81,7 @@ func filterSourceElements(source sourceTreeElement) []types.ScreenElement { return childElements } - hasIdentifier := source.Label != nil || source.Name != nil || source.RawIdentifier != nil || source.PlaceholderValue != nil + hasIdentifier := hasText(source.Label) || hasText(source.Name) || hasText(source.RawIdentifier) || hasText(source.PlaceholderValue) alwaysInclude := elementType == "TextField" || elementType == "TextView" || elementType == "SecureTextField" || elementType == "Button" || elementType == "Switch" || elementType == "SearchField" || elementType == "WebView" || elementType == "Slider" || elementType == "Picker" || elementType == "PickerWheel" if !hasIdentifier && !alwaysInclude { return childElements diff --git a/devices/devicekit/source_test.go b/devices/devicekit/source_test.go index 3df2dc2..b7de542 100644 --- a/devices/devicekit/source_test.go +++ b/devices/devicekit/source_test.go @@ -466,3 +466,26 @@ func TestFilterSourceElementsStillDropsUnlabeledContainers(t *testing.T) { t.Fatalf("expected only the hoisted Button, got %+v", output) } } + +func TestFilterSourceElementsTreatsEmptyIdentifiersAsAbsent(t *testing.T) { + // A container whose label/name/placeholder are present but empty is as + // anonymous as one without them: drop it and hoist its children. + output := filterSourceElements(sourceTreeElement{ + Type: "XCUIElementTypeTable", + Label: strPtr(""), + Name: strPtr(""), + PlaceholderValue: strPtr(""), + Rect: visibleRect(0, 0, 402, 874), + Children: []sourceTreeElement{ + { + Type: "XCUIElementTypeButton", + Label: strPtr("Join"), + Rect: visibleRect(338, 194, 31, 20), + }, + }, + }) + + if len(output) != 1 || output[0].Type != "Button" { + t.Fatalf("expected only the hoisted Button, got %+v", output) + } +}