From 1c17de6592ab5761721e5c3a152b9f1711c7aaf2 Mon Sep 17 00:00:00 2001 From: wobsoriano Date: Thu, 10 Sep 2026 20:47:09 -0700 Subject: [PATCH 01/14] test(expo): keep maestro reports and debug output from every run --- .changeset/maestro-ci-hardening.md | 2 ++ .github/workflows/expo-native-build.yml | 31 +++++++++++----------- integration/tests/expo-native/run-flows.sh | 6 ++++- 3 files changed, 22 insertions(+), 17 deletions(-) create mode 100644 .changeset/maestro-ci-hardening.md diff --git a/.changeset/maestro-ci-hardening.md b/.changeset/maestro-ci-hardening.md new file mode 100644 index 00000000000..a845151cc84 --- /dev/null +++ b/.changeset/maestro-ci-hardening.md @@ -0,0 +1,2 @@ +--- +--- diff --git a/.github/workflows/expo-native-build.yml b/.github/workflows/expo-native-build.yml index ce6a95119b6..0111cd0b1e7 100644 --- a/.github/workflows/expo-native-build.yml +++ b/.github/workflows/expo-native-build.yml @@ -28,6 +28,8 @@ env: E2E_INSTANCE_NAME: clerkstage-with-native-components BAPI_URL: https://api.clerkstage.dev MAESTRO_VERSION: '2.8.0' + # Skips a blocking network call on every CLI start. + MAESTRO_CLI_NO_ANALYTICS: '1' jobs: native-build: @@ -233,22 +235,17 @@ jobs: - name: Install maestro CLI if: steps.keys.outputs.pk != '' run: | - set -o pipefail if [ -x "$HOME/.maestro/bin/maestro" ]; then echo "Using cached Maestro" else - installed=0 - for i in 1 2 3; do - if curl -fLs --retry 3 --retry-delay 5 "https://get.maestro.mobile.dev" | MAESTRO_VERSION="$MAESTRO_VERSION" bash; then - if [ -x "$HOME/.maestro/bin/maestro" ]; then installed=1; break; fi - fi - echo "Maestro install attempt $i failed (or binary missing); retrying" - sleep 5 - done - [ "$installed" = 1 ] || { echo "::error::Maestro install failed after 3 attempts"; exit 1; } + archive="$RUNNER_TEMP/maestro.zip" + curl -fLs --retry 3 --retry-delay 5 -o "$archive" \ + "https://github.com/mobile-dev-inc/Maestro/releases/download/cli-$MAESTRO_VERSION/maestro.zip" + unzip -q "$archive" -d "$RUNNER_TEMP/maestro-dist" + mv "$RUNNER_TEMP/maestro-dist/maestro" "$HOME/.maestro" fi echo "$HOME/.maestro/bin" >> "$GITHUB_PATH" - "$HOME/.maestro/bin/maestro" --version + test "$("$HOME/.maestro/bin/maestro" --version)" = "$MAESTRO_VERSION" - name: Boot iOS simulator id: sim @@ -319,7 +316,7 @@ jobs: LOG_PID=$! cd "$GITHUB_WORKSPACE/$E2E_DIR" rc=0 - MAESTRO_DEBUG_OUTPUT="$RUNNER_TEMP/maestro-debug" ./run-flows.sh \ + MAESTRO_DEBUG_OUTPUT="$RUNNER_TEMP/maestro-debug" MAESTRO_UDID="$SIM_UDID" ./run-flows.sh \ xcrun simctl terminate "$SIM_UDID" com.clerk.exponativebuildfixture || rc=$? kill "$LOG_PID" 2>/dev/null || true exit "$rc" @@ -386,9 +383,10 @@ jobs: -H "Authorization: Bearer $CLERK_SECRET_KEY" || true # Test reports record flow env (and typed input) in plaintext; - # add-mask only covers step logs, not artifact contents. + # add-mask only covers step logs, not artifact contents. Must run whenever + # the upload does, or a passing run uploads the password. - name: Scrub test credentials from Maestro debug output - if: always() && (steps.run_e2e_ios.outcome == 'failure' || steps.run_e2e_android.outcome == 'failure') + if: always() && steps.user.outputs.user_id != '' env: CLERK_TEST_PASSWORD: ${{ steps.user.outputs.password }} run: | @@ -399,12 +397,13 @@ jobs: \( -name '*.html' -o -name '*.json' -o -name '*.log' -o -name '*.txt' -o -name '*.xml' -o -name '*.yaml' \) \ -exec perl -pi -e 's/\Q$ENV{CLERK_TEST_PASSWORD}\E/[REDACTED]/g' {} + - - name: Upload Maestro artifacts on e2e failure - if: always() && (steps.run_e2e_ios.outcome == 'failure' || steps.run_e2e_android.outcome == 'failure') + - name: Upload Maestro artifacts + if: always() && steps.user.outputs.user_id != '' uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: maestro-${{ matrix.platform }} path: ${{ runner.temp }}/maestro-debug + if-no-files-found: warn retention-days: 7 - name: Report e2e outcome diff --git a/integration/tests/expo-native/run-flows.sh b/integration/tests/expo-native/run-flows.sh index b3eb7b7f64c..1cc9dd99f70 100755 --- a/integration/tests/expo-native/run-flows.sh +++ b/integration/tests/expo-native/run-flows.sh @@ -11,7 +11,8 @@ # Local: ./run-flows.sh # # Required env: CLERK_TEST_EMAIL, CLERK_TEST_PASSWORD -# Optional env: MAESTRO_DEBUG_OUTPUT (directory for CI debug artifacts) +# Optional env: MAESTRO_DEBUG_OUTPUT (directory for CI debug artifacts), +# MAESTRO_UDID (target one device when several are connected) set -euo pipefail cd "$(dirname "${BASH_SOURCE[0]}")" @@ -34,8 +35,11 @@ run_flow() { local output_root=${MAESTRO_DEBUG_OUTPUT:-${TMPDIR:-/tmp}/clerk-expo-maestro-runner} maestro test \ + ${MAESTRO_UDID:+--udid "$MAESTRO_UDID"} \ --debug-output "$output_root/$output_name" \ --flatten-debug-output \ + --format JUNIT \ + --output "$output_root/$output_name/report.xml" \ "$@" } From ed98ed3a814f2e4968e97b3ff6276c6e233c4578 Mon Sep 17 00:00:00 2001 From: wobsoriano Date: Thu, 10 Sep 2026 22:28:15 -0700 Subject: [PATCH 02/14] ci(e2e): split the maestro flows across two devices per job Each e2e job now boots two devices and runs the flows once with --shard-split across them, then reruns only the flows the JUnit report marked failed. That keeps the per-flow retry while paying Maestro's startup once instead of once per flow. Two iPhone 17 Pro simulators on the 6 vCPU macOS runner halved the flow phase, 549s to 274s. The simulators boot right after checkout so they settle during the native build, and the keyboard and animation defaults move into boot-ios-simulators.sh, which replaces the simulator action. On Android run-android-flows.sh starts the extra emulator as a read-only instance of the booted AVD. MAESTRO_SHARDS sets the device count for both. --- .github/workflows/expo-native-build.yml | 59 +++---- .../tests/expo-native/boot-ios-simulators.sh | 53 ++++++ ...e-host-back.yaml => embedded-profile.yaml} | 0 .../session-persists-across-restart.yaml | 22 --- .../flows/user-profile-custom-pages.yaml | 45 ----- .../tests/expo-native/run-android-flows.sh | 66 ++++++-- integration/tests/expo-native/run-flows.sh | 156 ++++++++++-------- 7 files changed, 219 insertions(+), 182 deletions(-) create mode 100755 integration/tests/expo-native/boot-ios-simulators.sh rename integration/tests/expo-native/flows/{embedded-profile-host-back.yaml => embedded-profile.yaml} (100%) delete mode 100644 integration/tests/expo-native/flows/session-persists-across-restart.yaml delete mode 100644 integration/tests/expo-native/flows/user-profile-custom-pages.yaml diff --git a/.github/workflows/expo-native-build.yml b/.github/workflows/expo-native-build.yml index 0111cd0b1e7..109d6eaa547 100644 --- a/.github/workflows/expo-native-build.yml +++ b/.github/workflows/expo-native-build.yml @@ -30,6 +30,10 @@ env: MAESTRO_VERSION: '2.8.0' # Skips a blocking network call on every CLI start. MAESTRO_CLI_NO_ANALYTICS: '1' + # Devices per e2e job; the flows are split across them. A second driver + # took up to 130s to come up beside the first, past the default startup timeout. + MAESTRO_SHARDS: '2' + MAESTRO_DRIVER_STARTUP_TIMEOUT: '180000' jobs: native-build: @@ -68,6 +72,10 @@ jobs: with: persist-credentials: false + - name: Boot iOS simulators + if: matrix.platform == 'ios' && matrix.run-e2e == true + run: ${{ env.E2E_DIR }}/boot-ios-simulators.sh boot "$MAESTRO_SHARDS" + - name: Compute native build cache key id: native-build-key env: @@ -247,13 +255,6 @@ jobs: echo "$HOME/.maestro/bin" >> "$GITHUB_PATH" test "$("$HOME/.maestro/bin/maestro" --version)" = "$MAESTRO_VERSION" - - name: Boot iOS simulator - id: sim - if: matrix.platform == 'ios' && steps.keys.outputs.pk != '' - uses: futureware-tech/simulator-action@e89aa8f93d3aec35083ff49d2854d07f7186f7f5 # v5 - with: - model: 'iPhone 17 Pro' - # Provisioned as late as possible so live credentials exist for the # shortest window. - name: Provision test user via BAPI @@ -290,35 +291,23 @@ jobs: env: CLERK_TEST_EMAIL: ${{ steps.user.outputs.email }} CLERK_TEST_PASSWORD: ${{ steps.user.outputs.password }} - SIM_UDID: ${{ steps.sim.outputs.udid }} run: | - echo "Using simulator $SIM_UDID" - # Kill animations + predictive keyboard: animations add latency to - # every tap; predictive text hijacks inputText targets. - xcrun simctl spawn "$SIM_UDID" defaults write com.apple.UIKit UIAnimationDragCoefficient -float 0.01 || true - xcrun simctl spawn "$SIM_UDID" defaults write -g ApplePersistenceIgnoreState -bool YES || true - xcrun simctl spawn "$SIM_UDID" defaults write com.apple.keyboard.ContinuousPath -bool NO || true - xcrun simctl spawn "$SIM_UDID" defaults write com.apple.keyboard.AutoCapitalization -bool NO || true - xcrun simctl spawn "$SIM_UDID" defaults write com.apple.keyboard.AutoCorrection -bool NO || true - xcrun simctl spawn "$SIM_UDID" defaults write com.apple.keyboard.Prediction -bool NO || true - # The one-time keyboard tutorial sheets carry their own Continue - # button, which can hijack taps on the AuthView's Continue. - for key in DidShowContinuousPathIntroduction DidShowGestureKeyboardIntroduction KeyboardDidShowProductivityTutorial UIKeyboardDidShowInternationalInfoIntroduction; do - xcrun simctl spawn "$SIM_UDID" defaults write com.apple.keyboard.preferences "$key" -bool YES || true - done - xcrun simctl install "$SIM_UDID" ios/build/Build/Products/Release-iphonesimulator/ClerkExpoNativeBuildFixture.app - # Stream the app's console output into the debug artifact so a hang has - # actionable evidence (keychain/network errors) instead of just screenshots. + "$GITHUB_WORKSPACE/$E2E_DIR/boot-ios-simulators.sh" wait mkdir -p "$RUNNER_TEMP/maestro-debug" - xcrun simctl spawn "$SIM_UDID" log stream --style compact \ - --predicate 'processImagePath CONTAINS "ClerkExpoNativeBuildFixture"' \ - > "$RUNNER_TEMP/maestro-debug/sim-console.log" 2>&1 & - LOG_PID=$! - cd "$GITHUB_WORKSPACE/$E2E_DIR" + log_pids=() + for udid in ${MAESTRO_UDID//,/ }; do + xcrun simctl install "$udid" ios/build/Build/Products/Release-iphonesimulator/ClerkExpoNativeBuildFixture.app + # Stream the app's console output into the debug artifact so a hang + # has actionable evidence (keychain/network errors), not just screenshots. + xcrun simctl spawn "$udid" log stream --style compact \ + --predicate 'processImagePath CONTAINS "ClerkExpoNativeBuildFixture"' \ + > "$RUNNER_TEMP/maestro-debug/$udid-console.log" 2>&1 & + log_pids+=($!) + done rc=0 - MAESTRO_DEBUG_OUTPUT="$RUNNER_TEMP/maestro-debug" MAESTRO_UDID="$SIM_UDID" ./run-flows.sh \ - xcrun simctl terminate "$SIM_UDID" com.clerk.exponativebuildfixture || rc=$? - kill "$LOG_PID" 2>/dev/null || true + PLATFORM=ios MAESTRO_DEBUG_OUTPUT="$RUNNER_TEMP/maestro-debug" \ + "$GITHUB_WORKSPACE/$E2E_DIR/run-flows.sh" || rc=$? + kill "${log_pids[@]}" 2>/dev/null || true exit "$rc" - name: Enable KVM @@ -362,7 +351,9 @@ jobs: target: google_apis arch: x86_64 force-avd-creation: false - emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim + # read-only so the extra shard emulators can share this AVD: a + # writable instance holds a lock the others wait on forever. + emulator-options: -read-only -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim disable-animations: true # The action runs each script line in a separate sh -c; the folded # scalar (>-) plus && keeps everything in one shell invocation. diff --git a/integration/tests/expo-native/boot-ios-simulators.sh b/integration/tests/expo-native/boot-ios-simulators.sh new file mode 100755 index 00000000000..419bca907c4 --- /dev/null +++ b/integration/tests/expo-native/boot-ios-simulators.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Boots N simulators of one model on the newest installed iOS runtime and +# configures each for Maestro. `boot` returns as soon as the boots are issued +# so they overlap the native build; `wait` blocks until every one is usable. +# +# Usage: ./boot-ios-simulators.sh boot [model] # exports MAESTRO_UDID +# ./boot-ios-simulators.sh wait # reads MAESTRO_UDID +set -euo pipefail + +boot() { + local count=$1 model=${2:-iPhone 17 Pro} + local runtime device_type udids=() i + # The runtime matching the SDK the app was built with, so a newer beta + # runtime installed on the runner does not change the OS under test. + sdk=$(xcrun --sdk iphonesimulator --show-sdk-version) + runtime=$(xcrun simctl list runtimes available -j | jq -r --arg v "$sdk" \ + '[.runtimes[] | select(.platform == "iOS")] | (map(select(.version == $v)) + .) | first | .identifier') + device_type=$(xcrun simctl list devicetypes -j | jq -r --arg m "$model" '.devicetypes[] | select(.name == $m) | .identifier') + for i in $(seq 1 "$count"); do + udids+=("$(xcrun simctl create "$model $i" "$device_type" "$runtime")") + xcrun simctl boot "${udids[$((i - 1))]}" + done + MAESTRO_UDID=$(IFS=,; echo "${udids[*]}") + echo "Booting $count x $model on $runtime: $MAESTRO_UDID" + if [ -n "${GITHUB_ENV:-}" ]; then echo "MAESTRO_UDID=$MAESTRO_UDID" >> "$GITHUB_ENV"; fi +} + +wait_ready() { + local udid key + IFS=, read -r -a udids <<< "${MAESTRO_UDID:?MAESTRO_UDID is required}" + for udid in "${udids[@]}"; do + xcrun simctl bootstatus "$udid" -b + # Kill animations + predictive keyboard: animations add latency to every + # tap; predictive text hijacks inputText targets. + xcrun simctl spawn "$udid" defaults write com.apple.UIKit UIAnimationDragCoefficient -float 0.01 || true + xcrun simctl spawn "$udid" defaults write -g ApplePersistenceIgnoreState -bool YES || true + xcrun simctl spawn "$udid" defaults write com.apple.keyboard.ContinuousPath -bool NO || true + xcrun simctl spawn "$udid" defaults write com.apple.keyboard.AutoCapitalization -bool NO || true + xcrun simctl spawn "$udid" defaults write com.apple.keyboard.AutoCorrection -bool NO || true + xcrun simctl spawn "$udid" defaults write com.apple.keyboard.Prediction -bool NO || true + # The one-time keyboard tutorial sheets carry their own Continue button, + # which can hijack taps on the AuthView's Continue. + for key in DidShowContinuousPathIntroduction DidShowGestureKeyboardIntroduction KeyboardDidShowProductivityTutorial UIKeyboardDidShowInternationalInfoIntroduction; do + xcrun simctl spawn "$udid" defaults write com.apple.keyboard.preferences "$key" -bool YES || true + done + done +} + +case "${1:-}" in + boot) boot "${2:?count is required}" "${3:-}" ;; + wait) wait_ready ;; + *) echo "usage: $0 boot [model] | wait" >&2; exit 2 ;; +esac diff --git a/integration/tests/expo-native/flows/embedded-profile-host-back.yaml b/integration/tests/expo-native/flows/embedded-profile.yaml similarity index 100% rename from integration/tests/expo-native/flows/embedded-profile-host-back.yaml rename to integration/tests/expo-native/flows/embedded-profile.yaml diff --git a/integration/tests/expo-native/flows/session-persists-across-restart.yaml b/integration/tests/expo-native/flows/session-persists-across-restart.yaml deleted file mode 100644 index 95d8c900f6d..00000000000 --- a/integration/tests/expo-native/flows/session-persists-across-restart.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# Sign in via the native AuthView, restart WITHOUT clearing state, and assert -# the session is restored from secure-store with no re-auth (bridge + token -# cache persistence). -appId: com.clerk.exponativebuildfixture -name: Session persists across app restart ---- -- runFlow: subflows/open-app.yaml -- tapOn: - id: 'open-auth-view-button' -- runFlow: subflows/sign-in-email-password.yaml -- runFlow: subflows/assert-signed-in.yaml -- stopApp -- launchApp -- extendedWaitUntil: - visible: 'signed in' - timeout: 45000 -- assertVisible: - id: 'user-id' -# Leave the app signed out for whichever flow runs next. -- tapOn: - id: 'sign-out-button' -- runFlow: subflows/assert-signed-out.yaml diff --git a/integration/tests/expo-native/flows/user-profile-custom-pages.yaml b/integration/tests/expo-native/flows/user-profile-custom-pages.yaml deleted file mode 100644 index 0b408084b51..00000000000 --- a/integration/tests/expo-native/flows/user-profile-custom-pages.yaml +++ /dev/null @@ -1,45 +0,0 @@ -# A custom page's React Native content is mounted as a child of the native host -# and rehosted into the destination its row pushes, then survives the trip back. -appId: com.clerk.exponativebuildfixture -name: UserProfileView renders a custom page ---- -- runFlow: subflows/open-app.yaml -- tapOn: - id: 'open-auth-view-button' -- runFlow: subflows/sign-in-email-password.yaml -- runFlow: subflows/assert-signed-in.yaml -- tapOn: - id: 'open-embedded-profile-button' -# Retrying wait: the row list lands a frame after the native profile paints. -- extendedWaitUntil: - visible: 'E2E Custom Page' - timeout: 20000 -- tapOn: - text: 'E2E Custom Page' -- extendedWaitUntil: - visible: 'Rehosted RN body' - timeout: 15000 -# The Android destination is a bare AndroidView with no back chrome, unlike the -# iOS page which clerk-ios pushes onto its own NavigationStack. -- runFlow: - when: - platform: iOS - commands: - - tapOn: 'Back' -- runFlow: - when: - platform: Android - commands: - - back -- extendedWaitUntil: - visible: 'E2E Custom Page' - timeout: 15000 -# The host chevron firing onHostBack, which reads 'Back' on both platforms. -- tapOn: 'Back' -- extendedWaitUntil: - visible: - id: 'open-embedded-profile-button' - timeout: 15000 -- tapOn: - id: 'sign-out-button' -- runFlow: subflows/assert-signed-out.yaml diff --git a/integration/tests/expo-native/run-android-flows.sh b/integration/tests/expo-native/run-android-flows.sh index 29a8e165b84..b98aba5d4df 100755 --- a/integration/tests/expo-native/run-android-flows.sh +++ b/integration/tests/expo-native/run-android-flows.sh @@ -1,4 +1,11 @@ #!/usr/bin/env bash +# Installs the APK on every emulator, starting extra instances of the booted +# AVD when MAESTRO_SHARDS asks for more than one, then runs the flows across +# them. Meant to run inside reactivecircus/android-emulator-runner. +# +# Usage: ./run-android-flows.sh +# Optional env: MAESTRO_SHARDS (default 1), AVD_NAME (default test), +# MAESTRO_DEBUG_OUTPUT set -euo pipefail cd "$(dirname "${BASH_SOURCE[0]}")" @@ -8,21 +15,56 @@ command -v adb >/dev/null 2>&1 || { exit 1 } -adb install -r "$apk_path" +shards=${MAESTRO_SHARDS:-1} +devices=(emulator-5554) +sdk=$(dirname "$(dirname "$(command -v adb)")") +debug=${MAESTRO_DEBUG_OUTPUT:-${TMPDIR:-/tmp}/clerk-expo-maestro-runner} +mkdir -p "$debug" -logcat_pid= +# The action boots one emulator on 5554; extra shards start the same AVD +# read-only on the next even ports so they share its snapshot. +for ((i = 1; i < shards; i++)); do + port=$((5554 + i * 2)) + "$sdk/emulator/emulator" -avd "${AVD_NAME:-test}" -read-only -port "$port" \ + -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -no-snapshot-save \ + > "$debug/emulator-$port.log" 2>&1 & + devices+=("emulator-$port") +done + +# Bounded, because adb wait-for-device on an emulator that died at startup +# never returns and the job would sit until its timeout with no output. +wait_for_boot() { + local device=$1 elapsed=0 + until [ "$(adb -s "$device" shell getprop sys.boot_completed 2>/dev/null | tr -d '\r')" = 1 ]; do + if [ "$elapsed" -ge 240 ]; then + echo "::error::$device did not boot within ${elapsed}s" + [ -f "$debug/$device.log" ] && tail -n 40 "$debug/$device.log" + return 1 + fi + sleep 5 + elapsed=$((elapsed + 5)) + done + echo "$device booted" +} +for device in "${devices[@]}"; do + wait_for_boot "$device" + adb -s "$device" install -r "$apk_path" +done + +logcat_pids=() stop_logcat() { - [ -n "$logcat_pid" ] || return 0 - kill "$logcat_pid" >/dev/null 2>&1 || true - wait "$logcat_pid" 2>/dev/null || true + local pid + for pid in ${logcat_pids[@]+"${logcat_pids[@]}"}; do + kill "$pid" >/dev/null 2>&1 || true + wait "$pid" 2>/dev/null || true + done } trap stop_logcat EXIT -if [ -n "${MAESTRO_DEBUG_OUTPUT:-}" ]; then - mkdir -p "$MAESTRO_DEBUG_OUTPUT" - adb logcat -c || true - adb logcat -v threadtime > "$MAESTRO_DEBUG_OUTPUT/android-logcat.log" 2>&1 & - logcat_pid=$! -fi +for device in "${devices[@]}"; do + adb -s "$device" logcat -c || true + adb -s "$device" logcat -v threadtime > "$debug/$device-logcat.log" 2>&1 & + logcat_pids+=($!) +done -./run-flows.sh adb shell am force-stop com.clerk.exponativebuildfixture +PLATFORM=android MAESTRO_UDID=$(IFS=,; echo "${devices[*]}") ./run-flows.sh diff --git a/integration/tests/expo-native/run-flows.sh b/integration/tests/expo-native/run-flows.sh index 1cc9dd99f70..be28191c466 100755 --- a/integration/tests/expo-native/run-flows.sh +++ b/integration/tests/expo-native/run-flows.sh @@ -1,18 +1,16 @@ #!/usr/bin/env bash # Runs every top-level Maestro flow (flows/*.yaml; flows/subflows/ are -# runFlow-only pieces) as one CLI invocation per flow, so a hang or crash in -# one flow can't poison the rest, with one clean-state retry per flow. -# Whole-flow retry can mask app instability (the Maestro docs discourage it), -# so it is capped at a single retry purely to absorb emulator/simulator flake. +# runFlow-only pieces) once across the connected devices, then reruns only the +# flows that failed. Whole-flow retry can mask app instability (the Maestro +# docs discourage it), so it is capped at a single rerun purely to absorb +# emulator/simulator flake. # -# Usage: ./run-flows.sh [force-stop command...] -# CI iOS: ./run-flows.sh xcrun simctl terminate "$SIM_UDID" com.clerk.exponativebuildfixture -# CI Android: ./run-flows.sh adb shell am force-stop com.clerk.exponativebuildfixture -# Local: ./run-flows.sh +# Usage: PLATFORM= ./run-flows.sh # -# Required env: CLERK_TEST_EMAIL, CLERK_TEST_PASSWORD -# Optional env: MAESTRO_DEBUG_OUTPUT (directory for CI debug artifacts), -# MAESTRO_UDID (target one device when several are connected) +# Required env: PLATFORM, CLERK_TEST_EMAIL, CLERK_TEST_PASSWORD +# Optional env: MAESTRO_UDID (comma-separated device ids; the flows are split +# across them, one shard per device; unset lets maestro pick one), +# MAESTRO_DEBUG_OUTPUT (directory for CI debug artifacts) set -euo pipefail cd "$(dirname "${BASH_SOURCE[0]}")" @@ -24,95 +22,115 @@ command -v maestro >/dev/null 2>&1 || { exit 1 } +: "${PLATFORM:?PLATFORM (ios|android) is required}" : "${CLERK_TEST_EMAIL:?CLERK_TEST_EMAIL is required}" : "${CLERK_TEST_PASSWORD:?CLERK_TEST_PASSWORD is required}" -force_stop() { if [ "$#" -gt 0 ]; then "$@" >/dev/null 2>&1 || true; fi; } +app_id=com.clerk.exponativebuildfixture +output_root=${MAESTRO_DEBUG_OUTPUT:-${TMPDIR:-/tmp}/clerk-expo-maestro-runner} +IFS=, read -r -a devices <<< "${MAESTRO_UDID:-}" +device_count=${#devices[@]} +[ "$device_count" -gt 0 ] || device_count=1 -run_flow() { - local output_name=$1 - shift - local output_root=${MAESTRO_DEBUG_OUTPUT:-${TMPDIR:-/tmp}/clerk-expo-maestro-runner} +force_stop() { + local device + for device in ${devices[@]+"${devices[@]}"}; do + if [ "$PLATFORM" = ios ]; then + xcrun simctl terminate "$device" "$app_id" >/dev/null 2>&1 || true + else + adb -s "$device" shell am force-stop "$app_id" >/dev/null 2>&1 || true + fi + done +} - maestro test \ - ${MAESTRO_UDID:+--udid "$MAESTRO_UDID"} \ +# $1: output name, $2: shard mode (all|split), $3: shard count, rest: flows. +run_maestro() { + local output_name=$1 mode=$2 shards=$3 + shift 3 + local shard_args=() + if [ "$shards" -gt 1 ]; then shard_args=("--shard-$mode" "$shards"); fi + maestro ${MAESTRO_UDID:+--udid "$MAESTRO_UDID"} test \ + ${shard_args[@]+"${shard_args[@]}"} \ --debug-output "$output_root/$output_name" \ --flatten-debug-output \ --format JUNIT \ --output "$output_root/$output_name/report.xml" \ + -e CLERK_TEST_EMAIL="$CLERK_TEST_EMAIL" \ + -e CLERK_TEST_PASSWORD="$CLERK_TEST_PASSWORD" \ "$@" } -record_result() { - local flow=$1 - local result=$2 - local attempts=$3 - local duration=$4 - - echo "Flow $flow: $result after $attempts attempt(s) in ${duration}s" - if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then - printf '| `%s` | %s | %s | %ss |\n' "$flow" "$result" "$attempts" "$duration" >> "$GITHUB_STEP_SUMMARY" - fi +# Prints one "\t\t" line per testcase in a JUnit report. +report_rows() { + python3 - "$1" <<'PY' +import sys, xml.etree.ElementTree as ET +for case in ET.parse(sys.argv[1]).getroot().iter('testcase'): + print(f"{case.get('file')}\t{case.get('status')}\t{float(case.get('time') or 0):.0f}") +PY } -if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then - { - echo '### Maestro flow timings' - echo '| Flow | Result | Attempts | Duration |' - echo '| --- | --- | ---: | ---: |' - } >> "$GITHUB_STEP_SUMMARY" -fi - -# Warm up the JS bundle and accessibility tree before running the flows. -warmup_started=$SECONDS +started=$SECONDS warmup_result=failed for warmup_attempt in 1 2; do - if run_flow "warmup-attempt-$warmup_attempt" flows/subflows/_warmup.yaml; then + if run_maestro "warmup-attempt-$warmup_attempt" all "$device_count" flows/subflows/_warmup.yaml; then warmup_result=passed break fi - force_stop "$@" + force_stop if [ "$warmup_attempt" -eq 1 ]; then echo "::warning::Warmup failed attempt 1, retrying after 10s..." sleep 10 fi done -warmup_duration=$((SECONDS - warmup_started)) -record_result "_warmup" "$warmup_result" "$warmup_attempt" "$warmup_duration" +echo "Warmup: $warmup_result after $warmup_attempt attempt(s) in $((SECONDS - started))s" if [ "$warmup_result" != passed ]; then echo "::error::Warmup failed after 2 attempts; aborting Maestro flows" exit 1 fi # Force-stop so the first launchApp clearState doesn't race the warm process. -force_stop "$@" +force_stop + +run_maestro flows split "$device_count" flows/*.yaml || true +report_rows "$output_root/flows/report.xml" > "$output_root/attempt-1.tsv" + +failed=() +while IFS=$'\t' read -r file status _; do + [ "$status" = SUCCESS ] || failed+=("$file") +done < "$output_root/attempt-1.tsv" + +if [ "${#failed[@]}" -gt 0 ]; then + echo "::warning::${#failed[@]} flow(s) failed, rerunning after 10s: ${failed[*]}" + force_stop + sleep 10 + shards=$device_count + [ "${#failed[@]}" -lt "$shards" ] && shards=${#failed[@]} + run_maestro rerun split "$shards" "${failed[@]}" || true + report_rows "$output_root/rerun/report.xml" > "$output_root/attempt-2.tsv" +fi + +if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + { + echo "### Maestro flow timings ($device_count device(s))" + echo '| Flow | Result | Attempts | Duration |' + echo '| --- | --- | ---: | ---: |' + } >> "$GITHUB_STEP_SUMMARY" +fi -# Every flows/*.yaml is a cross-platform test (platform differences live in -# per-step `when: platform:` conditionals); flows/subflows/ are runFlow-only. status=0 -for flow in flows/*.yaml; do - [ -e "$flow" ] || continue - flow_started=$SECONDS - flow_result=failed - for attempt in 1 2; do - if run_flow "${flow##*/}-attempt-$attempt" \ - -e CLERK_TEST_EMAIL="$CLERK_TEST_EMAIL" \ - -e CLERK_TEST_PASSWORD="$CLERK_TEST_PASSWORD" \ - "$flow"; then - flow_result=passed - break +while IFS=$'\t' read -r file result seconds; do + attempts=1 + if [ -f "$output_root/attempt-2.tsv" ]; then + rerun=$(awk -F'\t' -v f="$file" '$1 == f { print $2 "\t" $3 }' "$output_root/attempt-2.tsv") + if [ -n "$rerun" ]; then + attempts=2 + IFS=$'\t' read -r result seconds <<< "$rerun" fi - if [ "$attempt" -eq 2 ]; then - echo "::error::Flow $flow failed after 2 attempts" - status=1 - force_stop "$@" - break - fi - echo "::warning::Flow $flow failed attempt $attempt, retrying after 10s..." - force_stop "$@" - sleep 10 - done - flow_duration=$((SECONDS - flow_started)) - record_result "$flow" "$flow_result" "$attempt" "$flow_duration" -done + fi + if [ "$result" = SUCCESS ]; then result=passed; else result=failed; status=1; fi + echo "Flow $file: $result after $attempts attempt(s) in ${seconds}s" + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + printf '| `%s` | %s | %s | %ss |\n' "$file" "$result" "$attempts" "$seconds" >> "$GITHUB_STEP_SUMMARY" + fi +done < "$output_root/attempt-1.tsv" exit $status From 68b599c3956b90122be98f1fb5c00ef11d69f1d2 Mon Sep 17 00:00:00 2001 From: wobsoriano Date: Thu, 10 Sep 2026 22:53:00 -0700 Subject: [PATCH 03/14] test(expo): fold the restart check into the sign-in flow session-persists-across-restart was sign-in.yaml with a stop and relaunch inserted before the JS sign-out, so the restart now lives in sign-in.yaml and the duplicate flow is gone. Every assertion survives in the order it ran before; one full sign-in fewer per run. The identifier eraseText drops from 100 to 60, which still covers the CI addresses, and the password field no longer erases at all since it has never held a value. Maestro erases one character at a time, so the count is the cost. --- ...e.yaml => embedded-profile-host-back.yaml} | 0 .../tests/expo-native/flows/sign-in.yaml | 13 +++++- .../subflows/sign-in-email-password.yaml | 3 +- .../flows/user-profile-custom-pages.yaml | 45 +++++++++++++++++++ 4 files changed, 57 insertions(+), 4 deletions(-) rename integration/tests/expo-native/flows/{embedded-profile.yaml => embedded-profile-host-back.yaml} (100%) create mode 100644 integration/tests/expo-native/flows/user-profile-custom-pages.yaml diff --git a/integration/tests/expo-native/flows/embedded-profile.yaml b/integration/tests/expo-native/flows/embedded-profile-host-back.yaml similarity index 100% rename from integration/tests/expo-native/flows/embedded-profile.yaml rename to integration/tests/expo-native/flows/embedded-profile-host-back.yaml diff --git a/integration/tests/expo-native/flows/sign-in.yaml b/integration/tests/expo-native/flows/sign-in.yaml index 80645825691..e6a6555e6fb 100644 --- a/integration/tests/expo-native/flows/sign-in.yaml +++ b/integration/tests/expo-native/flows/sign-in.yaml @@ -1,13 +1,22 @@ # Native AuthView email+password sign-in, asserting the native->JS session -# sync, then JS-side sign-out asserting the reverse direction. +# sync; then a restart WITHOUT clearing state, asserting the session is +# restored from secure-store with no re-auth (bridge + token cache +# persistence); then JS-side sign-out asserting the reverse direction. appId: com.clerk.exponativebuildfixture -name: Native AuthView sign-in syncs to JS +name: Native AuthView sign-in syncs to JS and survives a restart --- - runFlow: subflows/open-app.yaml - tapOn: id: 'open-auth-view-button' - runFlow: subflows/sign-in-email-password.yaml - runFlow: subflows/assert-signed-in.yaml +- stopApp +- launchApp +- extendedWaitUntil: + visible: 'signed in' + timeout: 45000 +- assertVisible: + id: 'user-id' - tapOn: id: 'sign-out-button' - runFlow: subflows/assert-signed-out.yaml diff --git a/integration/tests/expo-native/flows/subflows/sign-in-email-password.yaml b/integration/tests/expo-native/flows/subflows/sign-in-email-password.yaml index 704f8b26f07..17b4f364ac9 100644 --- a/integration/tests/expo-native/flows/subflows/sign-in-email-password.yaml +++ b/integration/tests/expo-native/flows/subflows/sign-in-email-password.yaml @@ -45,7 +45,7 @@ appId: com.clerk.exponativebuildfixture commands: - tapOn: text: 'Select All' - - eraseText: 100 + - eraseText: 60 - inputText: ${CLERK_TEST_EMAIL} # Wait for the typed value to land before tapping: the tap can race the # recomposition that enables the button, and this also catches a mangled @@ -79,7 +79,6 @@ appId: com.clerk.exponativebuildfixture commands: - tapOn: text: 'Enter your password' - - eraseText: 50 - inputText: ${CLERK_TEST_PASSWORD} # The password is masked so its value can't be asserted; settle for the # screen going stable so the tap doesn't race the enabling recomposition. diff --git a/integration/tests/expo-native/flows/user-profile-custom-pages.yaml b/integration/tests/expo-native/flows/user-profile-custom-pages.yaml new file mode 100644 index 00000000000..0b408084b51 --- /dev/null +++ b/integration/tests/expo-native/flows/user-profile-custom-pages.yaml @@ -0,0 +1,45 @@ +# A custom page's React Native content is mounted as a child of the native host +# and rehosted into the destination its row pushes, then survives the trip back. +appId: com.clerk.exponativebuildfixture +name: UserProfileView renders a custom page +--- +- runFlow: subflows/open-app.yaml +- tapOn: + id: 'open-auth-view-button' +- runFlow: subflows/sign-in-email-password.yaml +- runFlow: subflows/assert-signed-in.yaml +- tapOn: + id: 'open-embedded-profile-button' +# Retrying wait: the row list lands a frame after the native profile paints. +- extendedWaitUntil: + visible: 'E2E Custom Page' + timeout: 20000 +- tapOn: + text: 'E2E Custom Page' +- extendedWaitUntil: + visible: 'Rehosted RN body' + timeout: 15000 +# The Android destination is a bare AndroidView with no back chrome, unlike the +# iOS page which clerk-ios pushes onto its own NavigationStack. +- runFlow: + when: + platform: iOS + commands: + - tapOn: 'Back' +- runFlow: + when: + platform: Android + commands: + - back +- extendedWaitUntil: + visible: 'E2E Custom Page' + timeout: 15000 +# The host chevron firing onHostBack, which reads 'Back' on both platforms. +- tapOn: 'Back' +- extendedWaitUntil: + visible: + id: 'open-embedded-profile-button' + timeout: 15000 +- tapOn: + id: 'sign-out-button' +- runFlow: subflows/assert-signed-out.yaml From 449dc3e7f816eae497956142e40933935d2d65c0 Mon Sep 17 00:00:00 2001 From: wobsoriano Date: Fri, 11 Sep 2026 09:42:47 -0700 Subject: [PATCH 04/14] ci(e2e): fail the maestro run on any first-attempt flow failure The failed-only rerun decided the result, so a flow that failed and then passed left the check green. The rerun now only labels a failure flaky or deterministic in the summary, and any first-attempt failure fails the step with an error annotation naming the flow. --- integration/tests/expo-native/run-flows.sh | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/integration/tests/expo-native/run-flows.sh b/integration/tests/expo-native/run-flows.sh index be28191c466..9db6a14c136 100755 --- a/integration/tests/expo-native/run-flows.sh +++ b/integration/tests/expo-native/run-flows.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash # Runs every top-level Maestro flow (flows/*.yaml; flows/subflows/ are # runFlow-only pieces) once across the connected devices, then reruns only the -# flows that failed. Whole-flow retry can mask app instability (the Maestro -# docs discourage it), so it is capped at a single rerun purely to absorb -# emulator/simulator flake. +# flows that failed. The rerun is diagnostic: it labels a failure flaky or +# deterministic in the summary, and any first-attempt failure still fails the +# run, so app instability is never absorbed. # # Usage: PLATFORM= ./run-flows.sh # @@ -100,7 +100,7 @@ while IFS=$'\t' read -r file status _; do done < "$output_root/attempt-1.tsv" if [ "${#failed[@]}" -gt 0 ]; then - echo "::warning::${#failed[@]} flow(s) failed, rerunning after 10s: ${failed[*]}" + echo "::warning::${#failed[@]} flow(s) failed, rerunning after 10s to tell flaky from deterministic: ${failed[*]}" force_stop sleep 10 shards=$device_count @@ -120,14 +120,19 @@ fi status=0 while IFS=$'\t' read -r file result seconds; do attempts=1 - if [ -f "$output_root/attempt-2.tsv" ]; then - rerun=$(awk -F'\t' -v f="$file" '$1 == f { print $2 "\t" $3 }' "$output_root/attempt-2.tsv") + if [ "$result" = SUCCESS ]; then + result=passed + else + status=1 + result=failed + rerun=$(awk -F'\t' -v f="$file" '$1 == f { print $2 "\t" $3 }' "$output_root/attempt-2.tsv" 2>/dev/null || true) if [ -n "$rerun" ]; then attempts=2 - IFS=$'\t' read -r result seconds <<< "$rerun" + IFS=$'\t' read -r rerun_result seconds <<< "$rerun" + [ "$rerun_result" = SUCCESS ] && result='flaky (failed, then passed on the rerun)' fi + echo "::error::Flow $file: $result" fi - if [ "$result" = SUCCESS ]; then result=passed; else result=failed; status=1; fi echo "Flow $file: $result after $attempts attempt(s) in ${seconds}s" if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then printf '| `%s` | %s | %s | %ss |\n' "$file" "$result" "$attempts" "$seconds" >> "$GITHUB_STEP_SUMMARY" From f2b735b76ba5f54625b87c70edfaf742f265e753 Mon Sep 17 00:00:00 2001 From: wobsoriano Date: Fri, 11 Sep 2026 09:43:07 -0700 Subject: [PATCH 05/14] test(expo): report the sign-out outcome in the fixture Release builds log nothing from JS, so a sign-out that never ran and one that rejected look the same in the artifacts. The sign-out button now renders whether signOut is pending, resolved, or rejected with its message, readable from Maestro screenshots and hierarchies. --- integration/templates/expo-native/App.tsx | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/integration/templates/expo-native/App.tsx b/integration/templates/expo-native/App.tsx index 78fef6a0ea8..11d3670243a 100644 --- a/integration/templates/expo-native/App.tsx +++ b/integration/templates/expo-native/App.tsx @@ -17,6 +17,7 @@ function NativeBuildFixture() { const { user } = useUser(); const [isAuthOpen, setIsAuthOpen] = useState(false); const [isProfileOpen, setIsProfileOpen] = useState(false); + const [signOutResult, setSignOutResult] = useState(null); if (isProfileOpen) { return ( @@ -56,6 +57,7 @@ function NativeBuildFixture() { {isLoaded ? `signed ${isSignedIn ? 'in' : 'out'}` : 'loading'} {user?.id && {user.id}} + {signOutResult && {`sign out: ${signOutResult}`}}