From 46b6dded2a6334062ad577af51d37b0ce5d6a672 Mon Sep 17 00:00:00 2001
From: wobsoriano <sorianorobertc@gmail.com>
Date: Fri, 23 Oct 2026 09:54:43 -0700
Subject: [PATCH 1/5] ci(repo): post the Expo e2e flaky tests to Slack once a
 day

---
 .changeset/honest-lemons-travel.md        |   2 +
 .claude/skills/verify-clerk-expo/SKILL.md |   2 +-
 .github/workflows/ci.yml                  |   4 +-
 .github/workflows/expo-e2e-flaky.yml      |  58 ++++++
 .github/workflows/verify-e2e.yml          |  14 ++
 scripts/verify-flaky.mjs                  | 152 ++++++++++++++++
 scripts/verify-flaky.test.mjs             | 205 ++++++++++++++++++++++
 7 files changed, 434 insertions(+), 3 deletions(-)
 create mode 100644 .changeset/honest-lemons-travel.md
 create mode 100644 .github/workflows/expo-e2e-flaky.yml
 create mode 100644 scripts/verify-flaky.mjs
 create mode 100644 scripts/verify-flaky.test.mjs

diff --git a/.changeset/honest-lemons-travel.md b/.changeset/honest-lemons-travel.md
new file mode 100644
index 00000000000..a845151cc84
--- /dev/null
+++ b/.changeset/honest-lemons-travel.md
@@ -0,0 +1,2 @@
+---
+---
diff --git a/.claude/skills/verify-clerk-expo/SKILL.md b/.claude/skills/verify-clerk-expo/SKILL.md
index 5eb741c9064..37c4055ac35 100644
--- a/.claude/skills/verify-clerk-expo/SKILL.md
+++ b/.claude/skills/verify-clerk-expo/SKILL.md
@@ -319,7 +319,7 @@ For a remote device, `down` ends the runner job and waits for it to finish, and
 - `src/host.ts`, `src/fixture.ts`, `src/native-build.ts`, and `src/freshness.ts` are this repository's own: the build of the test app, the native build that CI keeps between runs, the Metro ports, and the check that Metro serves current JS. `specs/app.ts` names the test app and its entry for a dev client, and `specs/native.ts` holds the per-platform locators for the native views and the locators of the home's links. Both are this repository's own too.
 - `npm test --prefix integration/expo-native` runs the CLI's unit tests, with no network, key, or device. `npm run typecheck --prefix integration/expo-native` runs `tsc`. The `Expo Native Runner Tests` job in `.github/workflows/ci.yml` runs both on Linux when a pull request changes the package, the test app, or a package the test app links.
 - `run --github-report` hands the results of the run to `@e2e-dev/github` as one report. The reporter writes the report to the job summary. With a `GITHUB_TOKEN` that may write pull request comments, it also posts one comment on the pull request and updates that comment on later runs. The reporter never changes the exit code, and nothing is reported for a run with a tainted file.
-- `.github/workflows/verify-e2e.yml`, the `Expo E2E` workflow, runs `up --backend local`, `run --all --retries 1 --max-failures 5 --github-report`, and `down` on a runner for each platform, with a device and a Clerk application for each. It runs on the Version packages pull request (`changeset-release/main`), so each release is tested before it publishes, and the `Expo E2E` check fails when a spec fails. On other pull requests the device tests skip and the `Expo E2E` check passes. Start it by hand with `gh workflow run verify-e2e.yml --ref <branch>`. A test that fails and then passes on its one retry is `flaky` and does not fail the job. The runners are Blacksmith's `blacksmith-6vcpu-macos-26` and `blacksmith-8vcpu-ubuntu-2204`, and the repository variables `VERIFY_CI_RUNNER_IOS` and `VERIFY_CI_RUNNER_ANDROID` name other labels. The Platform API key comes from the `MOBILE_VERIFICATION_PLATFORM_API_KEY` repository secret. The workflow sets `VERIFY_LOCAL_BUILD=standalone` and `VERIFY_NATIVE_CACHE` ([freshness.md](references/freshness.md)). It uploads each run's `run.json`, app log, driver logs, video, screenshots, and e2e's `report.json`, `junit.xml`, summary, and failure pages for three days, and only when no secret is found in the run. `bin/configure-ios-simulators.sh` speeds up animations and turns off autocorrect and the keyboard tutorials on each simulator in `CLERK_TEST_DEVICES`, and this workflow and `.github/workflows/verify-remote.yml` call it.
+- `.github/workflows/verify-e2e.yml`, the `Expo E2E` workflow, runs `up --backend local`, `run --all --retries 1 --max-failures 5 --github-report`, and `down` on a runner for each platform, with a device and a Clerk application for each. It runs on the Version packages pull request (`changeset-release/main`), so each release is tested before it publishes, and the `Expo E2E` check fails when a spec fails. On other pull requests the device tests skip and the `Expo E2E` check passes. Start it by hand with `gh workflow run verify-e2e.yml --ref <branch>`. A test that fails and then passes on its one retry is `flaky` and does not fail the job. The runners are Blacksmith's `blacksmith-6vcpu-macos-26` and `blacksmith-8vcpu-ubuntu-2204`, and the repository variables `VERIFY_CI_RUNNER_IOS` and `VERIFY_CI_RUNNER_ANDROID` name other labels. The Platform API key comes from the `MOBILE_VERIFICATION_PLATFORM_API_KEY` repository secret. The workflow sets `VERIFY_LOCAL_BUILD=standalone` and `VERIFY_NATIVE_CACHE` ([freshness.md](references/freshness.md)). It uploads each run's `run.json`, app log, driver logs, video, screenshots, and e2e's `report.json`, `junit.xml`, summary, and failure pages for three days, and only when no secret is found in the run. Each job also keeps its flaky tests for three days, and `.github/workflows/expo-e2e-flaky.yml` posts the release pull request's flaky tests of the last 24 hours to #team-mobile-alerts every day. `bin/configure-ios-simulators.sh` speeds up animations and turns off autocorrect and the keyboard tutorials on each simulator in `CLERK_TEST_DEVICES`, and this workflow and `.github/workflows/verify-remote.yml` call it.
 - The `Expo` workflow, `.github/workflows/expo-native-build.yml`, runs nothing on a device. It builds the test app as a Release app on Expo SDK 55 and 58 for each platform, and runs the Android unit tests of `@clerk/expo-biometrics`.
 - `.github/workflows/verify-attach.yml` publishes the evidence that `attach` hands to a session's runner, with `scripts/verify-attach.mjs`. It reads the manifest that `src/core/remote/handoff.ts` of `@clerk/internal-mobile-e2e` writes. It and the package's `src/core/publish.ts` each replace the block the other wrote, each applies the same rules to the comments and to the start of the line, and each lays the files out in the same tables. The script is checked out alone, so it imports nothing from the package. So a change to the manifest, the comments, those rules, or the tables is a change in both places. `scripts/verify-attach.test.mjs` has its tests.
 - A remote session runs in `.github/workflows/verify-remote.yml`. `src/platform/session-device.ts` holds the steps that the session runs on its device, and `src/core/remote/` of `@clerk/internal-mobile-e2e` is the code on both ends of it. The runner installs the package with `npm ci`. A session keeps the core it installed when it started, so after a change to the commit pin of `@clerk/internal-mobile-e2e`, commit, push, `down`, then `up`.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 9e33e2f87dc..854be349382 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -379,9 +379,9 @@ jobs:
         env:
           NODE_VERSION: ${{ matrix.node-version }}
 
-      - name: Run verify-attach script tests
+      - name: Run verify script tests
         if: matrix.node-version == '24.15.0'
-        run: pnpm vitest run --root scripts verify-attach
+        run: pnpm vitest run --root scripts verify-attach verify-flaky
 
       - name: Upload Turbo Summary
         uses: actions/upload-artifact@v4.6.2
diff --git a/.github/workflows/expo-e2e-flaky.yml b/.github/workflows/expo-e2e-flaky.yml
new file mode 100644
index 00000000000..35975f7f594
--- /dev/null
+++ b/.github/workflows/expo-e2e-flaky.yml
@@ -0,0 +1,58 @@
+name: Expo E2E flaky tests
+
+on:
+  schedule:
+    - cron: '17 7 * * *'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  digest:
+    name: Post the flaky tests to Slack
+    if: github.repository == 'clerk/javascript'
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      actions: read
+      contents: read
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4.4.0
+        with:
+          persist-credentials: false
+          sparse-checkout: scripts/verify-flaky.mjs
+          sparse-checkout-cone-mode: false
+
+      - uses: actions/setup-node@v4.4.0
+        with:
+          node-version: 24.15.0
+
+      - name: Post the flaky tests of the last 24 hours
+        env:
+          GH_TOKEN: ${{ github.token }}
+          GH_REPO: ${{ github.repository }}
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_MOBILE_TEAM_ALERTS_WEBHOOK_URL }}
+        run: |
+          set -euo pipefail
+          since="$(date -u -d '24 hours ago' +%Y-%m-%dT%H:%M:%SZ)"
+          gh run list --workflow verify-e2e.yml --branch changeset-release/main --event pull_request --created ">=$since" --limit 200 --json databaseId,conclusion > "$RUNNER_TEMP/runs.json"
+          mapfile -t ids < <(jq -r '.[] | select(.conclusion != "" and .conclusion != "cancelled" and .conclusion != "skipped") | .databaseId' "$RUNNER_TEMP/runs.json")
+          mkdir -p "$RUNNER_TEMP/flaky"
+          for id in "${ids[@]}"; do
+            gh run download "$id" --pattern 'verify-flaky-*' --dir "$RUNNER_TEMP/flaky/$id" || echo "Run $id kept no flaky tests."
+          done
+          MESSAGE="$(node scripts/verify-flaky.mjs digest "$RUNNER_TEMP/flaky" --runs "${#ids[@]}" --hours 24)"
+          if [ -z "$MESSAGE" ]; then
+            echo "No flaky tests in the last 24 hours."
+            exit 0
+          fi
+          echo "$MESSAGE"
+          if [ -z "$SLACK_WEBHOOK_URL" ]; then
+            echo "::warning::SLACK_MOBILE_TEAM_ALERTS_WEBHOOK_URL is not configured; skipping Slack notification"
+            exit 0
+          fi
+          PAYLOAD="$(jq -n --arg text "$MESSAGE" '{text: $text, unfurl_links: false, unfurl_media: false}')"
+          curl --fail-with-body --connect-timeout 10 --max-time 30 --retry 3 --retry-delay 5 --retry-max-time 60 --retry-all-errors \
+            --request POST --header "Content-type: application/json" --data "$PAYLOAD" "$SLACK_WEBHOOK_URL"
diff --git a/.github/workflows/verify-e2e.yml b/.github/workflows/verify-e2e.yml
index d3cae0c03b9..45bf225cfa3 100644
--- a/.github/workflows/verify-e2e.yml
+++ b/.github/workflows/verify-e2e.yml
@@ -221,6 +221,20 @@ jobs:
           retention-days: 3
           if-no-files-found: warn
 
+      - name: Record the flaky tests
+        if: ${{ !cancelled() && steps.evidence.outputs.upload == 'true' }}
+        run: node scripts/verify-flaky.mjs record "$PACKAGE/.verify/runs" "$RUNNER_TEMP/verify-flaky.json"
+
+      - name: Keep the flaky tests for the daily digest
+        if: ${{ !cancelled() && steps.evidence.outputs.upload == 'true' }}
+        uses: actions/upload-artifact@v4.6.2
+        with:
+          name: verify-flaky-${{ matrix.platform }}
+          path: ${{ runner.temp }}/verify-flaky.json
+          retention-days: 3
+          overwrite: true
+          if-no-files-found: ignore
+
   verify-e2e:
     name: Expo E2E
     needs: e2e-tests
diff --git a/scripts/verify-flaky.mjs b/scripts/verify-flaky.mjs
new file mode 100644
index 00000000000..fc1ddd18b37
--- /dev/null
+++ b/scripts/verify-flaky.mjs
@@ -0,0 +1,152 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const MOST_TESTS = 10;
+const ERROR_CHARS = 300;
+
+function flakesFromReport(report) {
+  return report.run.results
+    .filter(result => result.status === 'flaky')
+    .map(result => ({
+      platform: result.platform,
+      file: result.file,
+      title: result.titlePath.join(' › '),
+      error: result.attempts.find(attempt => attempt.status === 'failed')?.error?.message ?? '',
+    }));
+}
+
+function normalizeError(message) {
+  const normalized = message
+    .replace(/[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g, '<email>')
+    .replace(/\b(user|sess|ins|app|org)_[A-Za-z0-9]+/g, '$1_<id>')
+    .replace(/after launch [0-9a-f]+/g, 'after launch <id>')
+    .replace(/; passes: \d+;.*$/s, '')
+    .replace(/\s+/g, ' ')
+    .trim();
+  return normalized.length > ERROR_CHARS ? `${normalized.slice(0, ERROR_CHARS)}…` : normalized;
+}
+
+const slackText = text => text.replaceAll('&', '&amp;').replaceAll('<', '&lt;').replaceAll('>', '&gt;');
+
+const mostCommon = counts => [...counts.entries()].sort((a, b) => b[1] - a[1])[0][0];
+
+function digestMessage(records, { runs, hours }) {
+  const tests = new Map();
+  for (const record of records) {
+    for (const flake of record.flakes) {
+      const key = `${flake.file}\n${flake.title}`;
+      const test = tests.get(key) ?? {
+        file: flake.file,
+        title: flake.title,
+        total: 0,
+        platforms: new Map(),
+        runs: new Set(),
+        errors: new Map(),
+        latestRunUrl: null,
+      };
+      const error = normalizeError(flake.error);
+      test.total += 1;
+      test.platforms.set(flake.platform, (test.platforms.get(flake.platform) ?? 0) + 1);
+      test.runs.add(record.runUrl ?? record);
+      test.errors.set(error, (test.errors.get(error) ?? 0) + 1);
+      test.latestRunUrl = record.runUrl ?? test.latestRunUrl;
+      tests.set(key, test);
+    }
+  }
+  if (tests.size === 0) {
+    return '';
+  }
+
+  const sorted = [...tests.values()].sort(
+    (a, b) => b.total - a.total || a.file.localeCompare(b.file) || a.title.localeCompare(b.title),
+  );
+  const lines = [`:warning: Expo E2E had flaky tests in ${runs} Version packages runs over the last ${hours} hours`];
+  for (const test of sorted.slice(0, MOST_TESTS)) {
+    const platforms = [...test.platforms.entries()]
+      .sort(([a], [b]) => a.localeCompare(b))
+      .map(([platform, count]) => `${platform} ×${count}`)
+      .join(', ');
+    const parts = [
+      `• \`${test.file.replace(/^specs\/features\//, '')}\` ${slackText(test.title)}`,
+      `${test.runs.size} of ${runs} runs`,
+      platforms,
+    ];
+    if (test.latestRunUrl !== null) {
+      parts.push(`<${test.latestRunUrl}|latest run>`);
+    }
+    lines.push(parts.join(' · '));
+    const error = mostCommon(test.errors);
+    if (error !== '') {
+      lines.push(`  first attempt failed: ${slackText(error)}`);
+    }
+  }
+  if (sorted.length > MOST_TESTS) {
+    lines.push(`…and ${sorted.length - MOST_TESTS} more`);
+  }
+  return lines.join('\n');
+}
+
+const readJson = file => JSON.parse(fs.readFileSync(file, 'utf8'));
+
+const byDirectory = (a, b) => path.dirname(a).localeCompare(path.dirname(b), 'en', { numeric: true });
+
+function recordStep(runsDir, outFile, env) {
+  const reports = fs.existsSync(runsDir)
+    ? fs
+        .globSync('*/e2e*/report.json', { cwd: runsDir })
+        .sort(byDirectory)
+        .map(file => readJson(path.join(runsDir, file)))
+    : [];
+  const runUrl =
+    env.GITHUB_SERVER_URL && env.GITHUB_REPOSITORY && env.GITHUB_RUN_ID
+      ? `${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}`
+      : null;
+  const record = {
+    runUrl,
+    commit: env.GITHUB_SHA || (reports[0]?.run.vcs?.commit ?? null),
+    flakes: reports.flatMap(flakesFromReport),
+  };
+  fs.writeFileSync(outFile, `${JSON.stringify(record, null, 2)}\n`);
+  return record;
+}
+
+function readRecords(dir) {
+  return fs
+    .globSync('**/*.json', { cwd: dir })
+    .sort(byDirectory)
+    .map(file => {
+      const record = readJson(path.join(dir, file));
+      if (!Array.isArray(record?.flakes)) {
+        throw new Error(`${file} is not a record of flaky tests`);
+      }
+      return record;
+    });
+}
+
+function main(argv, env, log = console.log) {
+  const [step, ...rest] = argv;
+  const flag = name => (rest.includes(name) ? rest[rest.indexOf(name) + 1] : undefined);
+  if (step === 'record' && rest.length === 2) {
+    const record = recordStep(rest[0], rest[1], env);
+    log(`Recorded ${record.flakes.length} flaky tests in ${rest[1]}.`);
+    return 0;
+  }
+  const runs = Number(flag('--runs'));
+  const hours = Number(flag('--hours'));
+  if (step === 'digest' && rest[0] !== undefined && Number.isInteger(runs) && Number.isInteger(hours)) {
+    const message = digestMessage(readRecords(rest[0]), { runs, hours });
+    if (message !== '') {
+      log(message);
+    }
+    return 0;
+  }
+  log('usage: verify-flaky.mjs record <runsDir> <outFile> | digest <dir> --runs <n> --hours <h>');
+  return 2;
+}
+
+export { digestMessage, flakesFromReport, main, normalizeError };
+
+if (process.argv[1] !== undefined && fs.realpathSync(process.argv[1]) === fileURLToPath(import.meta.url)) {
+  process.exit(main(process.argv.slice(2), process.env));
+}
diff --git a/scripts/verify-flaky.test.mjs b/scripts/verify-flaky.test.mjs
new file mode 100644
index 00000000000..c72b7accb37
--- /dev/null
+++ b/scripts/verify-flaky.test.mjs
@@ -0,0 +1,205 @@
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+
+import { describe, expect, it } from 'vitest';
+
+import { digestMessage, flakesFromReport, main, normalizeError } from './verify-flaky.mjs';
+
+const signedInError = (name, user, launch, session, passes = 2) =>
+  `the app did not show "Signed in as ${name}+clerk_test@example.com" with user ID ${user} and a session ID on the home after launch ${launch}; it shows its error screen: Something went wrong No session was found with id ${session}; passes: ${passes}; reads: ${passes * 2}, the slowest ${passes * 70}ms (the heading)`;
+
+const result = (status, attempts, change = {}) => ({
+  status,
+  platform: 'android',
+  file: 'specs/features/native-js-sync/sign-out-from-native.e2e.ts',
+  titlePath: ['signing out in the native profile signs out the JS hooks'],
+  attempts,
+  ...change,
+});
+
+const attempt = (status, message) => ({ status, error: message === undefined ? null : { message } });
+
+const report = results => ({ run: { vcs: { commit: 'a'.repeat(40) }, results } });
+
+const flake = (change = {}) => ({
+  platform: 'android',
+  file: 'specs/features/native-js-sync/sign-out-from-native.e2e.ts',
+  title: 'signing out in the native profile signs out the JS hooks',
+  error: 'No session was found with id sess_3L6DwJjKc2IDyokGGeluncGnY69',
+  ...change,
+});
+
+const runUrl = id => `https://github.com/clerk/javascript/actions/runs/${id}`;
+
+describe('flakesFromReport', () => {
+  it('keeps only the flaky results, with the error of the first failed attempt', () => {
+    const flakes = flakesFromReport(
+      report([
+        result('passed', [attempt('passed')]),
+        result('failed', [attempt('failed', 'failed twice'), attempt('failed', 'failed twice')]),
+        result('skipped', []),
+        result(
+          'flaky',
+          [attempt('failed', 'the first failure'), attempt('failed', 'the second failure'), attempt('passed')],
+          { platform: 'ios', titlePath: ['profile', 'signs out'] },
+        ),
+      ]),
+    );
+
+    expect(flakes).toEqual([
+      {
+        platform: 'ios',
+        file: 'specs/features/native-js-sync/sign-out-from-native.e2e.ts',
+        title: 'profile › signs out',
+        error: 'the first failure',
+      },
+    ]);
+  });
+});
+
+describe('normalizeError', () => {
+  it('gives the same text for the same failure in different runs', () => {
+    expect(
+      normalizeError(
+        signedInError(
+          'ben.a30f',
+          'user_3L6FBztORCO03ZuOC5J50LJzABj',
+          '2bbc58c2e52b4448',
+          'sess_3L6FCP4vbLeA13m6zfP70AEVLx7',
+        ),
+      ),
+    ).toBe(
+      normalizeError(
+        signedInError(
+          'eli.b71c',
+          'user_3L6FGxY6BKLgtHLELbgYi9pPd79',
+          '0ebe1bf6eff0cba0',
+          'sess_3L6FHLmSGr9hI2LbSBm2E4BHlHl',
+          3,
+        ),
+      ),
+    );
+    expect(
+      normalizeError(
+        signedInError(
+          'ben.a30f',
+          'user_3L6FBztORCO03ZuOC5J50LJzABj',
+          '2bbc58c2e52b4448',
+          'sess_3L6FCP4vbLeA13m6zfP70AEVLx7',
+        ),
+      ),
+    ).toBe(
+      'the app did not show "Signed in as <email>" with user ID user_<id> and a session ID on the home after launch <id>; it shows its error screen: Something went wrong No session was found with id sess_<id>',
+    );
+    expect(normalizeError('No session was found with id sess_3L6DwJjKc2IDyokGGeluncGnY69')).toBe(
+      normalizeError('No  session was found\nwith id sess_3L6FHLmSGr9hI2LbSBm2E4BHlHl'),
+    );
+    expect(normalizeError('ins_2a app_2b org_2c')).toBe('ins_<id> app_<id> org_<id>');
+  });
+
+  it('cuts a long message', () => {
+    expect(normalizeError('x'.repeat(500))).toBe(`${'x'.repeat(300)}…`);
+  });
+});
+
+describe('digestMessage', () => {
+  it('is empty when no test was flaky', () => {
+    expect(digestMessage([], { runs: 4, hours: 24 })).toBe('');
+    expect(digestMessage([{ runUrl: runUrl(1), commit: null, flakes: [] }], { runs: 4, hours: 24 })).toBe('');
+  });
+
+  it('groups a test across runs and platforms, and links its latest run', () => {
+    const signOut = flake();
+    const profile = flake({
+      file: 'specs/features/user-button-and-profile/profile.e2e.ts',
+      title: 'the home sign-out button ends the session',
+      error: 'a <b> & c',
+    });
+    const message = digestMessage(
+      [
+        { runUrl: runUrl(1), commit: null, flakes: [signOut, profile] },
+        { runUrl: runUrl(1), commit: null, flakes: [flake({ platform: 'ios' })] },
+        {
+          runUrl: runUrl(2),
+          commit: null,
+          flakes: [flake({ error: 'No session was found with id sess_3L6FCP4vbLeA13m6zfP70AEVLx7' })],
+        },
+        { runUrl: runUrl(3), commit: null, flakes: [flake({ error: 'something else' })] },
+      ],
+      { runs: 5, hours: 24 },
+    );
+
+    expect(message).toBe(
+      [
+        ':warning: Expo E2E had flaky tests in 5 Version packages runs over the last 24 hours',
+        `• \`native-js-sync/sign-out-from-native.e2e.ts\` signing out in the native profile signs out the JS hooks · 3 of 5 runs · android ×3, ios ×1 · <${runUrl(3)}|latest run>`,
+        '  first attempt failed: No session was found with id sess_&lt;id&gt;',
+        `• \`user-button-and-profile/profile.e2e.ts\` the home sign-out button ends the session · 1 of 5 runs · android ×1 · <${runUrl(1)}|latest run>`,
+        '  first attempt failed: a &lt;b&gt; &amp; c',
+      ].join('\n'),
+    );
+  });
+
+  it('lists the ten most flaky tests', () => {
+    const flakes = Array.from({ length: 12 }, (_, i) =>
+      Array.from({ length: i + 1 }, () => flake({ title: `test ${i}` })),
+    ).flat();
+    const lines = digestMessage([{ runUrl: null, commit: null, flakes }], { runs: 1, hours: 24 }).split('\n');
+
+    expect(lines.filter(line => line.startsWith('•')).map(line => line.split(' · ')[0])).toEqual(
+      Array.from({ length: 10 }, (_, i) => `• \`native-js-sync/sign-out-from-native.e2e.ts\` test ${11 - i}`),
+    );
+    expect(lines[1]).toBe('• `native-js-sync/sign-out-from-native.e2e.ts` test 11 · 1 of 1 runs · android ×12');
+    expect(lines.at(-1)).toBe('…and 2 more');
+  });
+});
+
+describe('the command line', () => {
+  it('records the flaky tests of every e2e report, then digests them', () => {
+    const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'verify-flaky-'));
+    const runs = path.join(dir, 'runs');
+    const write = (file, value) => {
+      fs.mkdirSync(path.dirname(file), { recursive: true });
+      fs.writeFileSync(file, JSON.stringify(value));
+    };
+    write(
+      path.join(runs, 'r20261023-143803-a30f', 'e2e', 'report.json'),
+      report([result('flaky', [attempt('failed', 'first'), attempt('passed')]), result('passed', [])]),
+    );
+    write(
+      path.join(runs, 'r20261023-143803-a30f', 'e2e-2', 'report.json'),
+      report([result('flaky', [attempt('failed', 'second'), attempt('passed')], { platform: 'ios' })]),
+    );
+    write(path.join(runs, 'r20261023-150000-b00c', 'e2e', 'report.json'), report([]));
+    const env = {
+      GITHUB_SERVER_URL: 'https://github.com',
+      GITHUB_REPOSITORY: 'clerk/javascript',
+      GITHUB_RUN_ID: '42',
+      GITHUB_SHA: 'b'.repeat(40),
+    };
+    const out = path.join(dir, 'records', '42', 'verify-flaky-android', 'verify-flaky.json');
+    fs.mkdirSync(path.dirname(out), { recursive: true });
+    const log = [];
+
+    expect(main(['record', runs, out], env, line => log.push(line))).toBe(0);
+    expect(JSON.parse(fs.readFileSync(out, 'utf8'))).toEqual({
+      runUrl: runUrl(42),
+      commit: 'b'.repeat(40),
+      flakes: [flake({ error: 'first' }), flake({ platform: 'ios', error: 'second' })],
+    });
+
+    const empty = path.join(dir, 'empty.json');
+    expect(main(['record', path.join(dir, 'no-runs'), empty], {}, () => {})).toBe(0);
+    expect(JSON.parse(fs.readFileSync(empty, 'utf8'))).toEqual({ runUrl: null, commit: null, flakes: [] });
+
+    const digest = [];
+    expect(main(['digest', path.join(dir, 'records'), '--runs', '2', '--hours', '24'], {}, l => digest.push(l))).toBe(
+      0,
+    );
+    expect(digest.join('\n').split('\n')[1]).toBe(
+      `• \`native-js-sync/sign-out-from-native.e2e.ts\` signing out in the native profile signs out the JS hooks · 1 of 2 runs · android ×1, ios ×1 · <${runUrl(42)}|latest run>`,
+    );
+    expect(main(['digest', path.join(dir, 'records')], {}, () => {})).toBe(2);
+  });
+});

From 17691afa85f52ff0a8c4569fda6117fc6364e624 Mon Sep 17 00:00:00 2001
From: wobsoriano <sorianorobertc@gmail.com>
Date: Fri, 23 Oct 2026 10:28:50 -0700
Subject: [PATCH 2/5] ci(repo): post the Expo e2e flaky tests to Slack after
 each release run

---
 .claude/skills/verify-clerk-expo/SKILL.md |  2 +-
 .github/workflows/expo-e2e-flaky.yml      | 58 ---------------------
 .github/workflows/verify-e2e.yml          | 51 ++++++++++++++++++-
 scripts/verify-flaky.mjs                  | 59 ++++++----------------
 scripts/verify-flaky.test.mjs             | 61 ++++++++---------------
 5 files changed, 87 insertions(+), 144 deletions(-)
 delete mode 100644 .github/workflows/expo-e2e-flaky.yml

diff --git a/.claude/skills/verify-clerk-expo/SKILL.md b/.claude/skills/verify-clerk-expo/SKILL.md
index 37c4055ac35..e03293718b9 100644
--- a/.claude/skills/verify-clerk-expo/SKILL.md
+++ b/.claude/skills/verify-clerk-expo/SKILL.md
@@ -319,7 +319,7 @@ For a remote device, `down` ends the runner job and waits for it to finish, and
 - `src/host.ts`, `src/fixture.ts`, `src/native-build.ts`, and `src/freshness.ts` are this repository's own: the build of the test app, the native build that CI keeps between runs, the Metro ports, and the check that Metro serves current JS. `specs/app.ts` names the test app and its entry for a dev client, and `specs/native.ts` holds the per-platform locators for the native views and the locators of the home's links. Both are this repository's own too.
 - `npm test --prefix integration/expo-native` runs the CLI's unit tests, with no network, key, or device. `npm run typecheck --prefix integration/expo-native` runs `tsc`. The `Expo Native Runner Tests` job in `.github/workflows/ci.yml` runs both on Linux when a pull request changes the package, the test app, or a package the test app links.
 - `run --github-report` hands the results of the run to `@e2e-dev/github` as one report. The reporter writes the report to the job summary. With a `GITHUB_TOKEN` that may write pull request comments, it also posts one comment on the pull request and updates that comment on later runs. The reporter never changes the exit code, and nothing is reported for a run with a tainted file.
-- `.github/workflows/verify-e2e.yml`, the `Expo E2E` workflow, runs `up --backend local`, `run --all --retries 1 --max-failures 5 --github-report`, and `down` on a runner for each platform, with a device and a Clerk application for each. It runs on the Version packages pull request (`changeset-release/main`), so each release is tested before it publishes, and the `Expo E2E` check fails when a spec fails. On other pull requests the device tests skip and the `Expo E2E` check passes. Start it by hand with `gh workflow run verify-e2e.yml --ref <branch>`. A test that fails and then passes on its one retry is `flaky` and does not fail the job. The runners are Blacksmith's `blacksmith-6vcpu-macos-26` and `blacksmith-8vcpu-ubuntu-2204`, and the repository variables `VERIFY_CI_RUNNER_IOS` and `VERIFY_CI_RUNNER_ANDROID` name other labels. The Platform API key comes from the `MOBILE_VERIFICATION_PLATFORM_API_KEY` repository secret. The workflow sets `VERIFY_LOCAL_BUILD=standalone` and `VERIFY_NATIVE_CACHE` ([freshness.md](references/freshness.md)). It uploads each run's `run.json`, app log, driver logs, video, screenshots, and e2e's `report.json`, `junit.xml`, summary, and failure pages for three days, and only when no secret is found in the run. Each job also keeps its flaky tests for three days, and `.github/workflows/expo-e2e-flaky.yml` posts the release pull request's flaky tests of the last 24 hours to #team-mobile-alerts every day. `bin/configure-ios-simulators.sh` speeds up animations and turns off autocorrect and the keyboard tutorials on each simulator in `CLERK_TEST_DEVICES`, and this workflow and `.github/workflows/verify-remote.yml` call it.
+- `.github/workflows/verify-e2e.yml`, the `Expo E2E` workflow, runs `up --backend local`, `run --all --retries 1 --max-failures 5 --github-report`, and `down` on a runner for each platform, with a device and a Clerk application for each. It runs on the Version packages pull request (`changeset-release/main`), so each release is tested before it publishes, and the `Expo E2E` check fails when a spec fails. On other pull requests the device tests skip and the `Expo E2E` check passes. Start it by hand with `gh workflow run verify-e2e.yml --ref <branch>`. A test that fails and then passes on its one retry is `flaky` and does not fail the job. The runners are Blacksmith's `blacksmith-6vcpu-macos-26` and `blacksmith-8vcpu-ubuntu-2204`, and the repository variables `VERIFY_CI_RUNNER_IOS` and `VERIFY_CI_RUNNER_ANDROID` name other labels. The Platform API key comes from the `MOBILE_VERIFICATION_PLATFORM_API_KEY` repository secret. The workflow sets `VERIFY_LOCAL_BUILD=standalone` and `VERIFY_NATIVE_CACHE` ([freshness.md](references/freshness.md)). It uploads each run's `run.json`, app log, driver logs, video, screenshots, and e2e's `report.json`, `junit.xml`, summary, and failure pages for three days, and only when no secret is found in the run. After a run on the release pull request, the `Post the flaky tests to Slack` job posts the tests that passed only on their retry to #team-mobile-alerts. `bin/configure-ios-simulators.sh` speeds up animations and turns off autocorrect and the keyboard tutorials on each simulator in `CLERK_TEST_DEVICES`, and this workflow and `.github/workflows/verify-remote.yml` call it.
 - The `Expo` workflow, `.github/workflows/expo-native-build.yml`, runs nothing on a device. It builds the test app as a Release app on Expo SDK 55 and 58 for each platform, and runs the Android unit tests of `@clerk/expo-biometrics`.
 - `.github/workflows/verify-attach.yml` publishes the evidence that `attach` hands to a session's runner, with `scripts/verify-attach.mjs`. It reads the manifest that `src/core/remote/handoff.ts` of `@clerk/internal-mobile-e2e` writes. It and the package's `src/core/publish.ts` each replace the block the other wrote, each applies the same rules to the comments and to the start of the line, and each lays the files out in the same tables. The script is checked out alone, so it imports nothing from the package. So a change to the manifest, the comments, those rules, or the tables is a change in both places. `scripts/verify-attach.test.mjs` has its tests.
 - A remote session runs in `.github/workflows/verify-remote.yml`. `src/platform/session-device.ts` holds the steps that the session runs on its device, and `src/core/remote/` of `@clerk/internal-mobile-e2e` is the code on both ends of it. The runner installs the package with `npm ci`. A session keeps the core it installed when it started, so after a change to the commit pin of `@clerk/internal-mobile-e2e`, commit, push, `down`, then `up`.
diff --git a/.github/workflows/expo-e2e-flaky.yml b/.github/workflows/expo-e2e-flaky.yml
deleted file mode 100644
index 35975f7f594..00000000000
--- a/.github/workflows/expo-e2e-flaky.yml
+++ /dev/null
@@ -1,58 +0,0 @@
-name: Expo E2E flaky tests
-
-on:
-  schedule:
-    - cron: '17 7 * * *'
-  workflow_dispatch:
-
-permissions:
-  contents: read
-
-jobs:
-  digest:
-    name: Post the flaky tests to Slack
-    if: github.repository == 'clerk/javascript'
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    permissions:
-      actions: read
-      contents: read
-    steps:
-      - name: Checkout repo
-        uses: actions/checkout@v4.4.0
-        with:
-          persist-credentials: false
-          sparse-checkout: scripts/verify-flaky.mjs
-          sparse-checkout-cone-mode: false
-
-      - uses: actions/setup-node@v4.4.0
-        with:
-          node-version: 24.15.0
-
-      - name: Post the flaky tests of the last 24 hours
-        env:
-          GH_TOKEN: ${{ github.token }}
-          GH_REPO: ${{ github.repository }}
-          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_MOBILE_TEAM_ALERTS_WEBHOOK_URL }}
-        run: |
-          set -euo pipefail
-          since="$(date -u -d '24 hours ago' +%Y-%m-%dT%H:%M:%SZ)"
-          gh run list --workflow verify-e2e.yml --branch changeset-release/main --event pull_request --created ">=$since" --limit 200 --json databaseId,conclusion > "$RUNNER_TEMP/runs.json"
-          mapfile -t ids < <(jq -r '.[] | select(.conclusion != "" and .conclusion != "cancelled" and .conclusion != "skipped") | .databaseId' "$RUNNER_TEMP/runs.json")
-          mkdir -p "$RUNNER_TEMP/flaky"
-          for id in "${ids[@]}"; do
-            gh run download "$id" --pattern 'verify-flaky-*' --dir "$RUNNER_TEMP/flaky/$id" || echo "Run $id kept no flaky tests."
-          done
-          MESSAGE="$(node scripts/verify-flaky.mjs digest "$RUNNER_TEMP/flaky" --runs "${#ids[@]}" --hours 24)"
-          if [ -z "$MESSAGE" ]; then
-            echo "No flaky tests in the last 24 hours."
-            exit 0
-          fi
-          echo "$MESSAGE"
-          if [ -z "$SLACK_WEBHOOK_URL" ]; then
-            echo "::warning::SLACK_MOBILE_TEAM_ALERTS_WEBHOOK_URL is not configured; skipping Slack notification"
-            exit 0
-          fi
-          PAYLOAD="$(jq -n --arg text "$MESSAGE" '{text: $text, unfurl_links: false, unfurl_media: false}')"
-          curl --fail-with-body --connect-timeout 10 --max-time 30 --retry 3 --retry-delay 5 --retry-max-time 60 --retry-all-errors \
-            --request POST --header "Content-type: application/json" --data "$PAYLOAD" "$SLACK_WEBHOOK_URL"
diff --git a/.github/workflows/verify-e2e.yml b/.github/workflows/verify-e2e.yml
index 45bf225cfa3..1291b22f367 100644
--- a/.github/workflows/verify-e2e.yml
+++ b/.github/workflows/verify-e2e.yml
@@ -225,16 +225,63 @@ jobs:
         if: ${{ !cancelled() && steps.evidence.outputs.upload == 'true' }}
         run: node scripts/verify-flaky.mjs record "$PACKAGE/.verify/runs" "$RUNNER_TEMP/verify-flaky.json"
 
-      - name: Keep the flaky tests for the daily digest
+      - name: Keep the flaky tests for the Slack post
         if: ${{ !cancelled() && steps.evidence.outputs.upload == 'true' }}
         uses: actions/upload-artifact@v4.6.2
         with:
           name: verify-flaky-${{ matrix.platform }}
           path: ${{ runner.temp }}/verify-flaky.json
-          retention-days: 3
+          retention-days: 1
           overwrite: true
           if-no-files-found: ignore
 
+  report-flaky:
+    name: Post the flaky tests to Slack
+    needs: e2e-tests
+    if: ${{ !cancelled() && needs.e2e-tests.result != 'skipped' && github.head_ref == 'changeset-release/main' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      contents: read
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4.4.0
+        with:
+          persist-credentials: false
+          sparse-checkout: scripts/verify-flaky.mjs
+          sparse-checkout-cone-mode: false
+
+      - uses: actions/setup-node@v4.4.0
+        with:
+          node-version: 24.15.0
+
+      - name: Download the flaky tests of both platforms
+        continue-on-error: true
+        uses: actions/download-artifact@v4.3.0
+        with:
+          pattern: verify-flaky-*
+          path: ${{ runner.temp }}/flaky
+
+      - name: Post the flaky tests
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_MOBILE_TEAM_ALERTS_WEBHOOK_URL }}
+        run: |
+          set -euo pipefail
+          mkdir -p "$RUNNER_TEMP/flaky"
+          MESSAGE="$(node scripts/verify-flaky.mjs message "$RUNNER_TEMP/flaky")"
+          if [ -z "$MESSAGE" ]; then
+            echo "No flaky tests in this run."
+            exit 0
+          fi
+          echo "$MESSAGE"
+          if [ -z "$SLACK_WEBHOOK_URL" ]; then
+            echo "::warning::SLACK_MOBILE_TEAM_ALERTS_WEBHOOK_URL is not configured; skipping Slack notification"
+            exit 0
+          fi
+          PAYLOAD="$(jq -n --arg text "$MESSAGE" '{text: $text, unfurl_links: false, unfurl_media: false}')"
+          curl --fail-with-body --connect-timeout 10 --max-time 30 --retry 3 --retry-delay 5 --retry-max-time 60 --retry-all-errors \
+            --request POST --header "Content-type: application/json" --data "$PAYLOAD" "$SLACK_WEBHOOK_URL"
+
   verify-e2e:
     name: Expo E2E
     needs: e2e-tests
diff --git a/scripts/verify-flaky.mjs b/scripts/verify-flaky.mjs
index fc1ddd18b37..3c8c9748f5c 100644
--- a/scripts/verify-flaky.mjs
+++ b/scripts/verify-flaky.mjs
@@ -2,7 +2,6 @@ import fs from 'node:fs';
 import path from 'node:path';
 import { fileURLToPath } from 'node:url';
 
-const MOST_TESTS = 10;
 const ERROR_CHARS = 300;
 
 function flakesFromReport(report) {
@@ -31,26 +30,15 @@ const slackText = text => text.replaceAll('&', '&amp;').replaceAll('<', '&lt;').
 
 const mostCommon = counts => [...counts.entries()].sort((a, b) => b[1] - a[1])[0][0];
 
-function digestMessage(records, { runs, hours }) {
+function flakyMessage(records) {
   const tests = new Map();
   for (const record of records) {
     for (const flake of record.flakes) {
       const key = `${flake.file}\n${flake.title}`;
-      const test = tests.get(key) ?? {
-        file: flake.file,
-        title: flake.title,
-        total: 0,
-        platforms: new Map(),
-        runs: new Set(),
-        errors: new Map(),
-        latestRunUrl: null,
-      };
+      const test = tests.get(key) ?? { file: flake.file, title: flake.title, platforms: new Set(), errors: new Map() };
       const error = normalizeError(flake.error);
-      test.total += 1;
-      test.platforms.set(flake.platform, (test.platforms.get(flake.platform) ?? 0) + 1);
-      test.runs.add(record.runUrl ?? record);
+      test.platforms.add(flake.platform);
       test.errors.set(error, (test.errors.get(error) ?? 0) + 1);
-      test.latestRunUrl = record.runUrl ?? test.latestRunUrl;
       tests.set(key, test);
     }
   }
@@ -58,32 +46,20 @@ function digestMessage(records, { runs, hours }) {
     return '';
   }
 
-  const sorted = [...tests.values()].sort(
-    (a, b) => b.total - a.total || a.file.localeCompare(b.file) || a.title.localeCompare(b.title),
-  );
-  const lines = [`:warning: Expo E2E had flaky tests in ${runs} Version packages runs over the last ${hours} hours`];
-  for (const test of sorted.slice(0, MOST_TESTS)) {
-    const platforms = [...test.platforms.entries()]
-      .sort(([a], [b]) => a.localeCompare(b))
-      .map(([platform, count]) => `${platform} ×${count}`)
-      .join(', ');
-    const parts = [
-      `• \`${test.file.replace(/^specs\/features\//, '')}\` ${slackText(test.title)}`,
-      `${test.runs.size} of ${runs} runs`,
-      platforms,
-    ];
-    if (test.latestRunUrl !== null) {
-      parts.push(`<${test.latestRunUrl}|latest run>`);
-    }
-    lines.push(parts.join(' · '));
+  const runUrl = records.find(record => record.runUrl)?.runUrl;
+  const count = tests.size === 1 ? '1 flaky test' : `${tests.size} flaky tests`;
+  const lines = [
+    `:warning: Expo E2E had ${count} on the Version packages pull request${runUrl ? ` · <${runUrl}|View the run>` : ''}`,
+  ];
+  const sorted = [...tests.values()].sort((a, b) => a.file.localeCompare(b.file) || a.title.localeCompare(b.title));
+  for (const test of sorted) {
+    const platforms = [...test.platforms].sort().join(', ');
+    lines.push(`• \`${test.file.replace(/^specs\/features\//, '')}\` ${slackText(test.title)} · ${platforms}`);
     const error = mostCommon(test.errors);
     if (error !== '') {
       lines.push(`  first attempt failed: ${slackText(error)}`);
     }
   }
-  if (sorted.length > MOST_TESTS) {
-    lines.push(`…and ${sorted.length - MOST_TESTS} more`);
-  }
   return lines.join('\n');
 }
 
@@ -126,26 +102,23 @@ function readRecords(dir) {
 
 function main(argv, env, log = console.log) {
   const [step, ...rest] = argv;
-  const flag = name => (rest.includes(name) ? rest[rest.indexOf(name) + 1] : undefined);
   if (step === 'record' && rest.length === 2) {
     const record = recordStep(rest[0], rest[1], env);
     log(`Recorded ${record.flakes.length} flaky tests in ${rest[1]}.`);
     return 0;
   }
-  const runs = Number(flag('--runs'));
-  const hours = Number(flag('--hours'));
-  if (step === 'digest' && rest[0] !== undefined && Number.isInteger(runs) && Number.isInteger(hours)) {
-    const message = digestMessage(readRecords(rest[0]), { runs, hours });
+  if (step === 'message' && rest.length === 1) {
+    const message = flakyMessage(readRecords(rest[0]));
     if (message !== '') {
       log(message);
     }
     return 0;
   }
-  log('usage: verify-flaky.mjs record <runsDir> <outFile> | digest <dir> --runs <n> --hours <h>');
+  log('usage: verify-flaky.mjs record <runsDir> <outFile> | message <dir>');
   return 2;
 }
 
-export { digestMessage, flakesFromReport, main, normalizeError };
+export { flakesFromReport, flakyMessage, main, normalizeError };
 
 if (process.argv[1] !== undefined && fs.realpathSync(process.argv[1]) === fileURLToPath(import.meta.url)) {
   process.exit(main(process.argv.slice(2), process.env));
diff --git a/scripts/verify-flaky.test.mjs b/scripts/verify-flaky.test.mjs
index c72b7accb37..af9704b6804 100644
--- a/scripts/verify-flaky.test.mjs
+++ b/scripts/verify-flaky.test.mjs
@@ -4,7 +4,7 @@ import path from 'node:path';
 
 import { describe, expect, it } from 'vitest';
 
-import { digestMessage, flakesFromReport, main, normalizeError } from './verify-flaky.mjs';
+import { flakesFromReport, flakyMessage, main, normalizeError } from './verify-flaky.mjs';
 
 const signedInError = (name, user, launch, session, passes = 2) =>
   `the app did not show "Signed in as ${name}+clerk_test@example.com" with user ID ${user} and a session ID on the home after launch ${launch}; it shows its error screen: Something went wrong No session was found with id ${session}; passes: ${passes}; reads: ${passes * 2}, the slowest ${passes * 70}ms (the heading)`;
@@ -103,60 +103,43 @@ describe('normalizeError', () => {
   });
 });
 
-describe('digestMessage', () => {
+describe('flakyMessage', () => {
   it('is empty when no test was flaky', () => {
-    expect(digestMessage([], { runs: 4, hours: 24 })).toBe('');
-    expect(digestMessage([{ runUrl: runUrl(1), commit: null, flakes: [] }], { runs: 4, hours: 24 })).toBe('');
+    expect(flakyMessage([])).toBe('');
+    expect(flakyMessage([{ runUrl: runUrl(1), commit: null, flakes: [] }])).toBe('');
   });
 
-  it('groups a test across runs and platforms, and links its latest run', () => {
-    const signOut = flake();
+  it('lists each flaky test once with its platforms and its first error, and links the run', () => {
     const profile = flake({
       file: 'specs/features/user-button-and-profile/profile.e2e.ts',
       title: 'the home sign-out button ends the session',
       error: 'a <b> & c',
     });
-    const message = digestMessage(
-      [
-        { runUrl: runUrl(1), commit: null, flakes: [signOut, profile] },
-        { runUrl: runUrl(1), commit: null, flakes: [flake({ platform: 'ios' })] },
-        {
-          runUrl: runUrl(2),
-          commit: null,
-          flakes: [flake({ error: 'No session was found with id sess_3L6FCP4vbLeA13m6zfP70AEVLx7' })],
-        },
-        { runUrl: runUrl(3), commit: null, flakes: [flake({ error: 'something else' })] },
-      ],
-      { runs: 5, hours: 24 },
-    );
+    const message = flakyMessage([
+      { runUrl: runUrl(1), commit: null, flakes: [profile, flake()] },
+      { runUrl: runUrl(1), commit: null, flakes: [flake({ platform: 'ios' })] },
+    ]);
 
     expect(message).toBe(
       [
-        ':warning: Expo E2E had flaky tests in 5 Version packages runs over the last 24 hours',
-        `• \`native-js-sync/sign-out-from-native.e2e.ts\` signing out in the native profile signs out the JS hooks · 3 of 5 runs · android ×3, ios ×1 · <${runUrl(3)}|latest run>`,
+        `:warning: Expo E2E had 2 flaky tests on the Version packages pull request · <${runUrl(1)}|View the run>`,
+        '• `native-js-sync/sign-out-from-native.e2e.ts` signing out in the native profile signs out the JS hooks · android, ios',
         '  first attempt failed: No session was found with id sess_&lt;id&gt;',
-        `• \`user-button-and-profile/profile.e2e.ts\` the home sign-out button ends the session · 1 of 5 runs · android ×1 · <${runUrl(1)}|latest run>`,
+        '• `user-button-and-profile/profile.e2e.ts` the home sign-out button ends the session · android',
         '  first attempt failed: a &lt;b&gt; &amp; c',
       ].join('\n'),
     );
   });
 
-  it('lists the ten most flaky tests', () => {
-    const flakes = Array.from({ length: 12 }, (_, i) =>
-      Array.from({ length: i + 1 }, () => flake({ title: `test ${i}` })),
-    ).flat();
-    const lines = digestMessage([{ runUrl: null, commit: null, flakes }], { runs: 1, hours: 24 }).split('\n');
-
-    expect(lines.filter(line => line.startsWith('•')).map(line => line.split(' · ')[0])).toEqual(
-      Array.from({ length: 10 }, (_, i) => `• \`native-js-sync/sign-out-from-native.e2e.ts\` test ${11 - i}`),
+  it('names one flaky test in the singular and leaves out the link without a run', () => {
+    expect(flakyMessage([{ runUrl: null, commit: null, flakes: [flake()] }]).split('\n')[0]).toBe(
+      ':warning: Expo E2E had 1 flaky test on the Version packages pull request',
     );
-    expect(lines[1]).toBe('• `native-js-sync/sign-out-from-native.e2e.ts` test 11 · 1 of 1 runs · android ×12');
-    expect(lines.at(-1)).toBe('…and 2 more');
   });
 });
 
 describe('the command line', () => {
-  it('records the flaky tests of every e2e report, then digests them', () => {
+  it('records the flaky tests of every e2e report, then writes the message', () => {
     const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'verify-flaky-'));
     const runs = path.join(dir, 'runs');
     const write = (file, value) => {
@@ -193,13 +176,11 @@ describe('the command line', () => {
     expect(main(['record', path.join(dir, 'no-runs'), empty], {}, () => {})).toBe(0);
     expect(JSON.parse(fs.readFileSync(empty, 'utf8'))).toEqual({ runUrl: null, commit: null, flakes: [] });
 
-    const digest = [];
-    expect(main(['digest', path.join(dir, 'records'), '--runs', '2', '--hours', '24'], {}, l => digest.push(l))).toBe(
-      0,
-    );
-    expect(digest.join('\n').split('\n')[1]).toBe(
-      `• \`native-js-sync/sign-out-from-native.e2e.ts\` signing out in the native profile signs out the JS hooks · 1 of 2 runs · android ×1, ios ×1 · <${runUrl(42)}|latest run>`,
+    const message = [];
+    expect(main(['message', path.join(dir, 'records')], {}, line => message.push(line))).toBe(0);
+    expect(message.join('\n').split('\n')[1]).toBe(
+      '• `native-js-sync/sign-out-from-native.e2e.ts` signing out in the native profile signs out the JS hooks · android, ios',
     );
-    expect(main(['digest', path.join(dir, 'records')], {}, () => {})).toBe(2);
+    expect(main(['message'], {}, () => {})).toBe(2);
   });
 });

From fe3d3df00082b0ceaf9de7c20b5c071c61f693e6 Mon Sep 17 00:00:00 2001
From: wobsoriano <sorianorobertc@gmail.com>
Date: Fri, 23 Oct 2026 10:29:59 -0700
Subject: [PATCH 3/5] ci(repo): pin download-artifact for the Expo e2e workflow

---
 .github/workflows/actions.lock | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/actions.lock b/.github/workflows/actions.lock
index 30b77fea432..520db96ba10 100644
--- a/.github/workflows/actions.lock
+++ b/.github/workflows/actions.lock
@@ -98,6 +98,7 @@ workflows:
     '.github/workflows/verify-e2e.yml':
         - 'actions/cache@v4.3.0'
         - 'actions/checkout@v4.4.0'
+        - 'actions/download-artifact@v4.3.0'
         - 'actions/setup-java@v4.9.1'
         - 'actions/setup-node@v4.4.0'
         - 'actions/upload-artifact@v4.6.2'

From 61fbb802d3ef2c88dc57deca7538b149223e47b0 Mon Sep 17 00:00:00 2001
From: wobsoriano <sorianorobertc@gmail.com>
Date: Fri, 23 Oct 2026 10:43:00 -0700
Subject: [PATCH 4/5] ci(repo): send the Expo e2e flaky tests with the Slack
 action like the other alerts

---
 .github/workflows/actions.lock   |  1 +
 .github/workflows/verify-e2e.yml | 33 +++++++------
 scripts/verify-flaky.mjs         | 38 +++++++++------
 scripts/verify-flaky.test.mjs    | 82 ++++++++++++++++++++++----------
 4 files changed, 99 insertions(+), 55 deletions(-)

diff --git a/.github/workflows/actions.lock b/.github/workflows/actions.lock
index 520db96ba10..710a02f8d5d 100644
--- a/.github/workflows/actions.lock
+++ b/.github/workflows/actions.lock
@@ -103,6 +103,7 @@ workflows:
         - 'actions/setup-node@v4.4.0'
         - 'actions/upload-artifact@v4.6.2'
         - 'pnpm/action-setup@v4.3.0'
+        - 'slackapi/slack-github-action@v1.27.1'
     '.github/workflows/verify-remote.yml':
         - 'actions/checkout@v4.4.0'
         - 'actions/setup-java@v4.9.1'
diff --git a/.github/workflows/verify-e2e.yml b/.github/workflows/verify-e2e.yml
index 1291b22f367..fbc58b3aae4 100644
--- a/.github/workflows/verify-e2e.yml
+++ b/.github/workflows/verify-e2e.yml
@@ -262,25 +262,30 @@ jobs:
           pattern: verify-flaky-*
           path: ${{ runner.temp }}/flaky
 
-      - name: Post the flaky tests
-        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_MOBILE_TEAM_ALERTS_WEBHOOK_URL }}
+      - name: Generate notification payload
+        id: notification
         run: |
-          set -euo pipefail
           mkdir -p "$RUNNER_TEMP/flaky"
-          MESSAGE="$(node scripts/verify-flaky.mjs message "$RUNNER_TEMP/flaky")"
-          if [ -z "$MESSAGE" ]; then
+          payload="$(node scripts/verify-flaky.mjs payload "$RUNNER_TEMP/flaky")"
+          if [ -z "$payload" ]; then
             echo "No flaky tests in this run."
             exit 0
           fi
-          echo "$MESSAGE"
-          if [ -z "$SLACK_WEBHOOK_URL" ]; then
-            echo "::warning::SLACK_MOBILE_TEAM_ALERTS_WEBHOOK_URL is not configured; skipping Slack notification"
-            exit 0
-          fi
-          PAYLOAD="$(jq -n --arg text "$MESSAGE" '{text: $text, unfurl_links: false, unfurl_media: false}')"
-          curl --fail-with-body --connect-timeout 10 --max-time 30 --retry 3 --retry-delay 5 --retry-max-time 60 --retry-all-errors \
-            --request POST --header "Content-type: application/json" --data "$PAYLOAD" "$SLACK_WEBHOOK_URL"
+          echo "$payload"
+          {
+            echo 'payload<<__NOTIFY_EOF__'
+            echo "$payload"
+            echo '__NOTIFY_EOF__'
+          } >> "$GITHUB_OUTPUT"
+
+      - name: Send the flaky tests to Slack
+        if: ${{ steps.notification.outputs.payload != '' }}
+        uses: slackapi/slack-github-action@v1.27.1
+        with:
+          payload: ${{ steps.notification.outputs.payload }}
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_MOBILE_TEAM_ALERTS_WEBHOOK_URL }}
+          SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
 
   verify-e2e:
     name: Expo E2E
diff --git a/scripts/verify-flaky.mjs b/scripts/verify-flaky.mjs
index 3c8c9748f5c..55322c6d727 100644
--- a/scripts/verify-flaky.mjs
+++ b/scripts/verify-flaky.mjs
@@ -30,7 +30,7 @@ const slackText = text => text.replaceAll('&', '&amp;').replaceAll('<', '&lt;').
 
 const mostCommon = counts => [...counts.entries()].sort((a, b) => b[1] - a[1])[0][0];
 
-function flakyMessage(records) {
+function flakyPayload(records) {
   const tests = new Map();
   for (const record of records) {
     for (const flake of record.flakes) {
@@ -43,24 +43,32 @@ function flakyMessage(records) {
     }
   }
   if (tests.size === 0) {
-    return '';
+    return null;
   }
 
+  const markdown = text => ({ type: 'section', text: { type: 'mrkdwn', text } });
   const runUrl = records.find(record => record.runUrl)?.runUrl;
-  const count = tests.size === 1 ? '1 flaky test' : `${tests.size} flaky tests`;
-  const lines = [
-    `:warning: Expo E2E had ${count} on the Version packages pull request${runUrl ? ` · <${runUrl}|View the run>` : ''}`,
+  const count = tests.size === 1 ? '1 test' : `${tests.size} tests`;
+  const header = [
+    '*:warning: Expo E2E flaky tests*',
+    '*Pull request:* Version packages',
+    `*Flaky:* ${count} passed only on a retry`,
+    ...(runUrl ? [`*Run:* <${runUrl}|View logs>`] : []),
   ];
   const sorted = [...tests.values()].sort((a, b) => a.file.localeCompare(b.file) || a.title.localeCompare(b.title));
+  const blocks = [markdown(header.join('\n'))];
   for (const test of sorted) {
-    const platforms = [...test.platforms].sort().join(', ');
-    lines.push(`• \`${test.file.replace(/^specs\/features\//, '')}\` ${slackText(test.title)} · ${platforms}`);
+    const lines = [
+      `\`${test.file.replace(/^specs\/features\//, '')}\` ${slackText(test.title)}`,
+      `*Platforms:* ${[...test.platforms].sort().join(', ')}`,
+    ];
     const error = mostCommon(test.errors);
     if (error !== '') {
-      lines.push(`  first attempt failed: ${slackText(error)}`);
+      lines.push(`*First attempt failed:* ${slackText(error)}`);
     }
+    blocks.push(markdown(lines.join('\n')));
   }
-  return lines.join('\n');
+  return { blocks, unfurl_links: false, unfurl_media: false };
 }
 
 const readJson = file => JSON.parse(fs.readFileSync(file, 'utf8'));
@@ -107,18 +115,18 @@ function main(argv, env, log = console.log) {
     log(`Recorded ${record.flakes.length} flaky tests in ${rest[1]}.`);
     return 0;
   }
-  if (step === 'message' && rest.length === 1) {
-    const message = flakyMessage(readRecords(rest[0]));
-    if (message !== '') {
-      log(message);
+  if (step === 'payload' && rest.length === 1) {
+    const payload = flakyPayload(readRecords(rest[0]));
+    if (payload !== null) {
+      log(JSON.stringify(payload));
     }
     return 0;
   }
-  log('usage: verify-flaky.mjs record <runsDir> <outFile> | message <dir>');
+  log('usage: verify-flaky.mjs record <runsDir> <outFile> | payload <dir>');
   return 2;
 }
 
-export { flakesFromReport, flakyMessage, main, normalizeError };
+export { flakesFromReport, flakyPayload, main, normalizeError };
 
 if (process.argv[1] !== undefined && fs.realpathSync(process.argv[1]) === fileURLToPath(import.meta.url)) {
   process.exit(main(process.argv.slice(2), process.env));
diff --git a/scripts/verify-flaky.test.mjs b/scripts/verify-flaky.test.mjs
index af9704b6804..373b1ed95ab 100644
--- a/scripts/verify-flaky.test.mjs
+++ b/scripts/verify-flaky.test.mjs
@@ -4,7 +4,7 @@ import path from 'node:path';
 
 import { describe, expect, it } from 'vitest';
 
-import { flakesFromReport, flakyMessage, main, normalizeError } from './verify-flaky.mjs';
+import { flakesFromReport, flakyPayload, main, normalizeError } from './verify-flaky.mjs';
 
 const signedInError = (name, user, launch, session, passes = 2) =>
   `the app did not show "Signed in as ${name}+clerk_test@example.com" with user ID ${user} and a session ID on the home after launch ${launch}; it shows its error screen: Something went wrong No session was found with id ${session}; passes: ${passes}; reads: ${passes * 2}, the slowest ${passes * 70}ms (the heading)`;
@@ -103,43 +103,75 @@ describe('normalizeError', () => {
   });
 });
 
-describe('flakyMessage', () => {
-  it('is empty when no test was flaky', () => {
-    expect(flakyMessage([])).toBe('');
-    expect(flakyMessage([{ runUrl: runUrl(1), commit: null, flakes: [] }])).toBe('');
+const section = text => ({ type: 'section', text: { type: 'mrkdwn', text } });
+
+describe('flakyPayload', () => {
+  it('is null when no test was flaky', () => {
+    expect(flakyPayload([])).toBeNull();
+    expect(flakyPayload([{ runUrl: runUrl(1), commit: null, flakes: [] }])).toBeNull();
   });
 
-  it('lists each flaky test once with its platforms and its first error, and links the run', () => {
+  it('gives each flaky test its own block with its platforms and first error, after a header that links the run', () => {
     const profile = flake({
       file: 'specs/features/user-button-and-profile/profile.e2e.ts',
       title: 'the home sign-out button ends the session',
       error: 'a <b> & c',
     });
-    const message = flakyMessage([
+    const payload = flakyPayload([
       { runUrl: runUrl(1), commit: null, flakes: [profile, flake()] },
       { runUrl: runUrl(1), commit: null, flakes: [flake({ platform: 'ios' })] },
     ]);
 
-    expect(message).toBe(
-      [
-        `:warning: Expo E2E had 2 flaky tests on the Version packages pull request · <${runUrl(1)}|View the run>`,
-        '• `native-js-sync/sign-out-from-native.e2e.ts` signing out in the native profile signs out the JS hooks · android, ios',
-        '  first attempt failed: No session was found with id sess_&lt;id&gt;',
-        '• `user-button-and-profile/profile.e2e.ts` the home sign-out button ends the session · android',
-        '  first attempt failed: a &lt;b&gt; &amp; c',
-      ].join('\n'),
-    );
+    expect(payload).toEqual({
+      blocks: [
+        section(
+          [
+            '*:warning: Expo E2E flaky tests*',
+            '*Pull request:* Version packages',
+            '*Flaky:* 2 tests passed only on a retry',
+            `*Run:* <${runUrl(1)}|View logs>`,
+          ].join('\n'),
+        ),
+        section(
+          [
+            '`native-js-sync/sign-out-from-native.e2e.ts` signing out in the native profile signs out the JS hooks',
+            '*Platforms:* android, ios',
+            '*First attempt failed:* No session was found with id sess_&lt;id&gt;',
+          ].join('\n'),
+        ),
+        section(
+          [
+            '`user-button-and-profile/profile.e2e.ts` the home sign-out button ends the session',
+            '*Platforms:* android',
+            '*First attempt failed:* a &lt;b&gt; &amp; c',
+          ].join('\n'),
+        ),
+      ],
+      unfurl_links: false,
+      unfurl_media: false,
+    });
   });
 
-  it('names one flaky test in the singular and leaves out the link without a run', () => {
-    expect(flakyMessage([{ runUrl: null, commit: null, flakes: [flake()] }]).split('\n')[0]).toBe(
-      ':warning: Expo E2E had 1 flaky test on the Version packages pull request',
+  it('names one flaky test in the singular and leaves out the run without a link', () => {
+    expect(flakyPayload([{ runUrl: null, commit: null, flakes: [flake()] }]).blocks[0]).toEqual(
+      section(
+        [
+          '*:warning: Expo E2E flaky tests*',
+          '*Pull request:* Version packages',
+          '*Flaky:* 1 test passed only on a retry',
+        ].join('\n'),
+      ),
     );
   });
+
+  it('keeps every block under the Slack limit of 3000 characters', () => {
+    const payload = flakyPayload([{ runUrl: runUrl(1), commit: null, flakes: [flake({ error: 'x'.repeat(5000) })] }]);
+    expect(Math.max(...payload.blocks.map(block => block.text.text.length))).toBeLessThan(3000);
+  });
 });
 
 describe('the command line', () => {
-  it('records the flaky tests of every e2e report, then writes the message', () => {
+  it('records the flaky tests of every e2e report, then writes the Slack payload', () => {
     const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'verify-flaky-'));
     const runs = path.join(dir, 'runs');
     const write = (file, value) => {
@@ -176,11 +208,9 @@ describe('the command line', () => {
     expect(main(['record', path.join(dir, 'no-runs'), empty], {}, () => {})).toBe(0);
     expect(JSON.parse(fs.readFileSync(empty, 'utf8'))).toEqual({ runUrl: null, commit: null, flakes: [] });
 
-    const message = [];
-    expect(main(['message', path.join(dir, 'records')], {}, line => message.push(line))).toBe(0);
-    expect(message.join('\n').split('\n')[1]).toBe(
-      '• `native-js-sync/sign-out-from-native.e2e.ts` signing out in the native profile signs out the JS hooks · android, ios',
-    );
-    expect(main(['message'], {}, () => {})).toBe(2);
+    const printed = [];
+    expect(main(['payload', path.join(dir, 'records')], {}, line => printed.push(line))).toBe(0);
+    expect(JSON.parse(printed.join('\n')).blocks[1].text.text.split('\n')[1]).toBe('*Platforms:* android, ios');
+    expect(main(['payload'], {}, () => {})).toBe(2);
   });
 });

From 74f8c0aac888f876fe13a9d9fd08a8b02a88644e Mon Sep 17 00:00:00 2001
From: wobsoriano <sorianorobertc@gmail.com>
Date: Fri, 23 Oct 2026 11:14:55 -0700
Subject: [PATCH 5/5] ci(repo): keep the flaky test record from failing the
 Expo e2e jobs

---
 .github/workflows/verify-e2e.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/verify-e2e.yml b/.github/workflows/verify-e2e.yml
index fbc58b3aae4..81437d712db 100644
--- a/.github/workflows/verify-e2e.yml
+++ b/.github/workflows/verify-e2e.yml
@@ -223,10 +223,12 @@ jobs:
 
       - name: Record the flaky tests
         if: ${{ !cancelled() && steps.evidence.outputs.upload == 'true' }}
+        continue-on-error: true
         run: node scripts/verify-flaky.mjs record "$PACKAGE/.verify/runs" "$RUNNER_TEMP/verify-flaky.json"
 
       - name: Keep the flaky tests for the Slack post
         if: ${{ !cancelled() && steps.evidence.outputs.upload == 'true' }}
+        continue-on-error: true
         uses: actions/upload-artifact@v4.6.2
         with:
           name: verify-flaky-${{ matrix.platform }}