From 66cf5b69c085335350fd105e19773603fe4dee82 Mon Sep 17 00:00:00 2001 From: Devansh Vashisht Date: Sat, 12 Sep 2026 01:27:17 +0530 Subject: [PATCH] WEB-1238: [CI/CD] Surface retry outcomes from Playwright E2E as a flake signal playwright.config.ts retries failed tests twice in CI. That keeps PRs unblocked on transient noise, but a test can fail and then pass on a retry inside a run that ends green. The github reporter names such tests in its run-summary annotation, yet the failed attempt is annotated as an error indistinguishable from a real failure, nothing is written to the job summary, and nothing is kept in machine-readable form. Add the json reporter to `playwright:ci`, writing to playwright-report/results.json so the existing always-on upload keeps it, and add a step that renders the report into the job summary: pass, fail, skip and flaky counts on every run, plus a table naming each flaky test with its project, attempt count and first failure. The json reporter is listed after html on purpose. Reporters finish in the order listed and the html reporter clears its output folder when it finishes, so the reverse order deletes results.json. The summary step runs with continue-on-error, so it can report on any run but never change its result. Retry and worker settings are unchanged. --- .github/workflows/playwright.yml | 11 ++++ package.json | 2 +- scripts/summarize-flaky-tests.js | 95 ++++++++++++++++++++++++++++++++ 3 files changed, 107 insertions(+), 1 deletion(-) create mode 100644 scripts/summarize-flaky-tests.js diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index d4e5abbce2..339155cea5 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -24,6 +24,11 @@ jobs: E2E_PASSWORD: password E2E_TENANT_ID: default NODE_TLS_REJECT_UNAUTHORIZED: '0' + # Written by the `json` reporter in `npm run playwright:ci`. It sits inside + # playwright-report/ so the existing upload keeps it on every run. That only + # holds while `json` is listed after `html`: reporters finish in the order + # listed, and the html reporter clears this folder when it finishes. + PLAYWRIGHT_JSON_OUTPUT_FILE: playwright-report/results.json steps: - name: Checkout code @@ -53,6 +58,12 @@ jobs: - name: Run Playwright tests run: npm run playwright:ci + - name: Summarize flaky tests + if: always() + # Reporting only: a problem in this step must never change the job result. + continue-on-error: true + run: node scripts/summarize-flaky-tests.js "$PLAYWRIGHT_JSON_OUTPUT_FILE" + - name: Dump Docker logs on failure if: failure() run: | diff --git a/package.json b/package.json index 02f78fb1a0..a50e567c4d 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ "playwright:ui": "playwright test --ui", "playwright:headed": "playwright test --headed", "playwright:debug": "playwright test --debug", - "playwright:ci": "playwright test --reporter=html,github --workers=1", + "playwright:ci": "playwright test --reporter=html,github,json --workers=1", "e2e:docker": "bash scripts/e2e-docker.sh", "e2e:docker:up": "docker compose -f docker-compose.e2e.yml up -d --build", "e2e:docker:down": "docker compose -f docker-compose.e2e.yml down -v --remove-orphans", diff --git a/scripts/summarize-flaky-tests.js b/scripts/summarize-flaky-tests.js new file mode 100644 index 0000000000..5bec5de5b4 --- /dev/null +++ b/scripts/summarize-flaky-tests.js @@ -0,0 +1,95 @@ +#!/usr/bin/env node + +/** + * Copyright since 2025 Mifos Initiative + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +const fs = require('fs'); +const path = require('path'); + +const HEADING = '## Playwright E2E — flaky tests'; +const MAX_ERROR_LENGTH = 160; + +function collectFlakyTests(report) { + const flaky = []; + const visit = (suite, titles) => { + for (const spec of suite.specs) { + for (const test of spec.tests) { + // Playwright's own outcome for "failed at least once, then passed on a retry". + if (test.status === 'flaky') { + flaky.push({ spec, titles: [...titles, spec.title], test }); + } + } + } + for (const child of suite.suites ?? []) { + visit(child, [...titles, child.title]); + } + }; + // Top-level suites are files, and their title is the path, which is rendered separately. + for (const fileSuite of report.suites) { + visit(fileSuite, []); + } + return flaky; +} + +function repoPath(rootDir, file) { + return path.relative(process.cwd(), path.resolve(rootDir, file)).split(path.sep).join('/'); +} + +function firstFailureMessage(results) { + const failed = results.find((result) => result.status !== 'passed' && result.status !== 'skipped'); + const message = failed?.errors?.[0]?.message ?? failed?.error?.message ?? ''; + const firstLine = + message + .replace(/\u001b\[[0-9;]*m/g, '') + .split('\n') + .map((line) => line.trim()) + .find(Boolean) ?? ''; + return firstLine.length > MAX_ERROR_LENGTH ? `${firstLine.slice(0, MAX_ERROR_LENGTH - 1)}…` : firstLine; +} + +function escapeCell(text) { + return text.replace(/\r?\n/g, ' ').replace(/\|/g, '\\|').replace(//g, '>'); +} + +function renderSummary(report) { + const { expected, unexpected, flaky, skipped } = report.stats; + const lines = [HEADING, '', `**${flaky} flaky** · ${expected} passed · ${unexpected} failed · ${skipped} skipped`]; + const flakyTests = collectFlakyTests(report); + if (flakyTests.length > 0) { + lines.push( + '', + 'Each of these failed at least once and then passed on a retry, so the run did not fail.', + '', + '| Test | Project | Attempts | First failure |', + '| --- | --- | --- | --- |' + ); + for (const { spec, titles, test } of flakyTests) { + const location = `\`${repoPath(report.config.rootDir, spec.file)}:${spec.line}\``; + const name = escapeCell(titles.join(' › ')); + const error = escapeCell(firstFailureMessage(test.results)); + lines.push(`| ${location} › ${name} | ${escapeCell(test.projectName)} | ${test.results.length} | ${error} |`); + } + } + return `${lines.join('\n')}\n`; +} + +const reportPath = process.argv[2]; +if (!reportPath) { + console.error('Usage: node scripts/summarize-flaky-tests.js '); + process.exit(1); +} + +const summary = fs.existsSync(reportPath) + ? renderSummary(JSON.parse(fs.readFileSync(reportPath, 'utf8'))) + : `${HEADING}\n\nNo JSON report at \`${reportPath}\`: the test step did not run to completion.\n`; + +if (process.env.GITHUB_STEP_SUMMARY) { + fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, summary); +} else { + process.stdout.write(summary); +}