diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index d4e5abbce2..339155cea5 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -24,6 +24,11 @@ jobs: E2E_PASSWORD: password E2E_TENANT_ID: default NODE_TLS_REJECT_UNAUTHORIZED: '0' + # Written by the `json` reporter in `npm run playwright:ci`. It sits inside + # playwright-report/ so the existing upload keeps it on every run. That only + # holds while `json` is listed after `html`: reporters finish in the order + # listed, and the html reporter clears this folder when it finishes. + PLAYWRIGHT_JSON_OUTPUT_FILE: playwright-report/results.json steps: - name: Checkout code @@ -53,6 +58,12 @@ jobs: - name: Run Playwright tests run: npm run playwright:ci + - name: Summarize flaky tests + if: always() + # Reporting only: a problem in this step must never change the job result. + continue-on-error: true + run: node scripts/summarize-flaky-tests.js "$PLAYWRIGHT_JSON_OUTPUT_FILE" + - name: Dump Docker logs on failure if: failure() run: | diff --git a/package.json b/package.json index 02f78fb1a0..a50e567c4d 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ "playwright:ui": "playwright test --ui", "playwright:headed": "playwright test --headed", "playwright:debug": "playwright test --debug", - "playwright:ci": "playwright test --reporter=html,github --workers=1", + "playwright:ci": "playwright test --reporter=html,github,json --workers=1", "e2e:docker": "bash scripts/e2e-docker.sh", "e2e:docker:up": "docker compose -f docker-compose.e2e.yml up -d --build", "e2e:docker:down": "docker compose -f docker-compose.e2e.yml down -v --remove-orphans", diff --git a/scripts/summarize-flaky-tests.js b/scripts/summarize-flaky-tests.js new file mode 100644 index 0000000000..5bec5de5b4 --- /dev/null +++ b/scripts/summarize-flaky-tests.js @@ -0,0 +1,95 @@ +#!/usr/bin/env node + +/** + * Copyright since 2025 Mifos Initiative + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +const fs = require('fs'); +const path = require('path'); + +const HEADING = '## Playwright E2E — flaky tests'; +const MAX_ERROR_LENGTH = 160; + +function collectFlakyTests(report) { + const flaky = []; + const visit = (suite, titles) => { + for (const spec of suite.specs) { + for (const test of spec.tests) { + // Playwright's own outcome for "failed at least once, then passed on a retry". + if (test.status === 'flaky') { + flaky.push({ spec, titles: [...titles, spec.title], test }); + } + } + } + for (const child of suite.suites ?? []) { + visit(child, [...titles, child.title]); + } + }; + // Top-level suites are files, and their title is the path, which is rendered separately. + for (const fileSuite of report.suites) { + visit(fileSuite, []); + } + return flaky; +} + +function repoPath(rootDir, file) { + return path.relative(process.cwd(), path.resolve(rootDir, file)).split(path.sep).join('/'); +} + +function firstFailureMessage(results) { + const failed = results.find((result) => result.status !== 'passed' && result.status !== 'skipped'); + const message = failed?.errors?.[0]?.message ?? failed?.error?.message ?? ''; + const firstLine = + message + .replace(/\u001b\[[0-9;]*m/g, '') + .split('\n') + .map((line) => line.trim()) + .find(Boolean) ?? ''; + return firstLine.length > MAX_ERROR_LENGTH ? `${firstLine.slice(0, MAX_ERROR_LENGTH - 1)}…` : firstLine; +} + +function escapeCell(text) { + return text.replace(/\r?\n/g, ' ').replace(/\|/g, '\\|').replace(//g, '>'); +} + +function renderSummary(report) { + const { expected, unexpected, flaky, skipped } = report.stats; + const lines = [HEADING, '', `**${flaky} flaky** · ${expected} passed · ${unexpected} failed · ${skipped} skipped`]; + const flakyTests = collectFlakyTests(report); + if (flakyTests.length > 0) { + lines.push( + '', + 'Each of these failed at least once and then passed on a retry, so the run did not fail.', + '', + '| Test | Project | Attempts | First failure |', + '| --- | --- | --- | --- |' + ); + for (const { spec, titles, test } of flakyTests) { + const location = `\`${repoPath(report.config.rootDir, spec.file)}:${spec.line}\``; + const name = escapeCell(titles.join(' › ')); + const error = escapeCell(firstFailureMessage(test.results)); + lines.push(`| ${location} › ${name} | ${escapeCell(test.projectName)} | ${test.results.length} | ${error} |`); + } + } + return `${lines.join('\n')}\n`; +} + +const reportPath = process.argv[2]; +if (!reportPath) { + console.error('Usage: node scripts/summarize-flaky-tests.js '); + process.exit(1); +} + +const summary = fs.existsSync(reportPath) + ? renderSummary(JSON.parse(fs.readFileSync(reportPath, 'utf8'))) + : `${HEADING}\n\nNo JSON report at \`${reportPath}\`: the test step did not run to completion.\n`; + +if (process.env.GITHUB_STEP_SUMMARY) { + fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, summary); +} else { + process.stdout.write(summary); +}