From 6b7773e917688f0a1c8aedb73ee5c70231f8aa1f Mon Sep 17 00:00:00 2001 From: omermorad Date: Sun, 27 Sep 2026 22:20:49 +0300 Subject: [PATCH 01/11] feat: run E2E against Dockerized SUT --- .../merge-e2e-evidence/assemble-evidence.mjs | 27 ++- .github/actions/run-e2e/action.yml | 198 +++++++++++++++--- .github/actions/run-e2e/complete-shard.mjs | 157 +++++++++++++- .../actions/run-e2e/playwright-selection.mjs | 57 +++++ .github/actions/run-e2e/prepare-shard.mjs | 70 ++++++- .github/workflows/e2e.yml | 24 +++ .github/workflows/reusable-e2e.yml | 32 ++- README.md | 148 ++++++++----- compose.e2e.yml | 15 ++ e2e/playwright.config.ts | 17 +- e2e/tests/echo.api.spec.ts | 86 ++++---- e2e/tests/echo.browser.spec.ts | 18 +- sut/.dockerignore | 3 + sut/Dockerfile | 18 ++ {e2e/sut => sut}/server.mjs | 2 +- 15 files changed, 722 insertions(+), 150 deletions(-) create mode 100644 .github/actions/run-e2e/playwright-selection.mjs create mode 100644 compose.e2e.yml create mode 100644 sut/.dockerignore create mode 100644 sut/Dockerfile rename {e2e/sut => sut}/server.mjs (98%) diff --git a/.github/actions/merge-e2e-evidence/assemble-evidence.mjs b/.github/actions/merge-e2e-evidence/assemble-evidence.mjs index 9e227cb..c55c185 100644 --- a/.github/actions/merge-e2e-evidence/assemble-evidence.mjs +++ b/.github/actions/merge-e2e-evidence/assemble-evidence.mjs @@ -93,6 +93,9 @@ for (const fileName of ctrfFiles) { if (summary.tests === 0 && combinedTests.length > 0) { summary.tests = combinedTests.length; } +if (summary.tests === 0) { + infrastructureFailures.push("No Playwright tests were executed"); +} if (starts.length > 0) summary.start = Math.min(...starts); if (stops.length > 0) summary.stop = Math.max(...stops); @@ -118,14 +121,34 @@ writeFileSync( const failedShards = (scan.statuses ?? []).filter( (status) => status.result === "failed", ); +const infrastructureErrorShards = (scan.statuses ?? []).filter( + (status) => status.result === "infrastructure-error", +); const incompleteShards = (scan.statuses ?? []).filter( - (status) => !["passed", "failed"].includes(status.result), + (status) => + !["passed", "failed", "infrastructure-error"].includes(status.result), +); +const shardInfrastructureFailures = (scan.statuses ?? []).flatMap((status) => + [status.primaryFailure, ...(status.secondaryFailures ?? [])] + .filter((failure) => failure?.kind === "infrastructure") + .map( + (failure) => + `Shard ${status.shard.index} ${failure.phase ?? "infrastructure"}: ${failure.message}`, + ), ); +if (infrastructureErrorShards.length > 0 && shardInfrastructureFailures.length === 0) { + shardInfrastructureFailures.push( + ...infrastructureErrorShards.map( + (status) => `Shard ${status.shard.index} reported an infrastructure error`, + ), + ); +} for (const status of incompleteShards) { - infrastructureFailures.push( + shardInfrastructureFailures.push( `Shard ${status.shard.index} did not complete; last state is ${status.result ?? "unknown"}`, ); } +infrastructureFailures.unshift(...shardInfrastructureFailures); const hasTestFailure = failedShards.length > 0 || summary.failed > 0; let primaryFailure = null; diff --git a/.github/actions/run-e2e/action.yml b/.github/actions/run-e2e/action.yml index 776c5b2..ff9ba52 100644 --- a/.github/actions/run-e2e/action.yml +++ b/.github/actions/run-e2e/action.yml @@ -10,6 +10,26 @@ inputs: description: Path to the Playwright configuration, relative to working-directory. required: false default: playwright.config.ts + compose-file: + description: Docker Compose file for the SUT, relative to the repository root. + required: false + default: compose.e2e.yml + base-url: + description: Base URL exposed by the Dockerized SUT. + required: false + default: http://127.0.0.1:4173 + projects: + description: Newline-delimited Playwright project names; empty selects all projects. + required: false + default: "" + labels: + description: Newline-delimited Playwright tags beginning with @; empty selects all tags. + required: false + default: "" + label-match: + description: Label matching mode, either all or any. + required: false + default: all shard-index: description: One-based shard index. required: true @@ -23,27 +43,32 @@ inputs: outputs: result: - description: Shard result, either passed or failed. - value: ${{ steps.execute.outputs.result }} + description: "Shard result: passed, failed, or infrastructure-error." + value: ${{ steps.finalize.outputs.result }} exit-code: - description: Playwright process exit code. - value: ${{ steps.execute.outputs.exit-code }} + description: Effective shard exit code after SUT, Playwright, evidence, and cleanup checks. + value: ${{ steps.finalize.outputs.exit-code }} shard-directory: description: Absolute path containing this shard's evidence. - value: ${{ steps.execute.outputs.shard-directory }} + value: ${{ steps.initialize.outputs.shard-directory }} status-file: description: Absolute path to this shard's status JSON. - value: ${{ steps.execute.outputs.status-file }} + value: ${{ steps.initialize.outputs.status-file }} runs: using: composite steps: - - name: Run Playwright shard - id: execute + - name: Initialize shard evidence + id: initialize shell: bash env: E2E_WORKING_DIRECTORY: ${{ inputs.working-directory }} E2E_PLAYWRIGHT_CONFIG: ${{ inputs.playwright-config }} + E2E_COMPOSE_FILE: ${{ inputs.compose-file }} + E2E_BASE_URL: ${{ inputs.base-url }} + E2E_PROJECTS: ${{ inputs.projects }} + E2E_LABELS: ${{ inputs.labels }} + E2E_LABEL_MATCH: ${{ inputs.label-match }} E2E_SHARD_INDEX: ${{ inputs.shard-index }} E2E_SHARD_TOTAL: ${{ inputs.shard-total }} E2E_RESULTS_DIRECTORY: ${{ inputs.results-directory }} @@ -54,40 +79,157 @@ runs: "$GITHUB_WORKSPACE" \ "$E2E_WORKING_DIRECTORY" \ "$E2E_PLAYWRIGHT_CONFIG" \ + "$E2E_COMPOSE_FILE" \ + "$E2E_BASE_URL" \ + "$E2E_PROJECTS" \ + "$E2E_LABELS" \ + "$E2E_LABEL_MATCH" \ "$E2E_SHARD_INDEX" \ "$E2E_SHARD_TOTAL" \ "$E2E_RESULTS_DIRECTORY")" + { + echo "working-directory=$E2E_ABSOLUTE_WORKING_DIRECTORY" + echo "config=$E2E_ABSOLUTE_CONFIG" + echo "compose-file=$E2E_ABSOLUTE_COMPOSE_FILE" + echo "base-url=$E2E_BASE_URL" + echo "compose-project=$E2E_COMPOSE_PROJECT" + echo "selection-file=$E2E_SELECTION_FILE" + echo "shard-directory=$E2E_SHARD_DIRECTORY" + echo "status-file=$E2E_STATUS_FILE" + } >> "$GITHUB_OUTPUT" + + - name: Start Dockerized SUT + id: start + if: always() && steps.initialize.outcome == 'success' + continue-on-error: true + shell: bash + env: + E2E_COMPOSE_FILE: ${{ steps.initialize.outputs.compose-file }} + E2E_COMPOSE_PROJECT: ${{ steps.initialize.outputs.compose-project }} + E2E_SHARD_DIRECTORY: ${{ steps.initialize.outputs.shard-directory }} + run: | + set +e + set -uo pipefail + startup_log="$E2E_SHARD_DIRECTORY/docker-compose-startup.log" + docker compose \ + --file "$E2E_COMPOSE_FILE" \ + --project-name "$E2E_COMPOSE_PROJECT" \ + up --detach --build --wait > "$startup_log" 2>&1 + startup_exit_code=$? + cat "$startup_log" + echo "exit-code=$startup_exit_code" >> "$GITHUB_OUTPUT" + exit "$startup_exit_code" + + - name: Run Playwright shard + id: playwright + if: steps.initialize.outcome == 'success' && steps.start.outputs.exit-code == '0' + continue-on-error: true + shell: bash + env: + E2E_WORKING_DIRECTORY: ${{ steps.initialize.outputs.working-directory }} + E2E_CONFIG: ${{ steps.initialize.outputs.config }} + E2E_BASE_URL: ${{ steps.initialize.outputs.base-url }} + E2E_SHARD_DIRECTORY: ${{ steps.initialize.outputs.shard-directory }} + E2E_SELECTION_FILE: ${{ steps.initialize.outputs.selection-file }} + E2E_SHARD_INDEX: ${{ inputs.shard-index }} + E2E_SHARD_TOTAL: ${{ inputs.shard-total }} + run: | set +e + selection_args_file="$E2E_SHARD_DIRECTORY/.playwright-selection-args" + node "$GITHUB_ACTION_PATH/playwright-selection.mjs" \ + emit-args "$E2E_SELECTION_FILE" > "$selection_args_file" + selection_exit_code=$? + if [[ "$selection_exit_code" -ne 0 ]]; then + echo "exit-code=$selection_exit_code" >> "$GITHUB_OUTPUT" + echo "failure-kind=infrastructure" >> "$GITHUB_OUTPUT" + exit "$selection_exit_code" + fi + + playwright_selection_args=() + while IFS= read -r -d '' argument; do + playwright_selection_args+=("$argument") + done < "$selection_args_file" ( - cd "$E2E_ABSOLUTE_WORKING_DIRECTORY" + cd "$E2E_WORKING_DIRECTORY" + BASE_URL="$E2E_BASE_URL" \ CI=true \ E2E_RESULTS_DIR="$E2E_SHARD_DIRECTORY/test-results" \ PLAYWRIGHT_BLOB_OUTPUT_DIR="$E2E_SHARD_DIRECTORY/blob-report" \ PLAYWRIGHT_OUTPUT_DIR="$E2E_SHARD_DIRECTORY/test-results" \ CTRF_OUTPUT_FILE="$E2E_SHARD_DIRECTORY/ctrf/ctrf-report.json" \ pnpm exec playwright test \ - --config "$E2E_ABSOLUTE_CONFIG" \ - --shard="$E2E_SHARD_INDEX/$E2E_SHARD_TOTAL" + --config "$E2E_CONFIG" \ + --shard="$E2E_SHARD_INDEX/$E2E_SHARD_TOTAL" \ + "${playwright_selection_args[@]}" ) playwright_exit_code=$? - set -e - - node "$GITHUB_ACTION_PATH/complete-shard.mjs" \ - "$E2E_STATUS_FILE" \ - "$playwright_exit_code" + echo "exit-code=$playwright_exit_code" >> "$GITHUB_OUTPUT" + echo "failure-kind=test" >> "$GITHUB_OUTPUT" + exit "$playwright_exit_code" - if [[ "$playwright_exit_code" -eq 0 ]]; then - result=passed - else - result=failed - fi + - name: Capture Dockerized SUT logs + id: logs + if: always() && steps.initialize.outcome == 'success' + continue-on-error: true + shell: bash + env: + E2E_COMPOSE_FILE: ${{ steps.initialize.outputs.compose-file }} + E2E_COMPOSE_PROJECT: ${{ steps.initialize.outputs.compose-project }} + E2E_SHARD_DIRECTORY: ${{ steps.initialize.outputs.shard-directory }} + run: | + set +e + service_log="$E2E_SHARD_DIRECTORY/docker-compose.log" + docker compose \ + --file "$E2E_COMPOSE_FILE" \ + --project-name "$E2E_COMPOSE_PROJECT" \ + logs --no-color --timestamps > "$service_log" 2>&1 + log_exit_code=$? + cat "$service_log" + echo "exit-code=$log_exit_code" >> "$GITHUB_OUTPUT" + exit "$log_exit_code" - { - echo "result=$result" - echo "exit-code=$playwright_exit_code" - echo "shard-directory=$E2E_SHARD_DIRECTORY" - echo "status-file=$E2E_STATUS_FILE" - } >> "$GITHUB_OUTPUT" + - name: Tear down Dockerized SUT + id: teardown + if: always() && steps.initialize.outcome == 'success' + continue-on-error: true + shell: bash + env: + E2E_COMPOSE_FILE: ${{ steps.initialize.outputs.compose-file }} + E2E_COMPOSE_PROJECT: ${{ steps.initialize.outputs.compose-project }} + E2E_SHARD_DIRECTORY: ${{ steps.initialize.outputs.shard-directory }} + run: | + set +e + teardown_log="$E2E_SHARD_DIRECTORY/docker-compose-teardown.log" + docker compose \ + --file "$E2E_COMPOSE_FILE" \ + --project-name "$E2E_COMPOSE_PROJECT" \ + down --volumes --remove-orphans > "$teardown_log" 2>&1 + teardown_exit_code=$? + cat "$teardown_log" + echo "exit-code=$teardown_exit_code" >> "$GITHUB_OUTPUT" + exit "$teardown_exit_code" - exit "$playwright_exit_code" + - name: Finalize shard evidence + id: finalize + if: always() && steps.initialize.outcome == 'success' + shell: bash + env: + E2E_STATUS_FILE: ${{ steps.initialize.outputs.status-file }} + E2E_STARTUP_EXIT_CODE: ${{ steps.start.outputs.exit-code }} + E2E_PLAYWRIGHT_EXIT_CODE: ${{ steps.playwright.outputs.exit-code }} + E2E_PLAYWRIGHT_FAILURE_KIND: ${{ steps.playwright.outputs.failure-kind }} + E2E_LOG_EXIT_CODE: ${{ steps.logs.outputs.exit-code }} + E2E_TEARDOWN_EXIT_CODE: ${{ steps.teardown.outputs.exit-code }} + run: | + set +e + node "$GITHUB_ACTION_PATH/complete-shard.mjs" \ + "$E2E_STATUS_FILE" \ + "$E2E_STARTUP_EXIT_CODE" \ + "$E2E_PLAYWRIGHT_EXIT_CODE" \ + "$E2E_PLAYWRIGHT_FAILURE_KIND" \ + "$E2E_LOG_EXIT_CODE" \ + "$E2E_TEARDOWN_EXIT_CODE" \ + "$GITHUB_OUTPUT" + completion_exit_code=$? + exit "$completion_exit_code" diff --git a/.github/actions/run-e2e/complete-shard.mjs b/.github/actions/run-e2e/complete-shard.mjs index 384415a..2be8091 100644 --- a/.github/actions/run-e2e/complete-shard.mjs +++ b/.github/actions/run-e2e/complete-shard.mjs @@ -1,19 +1,160 @@ -import { readFileSync, writeFileSync } from "node:fs"; +import { appendFileSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname } from "node:path"; -const [statusFile, requestedExitCode] = process.argv.slice(2); -const exitCode = Number(requestedExitCode); +const [ + statusFile, + requestedStartupExitCode, + requestedPlaywrightExitCode, + requestedPlaywrightFailureKind, + requestedLogExitCode, + requestedTeardownExitCode, + outputFile, +] = process.argv.slice(2); +const parseExitCode = (requestedValue, label) => { + if (requestedValue === "") return null; + const value = Number(requestedValue); + if (!Number.isSafeInteger(value) || value < 0) { + throw new Error(`Invalid ${label} exit code: ${requestedValue}`); + } + return value; +}; -if (!Number.isSafeInteger(exitCode) || exitCode < 0) { - throw new Error(`Invalid Playwright exit code: ${requestedExitCode}`); -} +const startupExitCode = parseExitCode( + requestedStartupExitCode, + "startup", +); +const playwrightExitCode = parseExitCode( + requestedPlaywrightExitCode, + "Playwright", +); +const logExitCode = parseExitCode(requestedLogExitCode, "log capture"); +const teardownExitCode = parseExitCode( + requestedTeardownExitCode, + "teardown", +); const status = JSON.parse(readFileSync(statusFile, "utf8")); const finishedAt = new Date(); const startedAt = new Date(status.startedAt); -status.result = exitCode === 0 ? "passed" : "failed"; -status.exitCode = exitCode; +const failures = []; +if (startupExitCode === null) { + failures.push({ + kind: "infrastructure", + phase: "sut-startup", + message: "Dockerized SUT startup did not report an exit code", + }); +} else if (startupExitCode !== 0) { + failures.push({ + kind: "infrastructure", + phase: "sut-startup", + message: `Dockerized SUT startup exited with ${startupExitCode}`, + }); +} +if (startupExitCode === 0 && playwrightExitCode === null) { + failures.push({ + kind: "infrastructure", + phase: "playwright-execution", + message: "Playwright execution did not report an exit code", + }); +} else if ( + playwrightExitCode !== null && + playwrightExitCode > 0 && + requestedPlaywrightFailureKind === "infrastructure" +) { + failures.push({ + kind: "infrastructure", + phase: "playwright-selection", + message: `Playwright selection preparation exited with ${playwrightExitCode}`, + }); +} else if (playwrightExitCode !== null && playwrightExitCode > 0) { + failures.push({ + kind: "test", + phase: "playwright", + message: `Playwright exited with ${playwrightExitCode}`, + }); +} +if (logExitCode === null) { + failures.push({ + kind: "infrastructure", + phase: "evidence", + message: "Docker Compose log capture did not report an exit code", + }); +} else if (logExitCode !== 0) { + failures.push({ + kind: "infrastructure", + phase: "evidence", + message: `Docker Compose log capture exited with ${logExitCode}`, + }); +} +if (teardownExitCode === null) { + failures.push({ + kind: "infrastructure", + phase: "cleanup", + message: "Docker Compose teardown did not report an exit code", + }); +} else if (teardownExitCode !== 0) { + failures.push({ + kind: "infrastructure", + phase: "cleanup", + message: `Docker Compose teardown exited with ${teardownExitCode}`, + }); +} + +const testFailure = failures.find((failure) => failure.kind === "test"); +const infrastructureFailure = failures.find( + (failure) => failure.kind === "infrastructure", +); +status.primaryFailure = testFailure ?? infrastructureFailure ?? null; +status.secondaryFailures = failures.filter( + (failure) => failure !== status.primaryFailure, +); +status.result = testFailure + ? "failed" + : infrastructureFailure + ? "infrastructure-error" + : "passed"; + +const effectiveExitCode = !status.primaryFailure + ? 0 + : status.primaryFailure.phase === "playwright" + ? (playwrightExitCode ?? 1) + : startupExitCode !== null && startupExitCode !== 0 + ? startupExitCode + : logExitCode !== null && logExitCode !== 0 + ? logExitCode + : teardownExitCode !== null && teardownExitCode !== 0 + ? teardownExitCode + : 1; +status.exitCode = effectiveExitCode; +status.lifecycle.sut.startupExitCode = startupExitCode; +status.lifecycle.playwright = { + exitCode: playwrightExitCode, + started: playwrightExitCode !== null, +}; +status.lifecycle.evidence.logCaptureExitCode = logExitCode; +status.lifecycle.cleanup.teardownExitCode = teardownExitCode; status.finishedAt = finishedAt.toISOString(); status.durationMs = Math.max(0, finishedAt.getTime() - startedAt.getTime()); writeFileSync(statusFile, `${JSON.stringify(status, null, 2)}\n`); + +for (const [name, value] of [ + ["result", status.result], + ["exit-code", effectiveExitCode], + ["shard-directory", dirname(statusFile)], + ["status-file", statusFile], +]) { + appendFileSync(outputFile, `${name}=${value}\n`); +} + +if (status.primaryFailure) { + console.error(`${status.primaryFailure.kind}: ${status.primaryFailure.message}`); +} +for (const failure of status.secondaryFailures) { + console.error(`additional ${failure.kind}: ${failure.message}`); +} + +if (status.result !== "passed") { + process.exit(effectiveExitCode || 1); +} diff --git a/.github/actions/run-e2e/playwright-selection.mjs b/.github/actions/run-e2e/playwright-selection.mjs new file mode 100644 index 0000000..ab1d4fa --- /dev/null +++ b/.github/actions/run-e2e/playwright-selection.mjs @@ -0,0 +1,57 @@ +import { readFileSync, writeFileSync } from "node:fs"; + +const parseLines = (value) => + value + .split(/\r?\n/) + .map((entry) => entry.trim()) + .filter(Boolean); + +const unique = (values) => [...new Set(values)]; + +const escapeRegex = (value) => + value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + +export const createSelection = (rawProjects, rawLabels, labelMatch) => { + if (!new Set(["all", "any"]).has(labelMatch)) { + throw new Error(`label-match must be all or any; received ${labelMatch}`); + } + + const projects = unique(parseLines(rawProjects)); + const labels = unique(parseLines(rawLabels)); + for (const label of labels) { + if (!label.startsWith("@") || label.length === 1) { + throw new Error(`Every label must begin with @ and contain a name; received ${label}`); + } + } + + const escapedLabels = labels.map(escapeRegex); + const grep = + escapedLabels.length === 0 + ? null + : labelMatch === "all" + ? escapedLabels.map((label) => `(?=.*${label})`).join("") + : `(?:${escapedLabels.join("|")})`; + const args = projects.map((project) => `--project=${project}`); + if (grep) args.push(`--grep=${grep}`); + + return { projects, labels, labelMatch, grep, args }; +}; + +const [command, ...args] = process.argv.slice(2); +if (command === "emit-args") { + const [selectionFile] = args; + const selection = JSON.parse(readFileSync(selectionFile, "utf8")); + for (const argument of selection.args ?? []) { + if (typeof argument !== "string" || argument.includes("\0")) { + throw new Error("Selection file contains an invalid Playwright argument"); + } + process.stdout.write(argument); + process.stdout.write("\0"); + } +} else if (command === "write") { + const [rawProjects, rawLabels, labelMatch, selectionFile] = args; + const selection = createSelection(rawProjects, rawLabels, labelMatch); + writeFileSync(selectionFile, `${JSON.stringify(selection, null, 2)}\n`); +} else if (process.argv[1]?.endsWith("playwright-selection.mjs")) { + throw new Error(`Unknown playwright-selection command: ${command ?? ""}`); +} diff --git a/.github/actions/run-e2e/prepare-shard.mjs b/.github/actions/run-e2e/prepare-shard.mjs index 5aeeb3a..502ad5b 100644 --- a/.github/actions/run-e2e/prepare-shard.mjs +++ b/.github/actions/run-e2e/prepare-shard.mjs @@ -1,5 +1,4 @@ import { - appendFileSync, existsSync, mkdirSync, realpathSync, @@ -7,11 +6,17 @@ import { writeFileSync, } from "node:fs"; import { isAbsolute, relative, resolve, sep } from "node:path"; +import { createSelection } from "./playwright-selection.mjs"; const [ workspace, requestedWorkingDirectory, requestedConfig, + requestedComposeFile, + requestedBaseUrl, + requestedProjects, + requestedLabels, + requestedLabelMatch, requestedShardIndex, requestedShardTotal, requestedResultsDirectory, @@ -39,8 +44,24 @@ const assertRelative = (value, label) => { assertRelative(requestedWorkingDirectory, "working-directory"); assertRelative(requestedConfig, "playwright-config"); +assertRelative(requestedComposeFile, "compose-file"); assertRelative(requestedResultsDirectory, "results-directory"); +let baseUrl; +try { + baseUrl = new URL(requestedBaseUrl); +} catch { + fail(`base-url must be a valid URL: ${requestedBaseUrl}`); +} +if (!["http:", "https:"].includes(baseUrl.protocol)) { + fail("base-url must use HTTP or HTTPS"); +} +const selection = createSelection( + requestedProjects, + requestedLabels, + requestedLabelMatch, +); + const shardIndex = Number(requestedShardIndex); const shardTotal = Number(requestedShardTotal); if (!Number.isSafeInteger(shardIndex) || !Number.isSafeInteger(shardTotal)) { @@ -73,6 +94,16 @@ if (!existsSync(configPath)) { fail(`Playwright configuration does not exist: ${requestedConfig}`); } +const composeFile = resolve(workspacePath, requestedComposeFile); +if (!existsSync(composeFile)) { + fail(`Docker Compose file does not exist: ${requestedComposeFile}`); +} +const resolvedComposeFile = realpathSync(composeFile); +const relativeComposeFile = relative(workspacePath, resolvedComposeFile); +if (relativeComposeFile.startsWith("..") || isAbsolute(relativeComposeFile)) { + fail("compose-file must not escape GITHUB_WORKSPACE"); +} + const resultsRoot = resolve( resolvedWorkingDirectory, requestedResultsDirectory, @@ -87,6 +118,8 @@ rmSync(shardDirectory, { recursive: true, force: true }); mkdirSync(resolve(shardDirectory, "blob-report"), { recursive: true }); mkdirSync(resolve(shardDirectory, "ctrf"), { recursive: true }); mkdirSync(resolve(shardDirectory, "test-results"), { recursive: true }); +const selectionFile = resolve(shardDirectory, "playwright-selection.json"); +writeFileSync(selectionFile, `${JSON.stringify(selection, null, 2)}\n`); const statusFile = resolve(shardDirectory, "shard-status.json"); const status = { @@ -97,6 +130,25 @@ const status = { startedAt: new Date().toISOString(), finishedAt: null, durationMs: null, + primaryFailure: null, + secondaryFailures: [], + selection: { + projects: selection.projects, + labels: selection.labels, + labelMatch: selection.labelMatch, + grep: selection.grep, + }, + lifecycle: { + sut: { + composeFile: requestedComposeFile, + baseUrl: baseUrl.toString(), + projectName: null, + startupExitCode: null, + }, + playwright: { exitCode: null, started: false }, + evidence: { logCaptureExitCode: null }, + cleanup: { teardownExitCode: null }, + }, github: { repository: process.env.GITHUB_REPOSITORY ?? null, sha: process.env.GITHUB_SHA ?? null, @@ -107,10 +159,26 @@ const status = { }; writeFileSync(statusFile, `${JSON.stringify(status, null, 2)}\n`); +const composeProject = [ + "e2e", + process.env.GITHUB_RUN_ID ?? "local", + process.env.GITHUB_RUN_ATTEMPT ?? "1", + shardIndex, +] + .join("-") + .toLowerCase() + .replaceAll(/[^a-z0-9_-]/g, "-"); +status.lifecycle.sut.projectName = composeProject; +writeFileSync(statusFile, `${JSON.stringify(status, null, 2)}\n`); + const quoteForShell = (value) => `'${value.replaceAll("'", `'"'"'`)}'`; const variables = { E2E_ABSOLUTE_WORKING_DIRECTORY: resolvedWorkingDirectory, E2E_ABSOLUTE_CONFIG: configPath, + E2E_ABSOLUTE_COMPOSE_FILE: resolvedComposeFile, + E2E_BASE_URL: baseUrl.toString(), + E2E_COMPOSE_PROJECT: composeProject, + E2E_SELECTION_FILE: selectionFile, E2E_SHARD_DIRECTORY: shardDirectory, E2E_STATUS_FILE: statusFile, }; diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 685453f..d638736 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -6,6 +6,25 @@ on: branches: - main workflow_dispatch: + inputs: + projects: + description: Playwright project names, one per line; empty runs all projects. + required: false + default: "" + type: string + labels: + description: Playwright tags beginning with @, one per line; empty runs all tags. + required: false + default: "" + type: string + label-match: + description: Require all labels or any label. + required: false + default: all + type: choice + options: + - all + - any permissions: contents: read @@ -20,4 +39,9 @@ jobs: uses: $/.github/workflows/reusable-e2e.yml with: working-directory: e2e + compose-file: compose.e2e.yml + base-url: http://127.0.0.1:4173 + projects: ${{ github.event_name == 'workflow_dispatch' && inputs.projects || '' }} + labels: ${{ github.event_name == 'workflow_dispatch' && inputs.labels || '' }} + label-match: ${{ github.event_name == 'workflow_dispatch' && inputs.label-match || 'all' }} shard-count: 2 diff --git a/.github/workflows/reusable-e2e.yml b/.github/workflows/reusable-e2e.yml index 1f0a9e4..89a355c 100644 --- a/.github/workflows/reusable-e2e.yml +++ b/.github/workflows/reusable-e2e.yml @@ -23,6 +23,31 @@ on: required: false default: playwright.config.ts type: string + compose-file: + description: Docker Compose file for the SUT, relative to the repository root. + required: false + default: compose.e2e.yml + type: string + base-url: + description: Base URL exposed by the Dockerized SUT. + required: false + default: http://127.0.0.1:4173 + type: string + projects: + description: Newline-delimited Playwright project names; empty selects all projects. + required: false + default: "" + type: string + labels: + description: Newline-delimited Playwright tags beginning with @; empty selects all tags. + required: false + default: "" + type: string + label-match: + description: Label matching mode, either all or any. + required: false + default: all + type: string shard-count: description: Number of Playwright shards to run in parallel (1-32). required: false @@ -107,12 +132,17 @@ jobs: with: working-directory: ${{ inputs.working-directory }} playwright-config: ${{ inputs.playwright-config }} + compose-file: ${{ inputs.compose-file }} + base-url: ${{ inputs.base-url }} + projects: ${{ inputs.projects }} + labels: ${{ inputs.labels }} + label-match: ${{ inputs.label-match }} shard-index: ${{ matrix.shard }} shard-total: ${{ inputs.shard-count }} results-directory: test-results - name: Upload shard evidence - if: always() && steps.run.outcome != 'skipped' + if: always() && steps.run.outputs.shard-directory != '' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: e2e-shard-${{ matrix.shard }} diff --git a/README.md b/README.md index 77b6557..b105bb1 100644 --- a/README.md +++ b/README.md @@ -1,50 +1,52 @@ # Playwright E2E CI Framework -A small, reusable GitHub Actions pipeline for running Playwright tests with reliable evidence and one fail-closed gate. +A reusable GitHub Actions pipeline for Playwright tests against a Dockerized system under test (SUT), with durable evidence and one fail-closed gate. -The repository includes a real example: Playwright starts a Node.js echo server, waits for its health endpoint, and tests it through API and Chromium clients. +This repository proves the contract with a root Docker Compose service. The service exposes a health endpoint and echoes requests; Playwright tests it through API and Chromium clients. ```text pull request / main / manual | - v - plan two shards - / \ - v v - shard 1 shard 2 - \ / - v v - merge evidence + +------+------+ + | | + shard 1 shard 2 + compose up compose up + Playwright Playwright + logs + down logs + down + | | + +------+------+ + | + merge evidence | - v E2E Gate ``` ## Run the example locally -Prerequisites: Node.js 22 and Corepack. +Prerequisites: Docker, Node.js 22, and Corepack. + +Install the E2E dependencies and Chromium once: ```bash corepack enable corepack prepare pnpm@10.26.2 --activate pnpm --dir e2e install --frozen-lockfile pnpm --dir e2e install:browsers -pnpm --dir e2e test ``` -The Playwright configuration owns the server lifecycle, so a separate server process is not required. - -Use the CI reporter set locally when you need to inspect the exact CI artifacts: +Start the SUT, run Playwright, and then remove the Compose resources: ```bash -pnpm --dir e2e test:ci +docker compose -f compose.e2e.yml up --detach --build --wait +BASE_URL=http://127.0.0.1:4173 pnpm --dir e2e test +docker compose -f compose.e2e.yml down --volumes --remove-orphans ``` -Run `pnpm --dir e2e test:static` to type-check the configuration and suite without starting the server. +Always run the final command, including after a failed test. Use `pnpm --dir e2e test:ci` for the CI reporters and `pnpm --dir e2e test:static` for a type check. ## Use the reusable workflow -Consumer repositories keep their Playwright package in a root-level `e2e/` directory. Call the workflow from a job and pin it to the full commit SHA of a released version: +Keep the Playwright package in `e2e/` and the Compose file at the repository root. Pin the reusable workflow to a full release commit SHA: ```yaml name: E2E @@ -60,29 +62,93 @@ jobs: uses: codotech/playwright-e2e-ci-framework/.github/workflows/reusable-e2e.yml@<40-character-release-commit-sha> with: working-directory: e2e + compose-file: compose.e2e.yml + base-url: http://127.0.0.1:4173 shard-count: 2 ``` -Replace `<40-character-release-commit-sha>` with the immutable SHA published for the version you adopt. The workflow needs no inherited secrets and does not write pull-request comments. +Replace the marker with the immutable SHA you adopt. The workflow needs no inherited secrets and does not write pull-request comments. + +| Input | Default | Purpose | +| --- | --- | --- | +| `working-directory` | `e2e` | Playwright package location | +| `playwright-config` | `playwright.config.ts` | Config path inside the package | +| `compose-file` | `compose.e2e.yml` | Compose file relative to the repository root | +| `base-url` | `http://127.0.0.1:4173` | URL Playwright uses after the SUT is healthy | +| `projects` | empty | Project names, one per line; empty selects all | +| `labels` | empty | Tags beginning with `@`, one per line; empty selects all | +| `label-match` | `all` | Require all listed labels or any listed label | +| `node-version` | `22` | Node.js runtime | +| `pnpm-version` | `10.26.2` | pnpm runtime | +| `shard-count` | `2` | Parallel shards, from 1 through 32 | +| `artifact-retention-days` | `10` | Evidence retention period | + +The outputs are `verdict`, `artifact-name`, `total`, `passed`, `failed`, and `skipped`. + +## Select projects and labels + +A [Playwright project](https://playwright.dev/docs/test-projects) is a configured execution variant, such as a browser, device, authentication state, or test group. Each `projects` line becomes an exact `--project` selection. + +A framework `label` means a [Playwright tag](https://playwright.dev/docs/test-annotations#tag-tests), not a project. Every label must start with `@`; the workflow converts the labels into one `--grep` expression. + +For example, a manual run can use multiline values: + +```text +projects: +api +chromium + +labels: +@smoke +@browser + +label-match: all +``` + +The repository's pull-request and `main` push runs leave both filters empty. Only a manual run supplies the form values above. + +`all` requires a test to carry both `@smoke` and `@browser`. `any` accepts a test carrying either tag. Blank project or label lines are ignored, and leaving a field empty disables that filter. + +Projects and labels are independent. The selected tests are the intersection of both filters, and Playwright shards that result afterward: + +```text +configured tests + | + selected projects + | + matching tags + | + shard 1 + shard 2 +``` + +The same values can be supplied by another workflow with YAML block scalars: -The framework targets GitHub.com. It uses GitHub's `$/` self-repository references so the reusable workflow and its composite actions always come from the same commit; that syntax is not available on GitHub Enterprise Server. +```yaml +with: + projects: | + api + chromium + labels: | + @smoke + @browser + label-match: any +``` -The supported inputs are: +## SUT lifecycle in CI -| Input | Default | Purpose | -| ------------------------- | ---------------------- | ---------------------------------- | -| `working-directory` | `e2e` | Playwright package location | -| `node-version` | `22` | Node.js runtime | -| `pnpm-version` | `10.26.2` | pnpm runtime | -| `playwright-config` | `playwright.config.ts` | Config path inside the package | -| `shard-count` | `2` | Parallel shards, from 1 through 32 | -| `artifact-retention-days` | `10` | Evidence retention period | +Each shard runs on its own runner and performs the full lifecycle: -The workflow exposes `verdict`, `artifact-name`, `total`, `passed`, `failed`, and `skipped` outputs. +1. Validate the Compose file and base URL. +2. Build and start the SUT with `docker compose up --detach --build --wait`. +3. Run Playwright with `BASE_URL` set to the ready service. +4. Capture startup, service, and teardown logs. +5. Run `docker compose down --volumes --remove-orphans`, even after failure. -## Read the evidence +A startup or cleanup failure is an infrastructure error. A Playwright failure remains the primary failure when log collection or cleanup also fails. -Every shard uploads its raw status, blob report, CTRF result, and Playwright test results. The merge job verifies that all expected shards are present and publishes one `e2e-evidence` artifact: +## Evidence and gate + +Every shard uploads its status, Compose logs, blob report, CTRF result, and Playwright test results. The merged `e2e-evidence` artifact contains: ```text e2e-evidence/ @@ -92,18 +158,6 @@ e2e-evidence/ +-- verdict.json ``` -The HTML report is for investigation. `verdict.json` is the machine-readable result. The Actions job summary shows the merged totals without requiring repository write permissions. - -## Fail-closed behavior - -`E2E Gate` fails when any of these conditions occurs: - -- A Playwright shard fails or is cancelled. -- A shard status or report is missing or duplicated. -- Reports cannot be merged. -- The evidence artifact cannot be published or downloaded. -- The authoritative verdict is absent or does not say `passed`. - -Evidence publication runs after test failure, but publishing a report never turns a failed test run green. +`E2E Gate` fails for a failed or cancelled shard, unhealthy SUT lifecycle, missing evidence, report merge failure, artifact failure, or a non-passing verdict. Evidence publication cannot turn a failed run green. -To verify these guarantees on a branch, intentionally break one echo assertion, interrupt the example server startup, or temporarily remove one shard upload. Each experiment must end with a red `E2E Gate` and retained evidence; revert the controlled change after verification. +The framework targets GitHub.com and uses `$/` references so the reusable workflow and its composite actions come from the same commit. That syntax is not available on GitHub Enterprise Server. diff --git a/compose.e2e.yml b/compose.e2e.yml new file mode 100644 index 0000000..cddb015 --- /dev/null +++ b/compose.e2e.yml @@ -0,0 +1,15 @@ +services: + sut: + build: + context: ./sut + dockerfile: Dockerfile + init: true + ports: + - "4173:4173" + read_only: true + tmpfs: + - /tmp + cap_drop: + - ALL + security_opt: + - no-new-privileges:true diff --git a/e2e/playwright.config.ts b/e2e/playwright.config.ts index 404e493..eed8675 100644 --- a/e2e/playwright.config.ts +++ b/e2e/playwright.config.ts @@ -1,10 +1,7 @@ import { defineConfig, devices } from "@playwright/test"; import { basename, dirname } from "node:path"; -const host = process.env.SUT_HOST ?? "127.0.0.1"; -const port = process.env.SUT_PORT ?? "4173"; -const localBaseUrl = `http://${host}:${port}`; -const baseURL = process.env.BASE_URL ?? localBaseUrl; +const baseURL = process.env.BASE_URL ?? "http://127.0.0.1:4173"; const isCi = Boolean(process.env.CI); const testResultsDir = process.env.E2E_RESULTS_DIR ?? @@ -53,16 +50,4 @@ export default defineConfig({ use: { ...devices["Desktop Chrome"] }, }, ], - webServer: process.env.BASE_URL - ? undefined - : { - command: "node ./sut/server.mjs", - env: { - SUT_HOST: host, - SUT_PORT: port, - }, - url: `${localBaseUrl}/health`, - reuseExistingServer: !isCi, - timeout: 30_000, - }, }); diff --git a/e2e/tests/echo.api.spec.ts b/e2e/tests/echo.api.spec.ts index eafd91f..cc00918 100644 --- a/e2e/tests/echo.api.spec.ts +++ b/e2e/tests/echo.api.spec.ts @@ -8,46 +8,56 @@ interface EchoResponse { body: unknown; } -test("reports that the SUT is healthy", async ({ request }): Promise => { - const response = await request.get("/health"); +test( + "reports that the SUT is healthy", + { tag: "@smoke" }, + async ({ request }): Promise => { + const response = await request.get("/health"); - expect(response.ok()).toBe(true); - await expect(response.json()).resolves.toEqual({ status: "ok" }); -}); + expect(response.ok()).toBe(true); + await expect(response.json()).resolves.toEqual({ status: "ok" }); + }, +); -test("echoes a GET request", async ({ request }): Promise => { - const response = await request.get("/echo", { - params: { message: "hello from Playwright" }, - headers: { "x-unused-demonstration-header": "test" }, - }); - const payload = (await response.json()) as EchoResponse; +test( + "echoes a GET request", + { tag: "@smoke" }, + async ({ request }): Promise => { + const response = await request.get("/echo", { + params: { message: "hello from Playwright" }, + headers: { "x-unused-demonstration-header": "test" }, + }); + const payload = (await response.json()) as EchoResponse; - expect(response.ok()).toBe(true); - expect(payload).toMatchObject({ - method: "GET", - path: "/echo", - query: { message: "hello from Playwright" }, - body: null, - }); - expect(payload.headers.accept).toBe("*/*"); -}); + expect(response.ok()).toBe(true); + expect(payload).toMatchObject({ + method: "GET", + path: "/echo", + query: { message: "hello from Playwright" }, + body: null, + }); + expect(payload.headers.accept).toBe("*/*"); + }, +); -test("echoes a POST request with a JSON body", async ({ - request, -}): Promise => { - const requestBody = { - message: "hello from a real process boundary", - sequence: 42, - }; - const response = await request.post("/echo", { data: requestBody }); - const payload = (await response.json()) as EchoResponse; +test( + "echoes a POST request with a JSON body", + { tag: "@regression" }, + async ({ request }): Promise => { + const requestBody = { + message: "hello from a real process boundary", + sequence: 42, + }; + const response = await request.post("/echo", { data: requestBody }); + const payload = (await response.json()) as EchoResponse; - expect(response.ok()).toBe(true); - expect(payload).toMatchObject({ - method: "POST", - path: "/echo", - query: {}, - body: requestBody, - }); - expect(payload.headers["content-type"]).toContain("application/json"); -}); + expect(response.ok()).toBe(true); + expect(payload).toMatchObject({ + method: "POST", + path: "/echo", + query: {}, + body: requestBody, + }); + expect(payload.headers["content-type"]).toContain("application/json"); + }, +); diff --git a/e2e/tests/echo.browser.spec.ts b/e2e/tests/echo.browser.spec.ts index c693362..47ef816 100644 --- a/e2e/tests/echo.browser.spec.ts +++ b/e2e/tests/echo.browser.spec.ts @@ -1,11 +1,13 @@ import { expect, test } from "@playwright/test"; -test("navigates to the echo endpoint in Chromium", async ({ - page, -}): Promise => { - const response = await page.goto("/echo?source=chromium"); +test( + "navigates to the echo endpoint in Chromium", + { tag: ["@smoke", "@browser"] }, + async ({ page }): Promise => { + const response = await page.goto("/echo?source=chromium"); - expect(response?.ok()).toBe(true); - await expect(page.locator("body")).toContainText('"method":"GET"'); - await expect(page.locator("body")).toContainText('"source":"chromium"'); -}); + expect(response?.ok()).toBe(true); + await expect(page.locator("body")).toContainText('"method":"GET"'); + await expect(page.locator("body")).toContainText('"source":"chromium"'); + }, +); diff --git a/sut/.dockerignore b/sut/.dockerignore new file mode 100644 index 0000000..281787e --- /dev/null +++ b/sut/.dockerignore @@ -0,0 +1,3 @@ +* +!server.mjs +!Dockerfile diff --git a/sut/Dockerfile b/sut/Dockerfile new file mode 100644 index 0000000..2fe8480 --- /dev/null +++ b/sut/Dockerfile @@ -0,0 +1,18 @@ +FROM node:22.22.1-alpine3.23@sha256:8094c002d08262dba12645a3b4a15cd6cd627d30bc782f53229a2ec13ee22a00 + +ENV NODE_ENV=production \ + SUT_HOST=0.0.0.0 \ + SUT_PORT=4173 + +WORKDIR /app + +COPY --chown=node:node server.mjs ./server.mjs + +USER node + +EXPOSE 4173 + +HEALTHCHECK --interval=2s --timeout=2s --start-period=2s --retries=10 \ + CMD ["node", "-e", "fetch('http://127.0.0.1:4173/health').then(response => { if (!response.ok) process.exit(1) }).catch(() => process.exit(1))"] + +CMD ["node", "server.mjs"] diff --git a/e2e/sut/server.mjs b/sut/server.mjs similarity index 98% rename from e2e/sut/server.mjs rename to sut/server.mjs index 3849329..5dbed04 100644 --- a/e2e/sut/server.mjs +++ b/sut/server.mjs @@ -1,6 +1,6 @@ import { createServer } from "node:http"; -const host = process.env.SUT_HOST ?? "127.0.0.1"; +const host = process.env.SUT_HOST ?? "0.0.0.0"; const port = Number.parseInt(process.env.SUT_PORT ?? "4173", 10); const maximumBodyBytes = 1024 * 1024; From a07b509a23c4d0128afa2146d9530d4cceffa565 Mon Sep 17 00:00:00 2001 From: omermorad Date: Sun, 27 Sep 2026 22:38:18 +0300 Subject: [PATCH 02/11] feat: publish sticky E2E PR report --- .github/workflows/e2e.yml | 1 + .github/workflows/reusable-e2e.yml | 111 +++++++++++++++++++++++++++++ README.md | 4 ++ 3 files changed, 116 insertions(+) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index d638736..f9baef3 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -28,6 +28,7 @@ on: permissions: contents: read + pull-requests: write concurrency: group: e2e-${{ github.event.pull_request.number || github.ref }} diff --git a/.github/workflows/reusable-e2e.yml b/.github/workflows/reusable-e2e.yml index 89a355c..e768305 100644 --- a/.github/workflows/reusable-e2e.yml +++ b/.github/workflows/reusable-e2e.yml @@ -58,6 +58,11 @@ on: required: false default: 10 type: number + comment-on-pr: + description: Create or update one E2E report comment on same-repository pull requests. + required: false + default: true + type: boolean outputs: verdict: description: Authoritative E2E verdict. @@ -80,6 +85,7 @@ on: permissions: contents: read + pull-requests: write jobs: plan: @@ -244,3 +250,108 @@ jobs: shard-job-result: ${{ needs.shards.result }} merge-job-result: ${{ needs.merge.result }} publication-result: ${{ steps.download.outcome }} + + pr_comment: + name: E2E PR report + needs: + - merge + - e2e_gate + if: >- + ${{ always() && inputs.comment-on-pr && github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name == github.repository }} + runs-on: ubuntu-latest + timeout-minutes: 2 + steps: + - name: Create or update E2E report + continue-on-error: true + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + E2E_GATE_JOB_RESULT: ${{ needs.e2e_gate.result }} + E2E_GATE_VERDICT: ${{ needs.e2e_gate.outputs.verdict }} + E2E_MERGE_VERDICT: ${{ needs.merge.outputs.verdict }} + E2E_PRIMARY_FAILURE: ${{ needs.e2e_gate.outputs.primary-failure }} + E2E_TOTAL: ${{ needs.merge.outputs.total }} + E2E_PASSED: ${{ needs.merge.outputs.passed }} + E2E_FAILED: ${{ needs.merge.outputs.failed }} + E2E_SKIPPED: ${{ needs.merge.outputs.skipped }} + E2E_ARTIFACT_NAME: ${{ needs.merge.outputs.artifact-name }} + E2E_PROJECTS: ${{ inputs.projects }} + E2E_LABELS: ${{ inputs.labels }} + E2E_LABEL_MATCH: ${{ inputs.label-match }} + with: + script: | + const marker = ''; + const clean = (value, fallback) => { + const normalized = String(value ?? '').trim(); + return normalized || fallback; + }; + const cell = (value) => String(value).replaceAll('|', '\\|').replaceAll('\n', '
'); + const list = (value, fallback) => { + const entries = String(value ?? '') + .split(/\r?\n/) + .map((entry) => entry.trim()) + .filter(Boolean); + return entries.length > 0 ? entries.map((entry) => `\`${entry}\``).join(', ') : fallback; + }; + + const gateResult = clean(process.env.E2E_GATE_JOB_RESULT, 'failure'); + const verdict = clean( + process.env.E2E_GATE_VERDICT, + clean(process.env.E2E_MERGE_VERDICT, gateResult === 'success' ? 'passed' : 'infrastructure-error'), + ); + const passed = verdict === 'passed'; + const status = passed ? '✅ Passed' : verdict === 'failed' ? '❌ Tests failed' : '⚠️ Infrastructure error'; + const total = clean(process.env.E2E_TOTAL, 'unknown'); + const passedCount = clean(process.env.E2E_PASSED, 'unknown'); + const failedCount = clean(process.env.E2E_FAILED, 'unknown'); + const skippedCount = clean(process.env.E2E_SKIPPED, 'unknown'); + const projects = list(process.env.E2E_PROJECTS, 'All configured projects'); + const labels = list(process.env.E2E_LABELS, 'All labels'); + const labelMatch = clean(process.env.E2E_LABEL_MATCH, 'all'); + const artifactName = clean(process.env.E2E_ARTIFACT_NAME, 'e2e-evidence'); + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + + const lines = [ + marker, + '## E2E confidence report', + '', + '| Signal | Result |', + '| --- | --- |', + `| Gate | ${status} |`, + `| Tests | ${cell(`${passedCount} passed · ${failedCount} failed · ${skippedCount} skipped · ${total} total`)} |`, + `| Projects | ${cell(projects)} |`, + `| Labels | ${cell(`${labels} · match ${labelMatch}`)} |`, + '| SUT | Docker Compose, isolated per shard |', + `| Evidence | [Open workflow run](${runUrl}) · \`${cell(artifactName)}\` |`, + ]; + + const primaryFailure = clean(process.env.E2E_PRIMARY_FAILURE, ''); + if (!passed && primaryFailure) { + lines.push('', `> ${primaryFailure.replaceAll('\n', '\n> ')}`); + } + lines.push('', `_Updated from run ${context.runId}._`); + const body = lines.join('\n'); + + const comments = await github.paginate(github.rest.issues.listComments, { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + per_page: 100, + }); + const existing = comments.find((comment) => comment.body?.includes(marker)); + + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + body, + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body, + }); + } diff --git a/README.md b/README.md index b105bb1..4f81e63 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,7 @@ on: permissions: contents: read + pull-requests: write jobs: e2e: @@ -82,6 +83,7 @@ Replace the marker with the immutable SHA you adopt. The workflow needs no inher | `pnpm-version` | `10.26.2` | pnpm runtime | | `shard-count` | `2` | Parallel shards, from 1 through 32 | | `artifact-retention-days` | `10` | Evidence retention period | +| `comment-on-pr` | `true` | Update one E2E report comment on same-repository pull requests | The outputs are `verdict`, `artifact-name`, `total`, `passed`, `failed`, and `skipped`. @@ -160,4 +162,6 @@ e2e-evidence/ `E2E Gate` fails for a failed or cancelled shard, unhealthy SUT lifecycle, missing evidence, report merge failure, artifact failure, or a non-passing verdict. Evidence publication cannot turn a failed run green. +For pull requests from the same repository, the workflow creates or updates one sticky `E2E confidence report` comment with the verdict, aggregate totals, active project and label filters, failure reason, and workflow evidence link. Grant `pull-requests: write` in the caller workflow, or set `comment-on-pr: false` when comments are not wanted. Comment publication is non-blocking and does not change the gate verdict. + The framework targets GitHub.com and uses `$/` references so the reusable workflow and its composite actions come from the same commit. That syntax is not available on GitHub Enterprise Server. From 6ef4b32f1eb4a73aadbcbe100bacbe40cc130ee8 Mon Sep 17 00:00:00 2001 From: omermorad Date: Sun, 27 Sep 2026 23:28:14 +0300 Subject: [PATCH 03/11] feat: publish portable E2E runner and reports --- .github/actions/build-e2e-runner/action.yml | 73 ++++++ .../build-e2e-runner/prepare-build.mjs | 94 ++++++++ .github/actions/merge-e2e-evidence/action.yml | 96 +++----- .../merge-e2e-evidence/assemble-evidence.mjs | 21 +- .../merge-e2e-evidence/evaluate-verdict.mjs | 103 --------- .../merge-e2e-evidence/stage-evidence.mjs | 10 +- .github/actions/package-e2e-report/Dockerfile | 20 ++ .github/actions/package-e2e-report/action.yml | 91 ++++++++ .../package-e2e-report/prepare-context.mjs | 163 ++++++++++++++ .github/actions/package-e2e-report/server.mjs | 196 +++++++++++++++++ .../actions/publish-e2e-evidence/action.yml | 116 ++++++++++ .../evaluate-publication.mjs | 63 ++++++ .../prepare-publication.mjs | 137 ++++++++++++ .github/actions/run-e2e/action.yml | 111 ++++++++-- .github/actions/run-e2e/complete-shard.mjs | 95 ++++++-- .github/actions/run-e2e/prepare-shard.mjs | 37 ++++ .github/workflows/reusable-e2e.yml | 207 ++++++++++++++---- README.md | 77 ++++++- e2e/.dockerignore | 81 +++++++ e2e/Dockerfile | 21 ++ e2e/runner-entrypoint.mjs | 57 +++++ 21 files changed, 1617 insertions(+), 252 deletions(-) create mode 100644 .github/actions/build-e2e-runner/action.yml create mode 100644 .github/actions/build-e2e-runner/prepare-build.mjs delete mode 100644 .github/actions/merge-e2e-evidence/evaluate-verdict.mjs create mode 100644 .github/actions/package-e2e-report/Dockerfile create mode 100644 .github/actions/package-e2e-report/action.yml create mode 100644 .github/actions/package-e2e-report/prepare-context.mjs create mode 100644 .github/actions/package-e2e-report/server.mjs create mode 100644 .github/actions/publish-e2e-evidence/action.yml create mode 100644 .github/actions/publish-e2e-evidence/evaluate-publication.mjs create mode 100644 .github/actions/publish-e2e-evidence/prepare-publication.mjs create mode 100644 e2e/.dockerignore create mode 100644 e2e/Dockerfile create mode 100644 e2e/runner-entrypoint.mjs diff --git a/.github/actions/build-e2e-runner/action.yml b/.github/actions/build-e2e-runner/action.yml new file mode 100644 index 0000000..5e5de4f --- /dev/null +++ b/.github/actions/build-e2e-runner/action.yml @@ -0,0 +1,73 @@ +name: Build E2E Runner +description: Build the Playwright runner once and save a portable zstd-compressed Docker archive. + +inputs: + working-directory: + description: Repository-relative Docker build context containing the E2E runner. + required: false + default: e2e + dockerfile: + description: Dockerfile path relative to working-directory. + required: false + default: Dockerfile + image-name: + description: Complete local Docker image reference, including an immutable tag. + required: true + archive-path: + description: Repository-relative output path for the zstd-compressed Docker image archive. + required: false + default: .e2e-runner/playwright-e2e-runner.tar.zst + +outputs: + image-name: + description: Local Docker image reference saved in the archive. + value: ${{ steps.build.outputs.image-name }} + image-id: + description: Content-addressed Docker image ID. + value: ${{ steps.build.outputs.image-id }} + archive-path: + description: Absolute path to the portable .tar.zst Docker image archive. + value: ${{ steps.build.outputs.archive-path }} + +runs: + using: composite + steps: + - id: build + name: Build and save runner image + shell: bash + env: + E2E_WORKING_DIRECTORY: ${{ inputs.working-directory }} + E2E_DOCKERFILE: ${{ inputs.dockerfile }} + E2E_RUNNER_IMAGE: ${{ inputs.image-name }} + E2E_ARCHIVE_PATH: ${{ inputs.archive-path }} + run: | + set -euo pipefail + + eval "$(node "$GITHUB_ACTION_PATH/prepare-build.mjs" \ + "$GITHUB_WORKSPACE" \ + "$E2E_WORKING_DIRECTORY" \ + "$E2E_DOCKERFILE" \ + "$E2E_RUNNER_IMAGE" \ + "$E2E_ARCHIVE_PATH")" + + docker build \ + --file "$E2E_ABSOLUTE_DOCKERFILE" \ + --tag "$E2E_RUNNER_IMAGE" \ + "$E2E_BUILD_CONTEXT" + + image_id="$(docker image inspect --format '{{.Id}}' "$E2E_RUNNER_IMAGE")" + if [[ ! "$image_id" =~ ^sha256:[a-f0-9]{64}$ ]]; then + echo "Unexpected Docker image ID: $image_id" >&2 + exit 1 + fi + + command -v zstd > /dev/null + docker save "$E2E_RUNNER_IMAGE" | \ + zstd -T0 -3 --force -o "$E2E_ABSOLUTE_ARCHIVE_PATH" + test -s "$E2E_ABSOLUTE_ARCHIVE_PATH" + + { + echo "image-name=$E2E_RUNNER_IMAGE" + echo "image-id=$image_id" + echo "archive-path=$E2E_ABSOLUTE_ARCHIVE_PATH" + } >> "$GITHUB_OUTPUT" diff --git a/.github/actions/build-e2e-runner/prepare-build.mjs b/.github/actions/build-e2e-runner/prepare-build.mjs new file mode 100644 index 0000000..a2a5439 --- /dev/null +++ b/.github/actions/build-e2e-runner/prepare-build.mjs @@ -0,0 +1,94 @@ +import { + existsSync, + lstatSync, + mkdirSync, + realpathSync, + rmSync, +} from "node:fs"; +import { dirname, isAbsolute, relative, resolve, sep } from "node:path"; + +const [ + workspace, + requestedWorkingDirectory, + requestedDockerfile, + imageName, + requestedArchivePath, +] = process.argv.slice(2); + +const assertRelative = (value, label) => { + if (!value || isAbsolute(value)) { + throw new Error(`${label} must be a non-empty repository-relative path`); + } + const relativePath = relative("/contract-root", resolve("/contract-root", value)); + if (relativePath === "" || relativePath === ".." || relativePath.startsWith(`..${sep}`)) { + throw new Error(`${label} must resolve below its parent directory`); + } +}; + +assertRelative(requestedWorkingDirectory, "working-directory"); +assertRelative(requestedDockerfile, "dockerfile"); +assertRelative(requestedArchivePath, "archive-path"); +if (!imageName || /[\s\0]/.test(imageName)) { + throw new Error("image-name must be a non-empty Docker reference without whitespace"); +} + +const workspacePath = realpathSync(workspace); +const buildContext = resolve(workspacePath, requestedWorkingDirectory); +if (!existsSync(buildContext) || !lstatSync(buildContext).isDirectory()) { + throw new Error(`E2E runner build context does not exist: ${requestedWorkingDirectory}`); +} +const resolvedBuildContext = realpathSync(buildContext); +const contextRelativePath = relative(workspacePath, resolvedBuildContext); +if (contextRelativePath.startsWith("..") || isAbsolute(contextRelativePath)) { + throw new Error("working-directory must not escape GITHUB_WORKSPACE through a symlink"); +} + +const dockerfile = resolve(resolvedBuildContext, requestedDockerfile); +if (!existsSync(dockerfile) || !lstatSync(dockerfile).isFile()) { + throw new Error(`E2E runner Dockerfile does not exist: ${requestedDockerfile}`); +} +const resolvedDockerfile = realpathSync(dockerfile); +const dockerfileRelativePath = relative(resolvedBuildContext, resolvedDockerfile); +if (dockerfileRelativePath.startsWith("..") || isAbsolute(dockerfileRelativePath)) { + throw new Error("dockerfile must not escape working-directory through a symlink"); +} + +const archivePath = resolve(workspacePath, requestedArchivePath); +const archiveRelativePath = relative(workspacePath, archivePath); +if (archiveRelativePath.startsWith("..") || isAbsolute(archiveRelativePath)) { + throw new Error("archive-path must remain inside GITHUB_WORKSPACE"); +} +let existingArchiveAncestor = archivePath; +while (!existsSync(existingArchiveAncestor)) { + const parent = dirname(existingArchiveAncestor); + if (parent === existingArchiveAncestor) { + throw new Error("archive-path has no existing parent directory"); + } + existingArchiveAncestor = parent; +} +const resolvedArchiveAncestor = realpathSync(existingArchiveAncestor); +const archiveAncestorRelativePath = relative( + workspacePath, + resolvedArchiveAncestor, +); +if ( + archiveAncestorRelativePath.startsWith("..") || + isAbsolute(archiveAncestorRelativePath) +) { + throw new Error("archive-path must not traverse a symlink outside GITHUB_WORKSPACE"); +} +if (existsSync(archivePath) && lstatSync(archivePath).isSymbolicLink()) { + throw new Error("archive-path must not be a symbolic link"); +} +mkdirSync(dirname(archivePath), { recursive: true }); +rmSync(archivePath, { force: true }); + +const quoteForShell = (value) => `'${value.replaceAll("'", `'"'"'`)}'`; +for (const [name, value] of Object.entries({ + E2E_BUILD_CONTEXT: resolvedBuildContext, + E2E_ABSOLUTE_DOCKERFILE: resolvedDockerfile, + E2E_RUNNER_IMAGE: imageName, + E2E_ABSOLUTE_ARCHIVE_PATH: archivePath, +})) { + process.stdout.write(`${name}=${quoteForShell(value)}\n`); +} diff --git a/.github/actions/merge-e2e-evidence/action.yml b/.github/actions/merge-e2e-evidence/action.yml index aaafd61..cfdc95a 100644 --- a/.github/actions/merge-e2e-evidence/action.yml +++ b/.github/actions/merge-e2e-evidence/action.yml @@ -1,11 +1,7 @@ name: Merge E2E Evidence -description: Validate all shard evidence, merge reports, publish evidence, and fail closed. +description: Validate shard evidence and merge reports inside the immutable E2E runner image. inputs: - working-directory: - description: Repository-relative directory containing the installed Playwright package. - required: false - default: e2e download-directory: description: Repository-relative directory containing downloaded shard artifacts. required: false @@ -17,37 +13,29 @@ inputs: expected-shards: description: Number of shard status files and report sets required. required: true - artifact-name: - description: Name of the durable merged evidence artifact. - required: false - default: e2e-evidence - retention-days: - description: Number of days GitHub retains the merged evidence artifact. - required: false - default: "10" + runner-image-id: + description: Content-addressed Docker image ID used to merge Playwright reports. + required: true outputs: verdict: description: "Authoritative verdict: passed, failed, or infrastructure-error." - value: ${{ steps.evaluate.outputs.verdict }} - artifact-name: - description: Merged evidence artifact name. - value: ${{ inputs.artifact-name }} + value: ${{ steps.assemble.outputs.verdict }} total: description: Aggregated test count. - value: ${{ steps.evaluate.outputs.total }} + value: ${{ steps.assemble.outputs.total }} passed: description: Aggregated passed test count. - value: ${{ steps.evaluate.outputs.passed }} + value: ${{ steps.assemble.outputs.passed }} failed: description: Aggregated failed test count. - value: ${{ steps.evaluate.outputs.failed }} + value: ${{ steps.assemble.outputs.failed }} skipped: description: Aggregated skipped test count. - value: ${{ steps.evaluate.outputs.skipped }} + value: ${{ steps.assemble.outputs.skipped }} verdict-file: description: Absolute path to verdict.json. - value: ${{ steps.evaluate.outputs.verdict-file }} + value: ${{ steps.assemble.outputs.verdict-file }} runs: using: composite @@ -56,7 +44,6 @@ runs: id: stage shell: bash env: - E2E_WORKING_DIRECTORY: ${{ inputs.working-directory }} E2E_DOWNLOAD_DIRECTORY: ${{ inputs.download-directory }} E2E_EVIDENCE_DIRECTORY: ${{ inputs.evidence-directory }} E2E_EXPECTED_SHARDS: ${{ inputs.expected-shards }} @@ -64,7 +51,6 @@ runs: set -euo pipefail node "$GITHUB_ACTION_PATH/stage-evidence.mjs" \ "$GITHUB_WORKSPACE" \ - "$E2E_WORKING_DIRECTORY" \ "$E2E_DOWNLOAD_DIRECTORY" \ "$E2E_EVIDENCE_DIRECTORY" \ "$E2E_EXPECTED_SHARDS" \ @@ -72,15 +58,13 @@ runs: - name: Merge Playwright blob reports id: merge-html - if: always() continue-on-error: true shell: bash env: E2E_CAN_MERGE: ${{ steps.stage.outputs.can-merge }} - E2E_WORKING_DIRECTORY: ${{ steps.stage.outputs.working-directory }} E2E_BLOB_DIRECTORY: ${{ steps.stage.outputs.blob-directory }} PLAYWRIGHT_HTML_OUTPUT_DIR: ${{ steps.stage.outputs.html-directory }} - PLAYWRIGHT_HTML_OPEN: never + E2E_RUNNER_IMAGE_ID: ${{ inputs.runner-image-id }} run: | set +e exit_code=0 @@ -88,11 +72,28 @@ runs: if [[ "$E2E_CAN_MERGE" != "true" ]]; then echo "Blob report merge cannot run because shard evidence is incomplete." exit_code=42 + elif [[ ! "$E2E_RUNNER_IMAGE_ID" =~ ^sha256:[a-f0-9]{64}$ ]]; then + echo "Blob report merge cannot run because the runner image ID is invalid." + exit_code=42 else - ( - cd "$E2E_WORKING_DIRECTORY" - pnpm exec playwright merge-reports --reporter html "$E2E_BLOB_DIRECTORY" - ) + docker run --rm \ + --init \ + --read-only \ + --network none \ + --cap-drop ALL \ + --security-opt no-new-privileges:true \ + --pids-limit 256 \ + --tmpfs /tmp:rw,nosuid,nodev,noexec,size=256m \ + --user "$(id -u):$(id -g)" \ + --env CI=true \ + --env HOME=/tmp \ + --env PLAYWRIGHT_HTML_OUTPUT_DIR=/merge-output \ + --env PLAYWRIGHT_HTML_OPEN=never \ + --volume "$E2E_BLOB_DIRECTORY:/merge-input/blob-report" \ + --volume "$PLAYWRIGHT_HTML_OUTPUT_DIR:/merge-output" \ + --entrypoint /workspace/e2e/node_modules/.bin/playwright \ + "$E2E_RUNNER_IMAGE_ID" \ + merge-reports --reporter html /merge-input/blob-report exit_code=$? fi @@ -101,7 +102,7 @@ runs: - name: Assemble deterministic evidence id: assemble - if: always() + if: always() && steps.stage.outcome == 'success' shell: bash env: E2E_EVIDENCE_DIRECTORY: ${{ steps.stage.outputs.evidence-directory }} @@ -112,34 +113,5 @@ runs: node "$GITHUB_ACTION_PATH/assemble-evidence.mjs" \ "$E2E_EVIDENCE_DIRECTORY" \ "$E2E_EXPECTED_SHARDS" \ - "${E2E_HTML_MERGE_EXIT_CODE:-99}" - - - name: Upload merged E2E evidence - id: publish - if: always() - continue-on-error: true - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: ${{ inputs.artifact-name }} - path: ${{ steps.stage.outputs.evidence-directory }}/ - if-no-files-found: error - retention-days: ${{ inputs.retention-days }} - - - name: Publish summary and enforce verdict - id: evaluate - if: always() - shell: bash - env: - E2E_EVIDENCE_DIRECTORY: ${{ steps.stage.outputs.evidence-directory }} - E2E_EXPECTED_SHARDS: ${{ inputs.expected-shards }} - E2E_ARTIFACT_NAME: ${{ inputs.artifact-name }} - E2E_PUBLICATION_OUTCOME: ${{ steps.publish.outcome }} - run: | - set -euo pipefail - node "$GITHUB_ACTION_PATH/evaluate-verdict.mjs" \ - "$E2E_EVIDENCE_DIRECTORY" \ - "$E2E_EXPECTED_SHARDS" \ - "$E2E_ARTIFACT_NAME" \ - "${E2E_PUBLICATION_OUTCOME:-failure}" \ - "$GITHUB_STEP_SUMMARY" \ + "${E2E_HTML_MERGE_EXIT_CODE:-99}" \ "$GITHUB_OUTPUT" diff --git a/.github/actions/merge-e2e-evidence/assemble-evidence.mjs b/.github/actions/merge-e2e-evidence/assemble-evidence.mjs index c55c185..a5931be 100644 --- a/.github/actions/merge-e2e-evidence/assemble-evidence.mjs +++ b/.github/actions/merge-e2e-evidence/assemble-evidence.mjs @@ -7,8 +7,12 @@ import { } from "node:fs"; import { resolve } from "node:path"; -const [evidenceDirectory, requestedExpectedShards, requestedHtmlMergeExitCode] = - process.argv.slice(2); +const [ + evidenceDirectory, + requestedExpectedShards, + requestedHtmlMergeExitCode, + outputFile, +] = process.argv.slice(2); const expectedShards = Number(requestedExpectedShards); const htmlMergeExitCode = Number(requestedHtmlMergeExitCode); const stagingDirectory = resolve(evidenceDirectory, ".merge-input"); @@ -214,4 +218,17 @@ writeFileSync( `${JSON.stringify(verdict, null, 2)}\n`, ); +if (outputFile) { + for (const [name, value] of [ + ["verdict", verdict.result], + ["total", verdict.summary.total], + ["passed", verdict.summary.passed], + ["failed", verdict.summary.failed], + ["skipped", verdict.summary.skipped], + ["verdict-file", resolve(evidenceDirectory, "verdict.json")], + ]) { + writeFileSync(outputFile, `${name}=${value}\n`, { flag: "a" }); + } +} + rmSync(stagingDirectory, { recursive: true, force: true }); diff --git a/.github/actions/merge-e2e-evidence/evaluate-verdict.mjs b/.github/actions/merge-e2e-evidence/evaluate-verdict.mjs deleted file mode 100644 index ddce94e..0000000 --- a/.github/actions/merge-e2e-evidence/evaluate-verdict.mjs +++ /dev/null @@ -1,103 +0,0 @@ -import { - appendFileSync, - existsSync, - readFileSync, - writeFileSync, -} from "node:fs"; -import { resolve } from "node:path"; - -const [ - evidenceDirectory, - requestedExpectedShards, - artifactName, - publicationOutcome, - summaryFile, - outputFile, -] = process.argv.slice(2); -const verdictFile = resolve(evidenceDirectory, "verdict.json"); - -let verdict; -if (existsSync(verdictFile)) { - try { - verdict = JSON.parse(readFileSync(verdictFile, "utf8")); - } catch (error) { - verdict = null; - } -} - -if (!verdict) { - verdict = { - schemaVersion: 1, - result: "infrastructure-error", - primaryFailure: { - kind: "infrastructure", - message: "Evidence assembly did not produce a readable verdict.json", - }, - secondaryFailures: [], - expectedShards: Number(requestedExpectedShards), - observedShards: [], - summary: { - total: 0, - passed: 0, - failed: 0, - skipped: 0, - pending: 0, - other: 0, - }, - shards: [], - reports: {}, - generatedAt: new Date().toISOString(), - }; -} - -let enforcedResult = verdict.result; -let enforcedPrimaryFailure = verdict.primaryFailure ?? null; -if (publicationOutcome !== "success") { - const publicationFailure = { - kind: "infrastructure", - message: `Evidence artifact publication finished with ${publicationOutcome || "unknown"}`, - }; - if (!enforcedPrimaryFailure) enforcedPrimaryFailure = publicationFailure; - if (enforcedResult === "passed") enforcedResult = "infrastructure-error"; -} - -const summary = verdict.summary ?? {}; -appendFileSync( - summaryFile, - [ - "## E2E Gate", - "", - `**Verdict:** ${enforcedResult}`, - "", - "| Result | Count |", - "| --- | ---: |", - `| Total | ${summary.total ?? 0} |`, - `| Passed | ${summary.passed ?? 0} |`, - `| Failed | ${summary.failed ?? 0} |`, - `| Skipped | ${summary.skipped ?? 0} |`, - "", - `**Shards:** ${(verdict.observedShards ?? []).length}/${verdict.expectedShards ?? 0}`, - "", - `**Evidence artifact:** \`${artifactName}\``, - ...(enforcedPrimaryFailure - ? ["", `**Primary failure:** ${enforcedPrimaryFailure.message}`] - : []), - "", - ].join("\n"), -); - -const appendOutput = (name, value) => { - appendFileSync(outputFile, `${name}=${value}\n`); -}; -appendOutput("verdict", enforcedResult); -appendOutput("total", Number(summary.total ?? 0)); -appendOutput("passed", Number(summary.passed ?? 0)); -appendOutput("failed", Number(summary.failed ?? 0)); -appendOutput("skipped", Number(summary.skipped ?? 0)); -appendOutput("verdict-file", verdictFile); - -if (enforcedResult !== "passed" || publicationOutcome !== "success") { - console.error(`E2E evidence is not healthy: ${enforcedResult}`); - if (enforcedPrimaryFailure) console.error(enforcedPrimaryFailure.message); - process.exit(1); -} diff --git a/.github/actions/merge-e2e-evidence/stage-evidence.mjs b/.github/actions/merge-e2e-evidence/stage-evidence.mjs index 8bdbd2a..121563e 100644 --- a/.github/actions/merge-e2e-evidence/stage-evidence.mjs +++ b/.github/actions/merge-e2e-evidence/stage-evidence.mjs @@ -13,7 +13,6 @@ import { basename, isAbsolute, relative, resolve, sep } from "node:path"; const [ workspace, - requestedWorkingDirectory, requestedDownloadDirectory, requestedEvidenceDirectory, requestedExpectedShards, @@ -47,7 +46,6 @@ const assertRelative = (value, label) => { }; for (const [value, label] of [ - [requestedWorkingDirectory, "working-directory"], [requestedDownloadDirectory, "download-directory"], [requestedEvidenceDirectory, "evidence-directory"], ]) { @@ -55,7 +53,6 @@ for (const [value, label] of [ } const workspacePath = realpathSync(workspace); -const workingDirectory = resolve(workspacePath, requestedWorkingDirectory); const downloadDirectory = resolve(workspacePath, requestedDownloadDirectory); const evidenceDirectory = resolve(workspacePath, requestedEvidenceDirectory); @@ -75,12 +72,10 @@ const htmlDirectory = resolve(evidenceDirectory, "playwright-report"); mkdirSync(blobDirectory, { recursive: true }); mkdirSync(ctrfDirectory, { recursive: true }); mkdirSync(statusDirectory, { recursive: true }); +mkdirSync(htmlDirectory, { recursive: true }); const errors = []; -for (const [path, label] of [ - [workingDirectory, "working-directory"], - [downloadDirectory, "download-directory"], -]) { +for (const [path, label] of [[downloadDirectory, "download-directory"]]) { if (!existsSync(path) || !lstatSync(path).isDirectory()) { errors.push(`${label} does not exist or is not a directory: ${path}`); continue; @@ -205,7 +200,6 @@ const appendOutput = (name, value) => { writeFileSync(outputFile, `${name}=${escaped}\n`, { flag: "a" }); }; -appendOutput("working-directory", workingDirectory); appendOutput("evidence-directory", evidenceDirectory); appendOutput("blob-directory", blobDirectory); appendOutput("html-directory", htmlDirectory); diff --git a/.github/actions/package-e2e-report/Dockerfile b/.github/actions/package-e2e-report/Dockerfile new file mode 100644 index 0000000..11f5199 --- /dev/null +++ b/.github/actions/package-e2e-report/Dockerfile @@ -0,0 +1,20 @@ +FROM node:22.22.1-alpine3.23@sha256:8094c002d08262dba12645a3b4a15cd6cd627d30bc782f53229a2ec13ee22a00 + +ENV NODE_ENV=production \ + REPORT_HOST=0.0.0.0 \ + REPORT_PORT=8080 \ + REPORT_ROOT=/app/report + +WORKDIR /app + +COPY --chown=node:node server.mjs ./server.mjs +COPY --chown=node:node report ./report + +USER node + +EXPOSE 8080 + +HEALTHCHECK --interval=5s --timeout=2s --start-period=2s --retries=5 \ + CMD ["node", "-e", "fetch('http://127.0.0.1:8080/').then(response => { if (!response.ok) process.exit(1) }).catch(() => process.exit(1))"] + +CMD ["node", "server.mjs"] diff --git a/.github/actions/package-e2e-report/action.yml b/.github/actions/package-e2e-report/action.yml new file mode 100644 index 0000000..780d6d2 --- /dev/null +++ b/.github/actions/package-e2e-report/action.yml @@ -0,0 +1,91 @@ +name: Package E2E Report Image +description: Build and upload a portable Docker image containing a merged Playwright HTML report. + +inputs: + report-directory: + description: Directory containing the merged Playwright HTML report and its index.html. + required: true + image-name: + description: Immutable-by-run Docker image name and tag supplied by the caller. + required: true + archive-directory: + description: Repository-relative directory where the Docker image archive is written. + required: false + default: e2e-report-image + artifact-name: + description: GitHub artifact containing the Docker image archive. + required: false + default: e2e-report-image + retention-days: + description: Number of days GitHub retains the image artifact. + required: false + default: "10" + +outputs: + image-name: + description: Docker image name and tag built by the action. + value: ${{ steps.package.outputs.image-name }} + image-id: + description: Immutable local Docker image ID. + value: ${{ steps.package.outputs.image-id }} + artifact-name: + description: GitHub artifact containing the Docker archive. + value: ${{ inputs.artifact-name }} + archive-path: + description: Absolute path to playwright-e2e-report.tar.zst on the runner. + value: ${{ steps.package.outputs.archive-path }} + +runs: + using: composite + steps: + - id: package + name: Build and archive report image + shell: bash + env: + E2E_REPORT_DIRECTORY: ${{ inputs.report-directory }} + E2E_REPORT_IMAGE_NAME: ${{ inputs.image-name }} + E2E_REPORT_ARCHIVE_DIRECTORY: ${{ inputs.archive-directory }} + run: | + set -euo pipefail + + eval "$(node "$GITHUB_ACTION_PATH/prepare-context.mjs" \ + "$GITHUB_WORKSPACE" \ + "$GITHUB_ACTION_PATH" \ + "$E2E_REPORT_DIRECTORY" \ + "$E2E_REPORT_IMAGE_NAME" \ + "$E2E_REPORT_ARCHIVE_DIRECTORY")" + + cleanup() { + rm -rf -- "$E2E_REPORT_BUILD_CONTEXT" + } + trap cleanup EXIT + + docker build \ + --file "$E2E_REPORT_BUILD_CONTEXT/Dockerfile" \ + --tag "$E2E_REPORT_IMAGE_NAME" \ + "$E2E_REPORT_BUILD_CONTEXT" + + image_id="$(docker image inspect --format '{{.Id}}' "$E2E_REPORT_IMAGE_NAME")" + if [[ ! "$image_id" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo "Invalid Docker image ID: $image_id" >&2 + exit 1 + fi + + docker save "$E2E_REPORT_IMAGE_NAME" | + zstd -T0 -3 --quiet -o "$E2E_REPORT_ARCHIVE_PATH" + test -s "$E2E_REPORT_ARCHIVE_PATH" + + { + echo "image-name=$E2E_REPORT_IMAGE_NAME" + echo "image-id=$image_id" + echo "archive-path=$E2E_REPORT_ARCHIVE_PATH" + } >> "$GITHUB_OUTPUT" + + - name: Upload report image archive + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: ${{ inputs.artifact-name }} + path: ${{ steps.package.outputs.archive-path }} + if-no-files-found: error + retention-days: ${{ inputs.retention-days }} + compression-level: 0 diff --git a/.github/actions/package-e2e-report/prepare-context.mjs b/.github/actions/package-e2e-report/prepare-context.mjs new file mode 100644 index 0000000..6cb8b1c --- /dev/null +++ b/.github/actions/package-e2e-report/prepare-context.mjs @@ -0,0 +1,163 @@ +import { + copyFileSync, + cpSync, + existsSync, + lstatSync, + mkdirSync, + readdirSync, + realpathSync, + rmSync, +} from "node:fs"; +import { basename, isAbsolute, relative, resolve, sep } from "node:path"; + +const [ + workspace, + actionPath, + requestedReportDirectory, + imageName, + requestedArchiveDirectory, +] = process.argv.slice(2); + +const fail = (message) => { + throw new Error(message); +}; + +if ( + !workspace || + !actionPath || + !requestedReportDirectory || + !imageName || + !requestedArchiveDirectory +) { + fail( + "prepare-context requires workspace, action path, report directory, " + + "image name, and archive directory", + ); +} + +const workspacePath = realpathSync(workspace); +const isInsideWorkspace = (path) => { + const relativePath = relative(workspacePath, path); + return ( + relativePath !== "" && + relativePath !== ".." && + !relativePath.startsWith(`..${sep}`) && + !isAbsolute(relativePath) + ); +}; + +const resolveWorkspaceInput = (value, label, mustExist) => { + let candidate = isAbsolute(value) + ? resolve(value) + : resolve(workspacePath, value); + if (mustExist && !existsSync(candidate)) { + fail(`${label} does not exist: ${value}`); + } + if (mustExist) { + candidate = realpathSync(candidate); + } else { + let existingAncestor = candidate; + while (!existsSync(existingAncestor)) { + const parent = resolve(existingAncestor, ".."); + if (parent === existingAncestor) { + fail(`${label} has no existing parent directory`); + } + existingAncestor = parent; + } + const realAncestor = realpathSync(existingAncestor); + candidate = resolve(realAncestor, relative(existingAncestor, candidate)); + } + if (!isInsideWorkspace(candidate)) { + fail(`${label} must resolve below GITHUB_WORKSPACE`); + } + return candidate; +}; + +const tagSeparator = imageName.lastIndexOf(":"); +const lastSlash = imageName.lastIndexOf("/"); +const repository = imageName.slice(0, tagSeparator); +const tag = imageName.slice(tagSeparator + 1); +if ( + imageName.length > 255 || + tagSeparator <= lastSlash || + !/^[a-z0-9][a-z0-9._/:-]*$/.test(repository) || + !/^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$/.test(tag) +) { + fail( + `image-name must be a tagged Docker image reference; received ${imageName}`, + ); +} + +const reportDirectory = resolveWorkspaceInput( + requestedReportDirectory, + "report-directory", + true, +); +if (!lstatSync(reportDirectory).isDirectory()) { + fail("report-directory must be a directory"); +} +if (!existsSync(resolve(reportDirectory, "index.html"))) { + fail("report-directory must contain index.html"); +} + +const assertPortableTree = (directory) => { + for (const entry of readdirSync(directory, { withFileTypes: true })) { + const entryPath = resolve(directory, entry.name); + if (entry.isSymbolicLink()) { + fail(`report-directory must not contain symbolic links: ${entry.name}`); + } + if (entry.isDirectory()) { + assertPortableTree(entryPath); + } else if (!entry.isFile()) { + fail(`report-directory contains an unsupported file type: ${entry.name}`); + } + } +}; +assertPortableTree(reportDirectory); + +const archiveDirectory = resolveWorkspaceInput( + requestedArchiveDirectory, + "archive-directory", + false, +); +const isSameOrInside = (parent, child) => { + const relativePath = relative(parent, child); + return ( + relativePath === "" || + (!relativePath.startsWith(`..${sep}`) && !isAbsolute(relativePath)) + ); +}; +if ( + isSameOrInside(reportDirectory, archiveDirectory) || + isSameOrInside(archiveDirectory, reportDirectory) +) { + fail("archive-directory and report-directory must not overlap"); +} +mkdirSync(archiveDirectory, { recursive: true }); + +const buildContext = resolve(archiveDirectory, ".build-context"); +rmSync(buildContext, { recursive: true, force: true }); +mkdirSync(buildContext); +copyFileSync( + resolve(actionPath, "Dockerfile"), + resolve(buildContext, "Dockerfile"), +); +copyFileSync( + resolve(actionPath, "server.mjs"), + resolve(buildContext, "server.mjs"), +); +cpSync(reportDirectory, resolve(buildContext, "report"), { recursive: true }); + +const archivePath = resolve(archiveDirectory, "playwright-e2e-report.tar.zst"); +rmSync(archivePath, { force: true }); +const quoteForShell = (value) => `'${value.replaceAll("'", `'"'"'`)}'`; +for (const [name, value] of Object.entries({ + E2E_REPORT_BUILD_CONTEXT: buildContext, + E2E_REPORT_ARCHIVE_PATH: archivePath, +})) { + process.stdout.write(`${name}=${quoteForShell(value)}\n`); +} + +process.stderr.write( + `Packaging ${basename(reportDirectory)} as ${imageName} into ${archivePath}\n`, +); diff --git a/.github/actions/package-e2e-report/server.mjs b/.github/actions/package-e2e-report/server.mjs new file mode 100644 index 0000000..52bf08a --- /dev/null +++ b/.github/actions/package-e2e-report/server.mjs @@ -0,0 +1,196 @@ +import { createReadStream, existsSync, realpathSync, statSync } from "node:fs"; +import { createServer } from "node:http"; +import { extname, isAbsolute, relative, resolve, sep } from "node:path"; + +const host = process.env.REPORT_HOST ?? "0.0.0.0"; +const port = Number.parseInt(process.env.REPORT_PORT ?? "8080", 10); +const reportRoot = realpathSync(process.env.REPORT_ROOT ?? "/app/report"); +const indexFile = resolve(reportRoot, "index.html"); + +if (!Number.isInteger(port) || port < 1 || port > 65_535) { + throw new Error( + `REPORT_PORT must be an integer from 1 through 65535; received ${port}`, + ); +} +if (!existsSync(indexFile) || !statSync(indexFile).isFile()) { + throw new Error(`Playwright report index does not exist: ${indexFile}`); +} + +const contentTypes = new Map([ + [".css", "text/css; charset=utf-8"], + [".gif", "image/gif"], + [".gz", "application/gzip"], + [".html", "text/html; charset=utf-8"], + [".ico", "image/x-icon"], + [".jpeg", "image/jpeg"], + [".jpg", "image/jpeg"], + [".js", "text/javascript; charset=utf-8"], + [".json", "application/json; charset=utf-8"], + [".map", "application/json; charset=utf-8"], + [".mjs", "text/javascript; charset=utf-8"], + [".mp4", "video/mp4"], + [".png", "image/png"], + [".svg", "image/svg+xml"], + [".ttf", "font/ttf"], + [".txt", "text/plain; charset=utf-8"], + [".webm", "video/webm"], + [".wasm", "application/wasm"], + [".webmanifest", "application/manifest+json"], + [".webp", "image/webp"], + [".woff", "font/woff"], + [".woff2", "font/woff2"], + [".xml", "application/xml; charset=utf-8"], + [".zip", "application/zip"], +]); + +const sendText = (response, statusCode, message, method = "GET") => { + const body = `${message}\n`; + response.writeHead(statusCode, { + "content-length": Buffer.byteLength(body), + "content-type": "text/plain; charset=utf-8", + "x-content-type-options": "nosniff", + }); + if (method === "HEAD") response.end(); + else response.end(body); +}; + +const isInsideReport = (path) => { + const relativePath = relative(reportRoot, path); + return ( + relativePath !== ".." && + !relativePath.startsWith(`..${sep}`) && + !isAbsolute(relativePath) + ); +}; + +const findFile = (pathname, acceptsHtml) => { + let decodedPath; + try { + decodedPath = decodeURIComponent(pathname); + } catch { + return { error: 400 }; + } + if (decodedPath.includes("\0")) return { error: 400 }; + + const candidate = resolve(reportRoot, `.${decodedPath}`); + if (!isInsideReport(candidate)) return { error: 403 }; + + let selected = candidate; + if (existsSync(selected) && statSync(selected).isDirectory()) { + selected = resolve(selected, "index.html"); + } + + if (!existsSync(selected) || !statSync(selected).isFile()) { + const hasExtension = extname(decodedPath) !== ""; + if (acceptsHtml && !hasExtension) return { path: indexFile }; + return { error: 404 }; + } + + const realPath = realpathSync(selected); + if (!isInsideReport(realPath)) return { error: 403 }; + return { path: realPath }; +}; + +const parseRange = (header, size) => { + if (!header) return null; + const match = /^bytes=(\d*)-(\d*)$/.exec(header); + if (!match || (match[1] === "" && match[2] === "")) return { error: true }; + + let start; + let end; + if (match[1] === "") { + const suffixLength = Number(match[2]); + if (!Number.isSafeInteger(suffixLength) || suffixLength < 1) { + return { error: true }; + } + start = Math.max(0, size - suffixLength); + end = size - 1; + } else { + start = Number(match[1]); + end = match[2] === "" ? size - 1 : Number(match[2]); + } + + if ( + !Number.isSafeInteger(start) || + !Number.isSafeInteger(end) || + start < 0 || + end < start || + start >= size + ) { + return { error: true }; + } + return { start, end: Math.min(end, size - 1) }; +}; + +const server = createServer((request, response) => { + const method = request.method ?? "GET"; + if (method !== "GET" && method !== "HEAD") { + response.setHeader("allow", "GET, HEAD"); + sendText(response, 405, "Method not allowed", method); + return; + } + + const url = new URL(request.url ?? "/", "http://localhost"); + const acceptsHtml = (request.headers.accept ?? "").includes("text/html"); + const selection = findFile(url.pathname, acceptsHtml); + if (selection.error) { + sendText( + response, + selection.error, + selection.error === 404 ? "Not found" : "Invalid path", + method, + ); + return; + } + + const filePath = selection.path; + const size = statSync(filePath).size; + const range = parseRange(request.headers.range, size); + if (range?.error) { + response.writeHead(416, { + "content-range": `bytes */${size}`, + "content-length": "0", + }); + response.end(); + return; + } + + const start = range?.start ?? 0; + const end = range?.end ?? Math.max(0, size - 1); + const statusCode = range ? 206 : 200; + const headers = { + "accept-ranges": "bytes", + "cache-control": + extname(filePath) === ".html" ? "no-cache" : "public, max-age=3600", + "content-length": String(size === 0 ? 0 : end - start + 1), + "content-type": + contentTypes.get(extname(filePath).toLowerCase()) ?? + "application/octet-stream", + "x-content-type-options": "nosniff", + }; + if (range) headers["content-range"] = `bytes ${start}-${end}/${size}`; + response.writeHead(statusCode, headers); + if (method === "HEAD" || size === 0) { + response.end(); + return; + } + + const stream = createReadStream(filePath, { start, end }); + stream.on("error", () => response.destroy()); + stream.pipe(response); +}); + +server.listen(port, host, () => { + console.log(`Playwright report available at http://${host}:${port}`); +}); + +const shutdown = () => { + server.close((error) => { + if (error) { + console.error(error); + process.exit(1); + } + }); +}; +process.once("SIGINT", shutdown); +process.once("SIGTERM", shutdown); diff --git a/.github/actions/publish-e2e-evidence/action.yml b/.github/actions/publish-e2e-evidence/action.yml new file mode 100644 index 0000000..4479685 --- /dev/null +++ b/.github/actions/publish-e2e-evidence/action.yml @@ -0,0 +1,116 @@ +name: Publish E2E Evidence +description: Finalize, publish, summarize, and enforce the authoritative E2E verdict. + +inputs: + evidence-directory: + description: Repository-relative directory containing merged E2E evidence. + required: false + default: e2e-evidence + expected-shards: + description: Number of shard results required by the run. + required: true + artifact-name: + description: Name of the durable merged evidence artifact. + required: false + default: e2e-evidence + retention-days: + description: Number of days GitHub retains the merged evidence artifact. + required: false + default: "10" + runner-download-outcome: + description: Outcome of downloading the immutable runner archive. + required: true + runner-load-outcome: + description: Outcome of loading the immutable runner archive. + required: true + runner-verification-outcome: + description: Outcome of verifying the loaded runner image ID. + required: true + shard-download-outcome: + description: Outcome of downloading shard evidence. + required: true + merge-outcome: + description: Outcome of deterministic evidence assembly. + required: true + report-publication-outcome: + description: Outcome of building and publishing the report image archive. + required: true + +outputs: + verdict: + description: "Authoritative verdict: passed, failed, or infrastructure-error." + value: ${{ steps.evaluate.outputs.verdict }} + artifact-name: + description: Merged evidence artifact name. + value: ${{ inputs.artifact-name }} + total: + description: Aggregated test count. + value: ${{ steps.evaluate.outputs.total }} + passed: + description: Aggregated passed test count. + value: ${{ steps.evaluate.outputs.passed }} + failed: + description: Aggregated failed test count. + value: ${{ steps.evaluate.outputs.failed }} + skipped: + description: Aggregated skipped test count. + value: ${{ steps.evaluate.outputs.skipped }} + verdict-file: + description: Absolute path to the finalized verdict.json. + value: ${{ steps.prepare.outputs.verdict-file }} + +runs: + using: composite + steps: + - id: prepare + name: Finalize authoritative verdict + shell: bash + env: + E2E_EVIDENCE_DIRECTORY: ${{ inputs.evidence-directory }} + E2E_EXPECTED_SHARDS: ${{ inputs.expected-shards }} + E2E_RUNNER_DOWNLOAD_OUTCOME: ${{ inputs.runner-download-outcome }} + E2E_RUNNER_LOAD_OUTCOME: ${{ inputs.runner-load-outcome }} + E2E_RUNNER_VERIFICATION_OUTCOME: ${{ inputs.runner-verification-outcome }} + E2E_SHARD_DOWNLOAD_OUTCOME: ${{ inputs.shard-download-outcome }} + E2E_MERGE_OUTCOME: ${{ inputs.merge-outcome }} + E2E_REPORT_PUBLICATION_OUTCOME: ${{ inputs.report-publication-outcome }} + run: | + set -euo pipefail + node "$GITHUB_ACTION_PATH/prepare-publication.mjs" \ + "$GITHUB_WORKSPACE" \ + "$E2E_EVIDENCE_DIRECTORY" \ + "$E2E_EXPECTED_SHARDS" \ + "$E2E_RUNNER_DOWNLOAD_OUTCOME" \ + "$E2E_RUNNER_LOAD_OUTCOME" \ + "$E2E_RUNNER_VERIFICATION_OUTCOME" \ + "$E2E_SHARD_DOWNLOAD_OUTCOME" \ + "$E2E_MERGE_OUTCOME" \ + "$E2E_REPORT_PUBLICATION_OUTCOME" \ + "$GITHUB_OUTPUT" + + - id: publish + name: Upload authoritative E2E evidence + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: ${{ inputs.artifact-name }} + path: ${{ steps.prepare.outputs.evidence-directory }}/ + if-no-files-found: error + retention-days: ${{ inputs.retention-days }} + + - id: evaluate + name: Publish summary and enforce verdict + if: always() && steps.prepare.outcome == 'success' + shell: bash + env: + E2E_VERDICT_FILE: ${{ steps.prepare.outputs.verdict-file }} + E2E_ARTIFACT_NAME: ${{ inputs.artifact-name }} + E2E_PUBLICATION_OUTCOME: ${{ steps.publish.outcome }} + run: | + set -euo pipefail + node "$GITHUB_ACTION_PATH/evaluate-publication.mjs" \ + "$E2E_VERDICT_FILE" \ + "$E2E_ARTIFACT_NAME" \ + "${E2E_PUBLICATION_OUTCOME:-failure}" \ + "$GITHUB_STEP_SUMMARY" \ + "$GITHUB_OUTPUT" diff --git a/.github/actions/publish-e2e-evidence/evaluate-publication.mjs b/.github/actions/publish-e2e-evidence/evaluate-publication.mjs new file mode 100644 index 0000000..a745e81 --- /dev/null +++ b/.github/actions/publish-e2e-evidence/evaluate-publication.mjs @@ -0,0 +1,63 @@ +import { appendFileSync, readFileSync } from "node:fs"; + +const [verdictFile, artifactName, publicationOutcome, summaryFile, outputFile] = + process.argv.slice(2); +const verdict = JSON.parse(readFileSync(verdictFile, "utf8")); +const summary = verdict.summary ?? {}; +const enforcedResult = + verdict.result === "passed" && publicationOutcome !== "success" + ? "infrastructure-error" + : verdict.result; +const publicationFailure = + publicationOutcome === "success" + ? null + : `Evidence artifact publication finished with ${publicationOutcome || "unknown"}`; +const primaryFailure = verdict.primaryFailure?.message ?? publicationFailure; + +appendFileSync( + summaryFile, + [ + "## E2E Gate", + "", + `**Verdict:** ${enforcedResult}`, + "", + "| Result | Count |", + "| --- | ---: |", + `| Total | ${summary.total ?? 0} |`, + `| Passed | ${summary.passed ?? 0} |`, + `| Failed | ${summary.failed ?? 0} |`, + `| Skipped | ${summary.skipped ?? 0} |`, + "", + `**Shards:** ${(verdict.observedShards ?? []).length}/${verdict.expectedShards ?? 0}`, + "", + `**Evidence artifact:** \`${artifactName}\``, + ...(primaryFailure ? ["", `**Primary failure:** ${primaryFailure}`] : []), + ...(publicationFailure && publicationFailure !== primaryFailure + ? ["", `**Publication failure:** ${publicationFailure}`] + : []), + "", + ].join("\n"), +); + +const appendOutput = (name, value) => { + const escaped = String(value) + .replaceAll("%", "%25") + .replaceAll("\r", "%0D") + .replaceAll("\n", "%0A"); + appendFileSync(outputFile, `${name}=${escaped}\n`); +}; +appendOutput("verdict", enforcedResult); +appendOutput("total", Number(summary.total ?? 0)); +appendOutput("passed", Number(summary.passed ?? 0)); +appendOutput("failed", Number(summary.failed ?? 0)); +appendOutput("skipped", Number(summary.skipped ?? 0)); +appendOutput("primary-failure", primaryFailure ?? ""); + +if (enforcedResult !== "passed" || publicationOutcome !== "success") { + console.error(`E2E evidence is not healthy: ${enforcedResult}`); + if (primaryFailure) console.error(primaryFailure); + if (publicationFailure && publicationFailure !== primaryFailure) { + console.error(publicationFailure); + } + process.exit(1); +} diff --git a/.github/actions/publish-e2e-evidence/prepare-publication.mjs b/.github/actions/publish-e2e-evidence/prepare-publication.mjs new file mode 100644 index 0000000..e9229c0 --- /dev/null +++ b/.github/actions/publish-e2e-evidence/prepare-publication.mjs @@ -0,0 +1,137 @@ +import { + existsSync, + lstatSync, + mkdirSync, + readFileSync, + realpathSync, + writeFileSync, +} from "node:fs"; +import { isAbsolute, relative, resolve, sep } from "node:path"; + +const [ + workspace, + requestedEvidenceDirectory, + requestedExpectedShards, + runnerDownloadOutcome, + runnerLoadOutcome, + runnerVerificationOutcome, + shardDownloadOutcome, + mergeOutcome, + reportPublicationOutcome, + outputFile, +] = process.argv.slice(2); + +const expectedShards = Number(requestedExpectedShards); +if ( + !Number.isSafeInteger(expectedShards) || + expectedShards < 1 || + expectedShards > 100 +) { + throw new Error("expected-shards must be an integer between 1 and 100"); +} +if (!requestedEvidenceDirectory || isAbsolute(requestedEvidenceDirectory)) { + throw new Error("evidence-directory must be a non-empty repository-relative path"); +} + +const workspacePath = realpathSync(workspace); +const evidenceDirectory = resolve(workspacePath, requestedEvidenceDirectory); +const relativeEvidenceDirectory = relative(workspacePath, evidenceDirectory); +if ( + relativeEvidenceDirectory === "" || + relativeEvidenceDirectory === ".." || + relativeEvidenceDirectory.startsWith(`..${sep}`) || + isAbsolute(relativeEvidenceDirectory) +) { + throw new Error("evidence-directory must resolve below GITHUB_WORKSPACE"); +} +if ( + existsSync(evidenceDirectory) && + lstatSync(evidenceDirectory).isSymbolicLink() +) { + throw new Error("evidence-directory must not be a symbolic link"); +} +mkdirSync(evidenceDirectory, { recursive: true }); + +const verdictFile = resolve(evidenceDirectory, "verdict.json"); +let verdict = null; +if (existsSync(verdictFile)) { + try { + verdict = JSON.parse(readFileSync(verdictFile, "utf8")); + } catch { + verdict = null; + } +} +if (!verdict || typeof verdict !== "object") { + verdict = { + schemaVersion: 1, + result: "infrastructure-error", + primaryFailure: { + kind: "infrastructure", + phase: "evidence-assembly", + message: "Evidence assembly did not produce a readable verdict.json", + }, + secondaryFailures: [], + expectedShards, + observedShards: [], + summary: { + total: 0, + passed: 0, + failed: 0, + skipped: 0, + pending: 0, + other: 0, + }, + shards: [], + reports: {}, + generatedAt: new Date().toISOString(), + }; +} + +const checks = [ + ["runner-archive-download", runnerDownloadOutcome], + ["runner-image-load", runnerLoadOutcome], + ["runner-image-verification", runnerVerificationOutcome], + ["shard-evidence-download", shardDownloadOutcome], + ["evidence-merge", mergeOutcome], + ["report-image-publication", reportPublicationOutcome], +]; +const newFailures = checks + .filter(([, outcome]) => outcome !== "success") + .map(([phase, outcome]) => ({ + kind: "infrastructure", + phase, + message: `${phase} finished with ${outcome || "unknown"}`, + })); + +verdict.secondaryFailures = Array.isArray(verdict.secondaryFailures) + ? verdict.secondaryFailures + : []; +if (newFailures.length > 0) { + if (!verdict.primaryFailure) { + verdict.primaryFailure = newFailures.shift(); + } + verdict.secondaryFailures.push(...newFailures); + if (verdict.result === "passed" || !verdict.result) { + verdict.result = "infrastructure-error"; + } +} +verdict.expectedShards = expectedShards; +verdict.artifacts = { + ...(verdict.artifacts ?? {}), + reportImage: + reportPublicationOutcome === "success" + ? { published: true, name: "e2e-report-image" } + : { published: false, name: null }, +}; +verdict.finalizedAt = new Date().toISOString(); +writeFileSync(verdictFile, `${JSON.stringify(verdict, null, 2)}\n`); + +const appendOutput = (name, value) => { + const escaped = String(value) + .replaceAll("%", "%25") + .replaceAll("\r", "%0D") + .replaceAll("\n", "%0A"); + writeFileSync(outputFile, `${name}=${escaped}\n`, { flag: "a" }); +}; +appendOutput("evidence-directory", evidenceDirectory); +appendOutput("verdict-file", verdictFile); diff --git a/.github/actions/run-e2e/action.yml b/.github/actions/run-e2e/action.yml index ff9ba52..0949ee6 100644 --- a/.github/actions/run-e2e/action.yml +++ b/.github/actions/run-e2e/action.yml @@ -18,6 +18,18 @@ inputs: description: Base URL exposed by the Dockerized SUT. required: false default: http://127.0.0.1:4173 + runner-image-name: + description: Exact local Docker image reference loaded from the runner artifact. + required: true + runner-image-id: + description: Expected content-addressed Docker image ID. + required: true + runner-download-outcome: + description: Outcome of the workflow step that downloaded the runner image artifact. + required: true + runner-load-outcome: + description: Outcome of the workflow step that loaded the runner image archive. + required: true projects: description: Newline-delimited Playwright project names; empty selects all projects. required: false @@ -66,6 +78,10 @@ runs: E2E_PLAYWRIGHT_CONFIG: ${{ inputs.playwright-config }} E2E_COMPOSE_FILE: ${{ inputs.compose-file }} E2E_BASE_URL: ${{ inputs.base-url }} + E2E_RUNNER_IMAGE_NAME: ${{ inputs.runner-image-name }} + E2E_RUNNER_IMAGE_ID: ${{ inputs.runner-image-id }} + E2E_RUNNER_DOWNLOAD_OUTCOME: ${{ inputs.runner-download-outcome }} + E2E_RUNNER_LOAD_OUTCOME: ${{ inputs.runner-load-outcome }} E2E_PROJECTS: ${{ inputs.projects }} E2E_LABELS: ${{ inputs.labels }} E2E_LABEL_MATCH: ${{ inputs.label-match }} @@ -81,6 +97,10 @@ runs: "$E2E_PLAYWRIGHT_CONFIG" \ "$E2E_COMPOSE_FILE" \ "$E2E_BASE_URL" \ + "$E2E_RUNNER_IMAGE_NAME" \ + "$E2E_RUNNER_IMAGE_ID" \ + "$E2E_RUNNER_DOWNLOAD_OUTCOME" \ + "$E2E_RUNNER_LOAD_OUTCOME" \ "$E2E_PROJECTS" \ "$E2E_LABELS" \ "$E2E_LABEL_MATCH" \ @@ -90,18 +110,54 @@ runs: { echo "working-directory=$E2E_ABSOLUTE_WORKING_DIRECTORY" - echo "config=$E2E_ABSOLUTE_CONFIG" + echo "config=$E2E_CONTAINER_CONFIG" echo "compose-file=$E2E_ABSOLUTE_COMPOSE_FILE" echo "base-url=$E2E_BASE_URL" echo "compose-project=$E2E_COMPOSE_PROJECT" + echo "runner-image-name=$E2E_RUNNER_IMAGE_NAME" + echo "runner-image-id=$E2E_RUNNER_IMAGE_ID" echo "selection-file=$E2E_SELECTION_FILE" echo "shard-directory=$E2E_SHARD_DIRECTORY" echo "status-file=$E2E_STATUS_FILE" } >> "$GITHUB_OUTPUT" + - name: Verify loaded E2E runner image + id: runner + if: >- + always() && steps.initialize.outcome == 'success' && + inputs.runner-download-outcome == 'success' && + inputs.runner-load-outcome == 'success' + continue-on-error: true + shell: bash + env: + E2E_RUNNER_IMAGE_NAME: ${{ steps.initialize.outputs.runner-image-name }} + E2E_RUNNER_IMAGE_ID: ${{ steps.initialize.outputs.runner-image-id }} + E2E_SHARD_DIRECTORY: ${{ steps.initialize.outputs.shard-directory }} + run: | + set +e + runner_log="$E2E_SHARD_DIRECTORY/runner-image.log" + actual_image_id="$(docker image inspect \ + --format '{{.Id}}' "$E2E_RUNNER_IMAGE_NAME" 2> "$runner_log")" + runner_exit_code=$? + + if [[ "$runner_exit_code" -eq 0 && "$actual_image_id" != "$E2E_RUNNER_IMAGE_ID" ]]; then + echo "Runner image ID mismatch: expected $E2E_RUNNER_IMAGE_ID, got $actual_image_id" \ + >> "$runner_log" + runner_exit_code=42 + fi + + cat "$runner_log" + { + echo "exit-code=$runner_exit_code" + echo "actual-image-id=$actual_image_id" + } >> "$GITHUB_OUTPUT" + exit "$runner_exit_code" + - name: Start Dockerized SUT id: start - if: always() && steps.initialize.outcome == 'success' + if: >- + always() && steps.initialize.outcome == 'success' && + steps.runner.outputs.exit-code == '0' continue-on-error: true shell: bash env: @@ -123,13 +179,16 @@ runs: - name: Run Playwright shard id: playwright - if: steps.initialize.outcome == 'success' && steps.start.outputs.exit-code == '0' + if: >- + steps.initialize.outcome == 'success' && + steps.runner.outputs.exit-code == '0' && + steps.start.outputs.exit-code == '0' continue-on-error: true shell: bash env: - E2E_WORKING_DIRECTORY: ${{ steps.initialize.outputs.working-directory }} E2E_CONFIG: ${{ steps.initialize.outputs.config }} E2E_BASE_URL: ${{ steps.initialize.outputs.base-url }} + E2E_RUNNER_IMAGE_ID: ${{ steps.runner.outputs.actual-image-id }} E2E_SHARD_DIRECTORY: ${{ steps.initialize.outputs.shard-directory }} E2E_SELECTION_FILE: ${{ steps.initialize.outputs.selection-file }} E2E_SHARD_INDEX: ${{ inputs.shard-index }} @@ -142,7 +201,7 @@ runs: selection_exit_code=$? if [[ "$selection_exit_code" -ne 0 ]]; then echo "exit-code=$selection_exit_code" >> "$GITHUB_OUTPUT" - echo "failure-kind=infrastructure" >> "$GITHUB_OUTPUT" + echo "failure-kind=selection" >> "$GITHUB_OUTPUT" exit "$selection_exit_code" fi @@ -151,21 +210,37 @@ runs: playwright_selection_args+=("$argument") done < "$selection_args_file" ( - cd "$E2E_WORKING_DIRECTORY" - BASE_URL="$E2E_BASE_URL" \ - CI=true \ - E2E_RESULTS_DIR="$E2E_SHARD_DIRECTORY/test-results" \ - PLAYWRIGHT_BLOB_OUTPUT_DIR="$E2E_SHARD_DIRECTORY/blob-report" \ - PLAYWRIGHT_OUTPUT_DIR="$E2E_SHARD_DIRECTORY/test-results" \ - CTRF_OUTPUT_FILE="$E2E_SHARD_DIRECTORY/ctrf/ctrf-report.json" \ - pnpm exec playwright test \ + docker run --rm \ + --init \ + --network=host \ + --ipc=host \ + --read-only \ + --tmpfs /tmp:rw,nosuid,nodev,size=512m \ + --cap-drop=ALL \ + --security-opt=no-new-privileges:true \ + --pids-limit=1024 \ + --user "$(id -u):$(id -g)" \ + --env BASE_URL="$E2E_BASE_URL" \ + --env CI=true \ + --env HOME=/tmp \ + --env E2E_RESULTS_DIR=/evidence/test-results \ + --env PLAYWRIGHT_BLOB_OUTPUT_DIR=/evidence/blob-report \ + --env PLAYWRIGHT_OUTPUT_DIR=/evidence/test-results \ + --env CTRF_OUTPUT_FILE=/evidence/ctrf/ctrf-report.json \ + --volume "$E2E_SHARD_DIRECTORY:/evidence" \ + "$E2E_RUNNER_IMAGE_ID" \ --config "$E2E_CONFIG" \ --shard="$E2E_SHARD_INDEX/$E2E_SHARD_TOTAL" \ "${playwright_selection_args[@]}" ) playwright_exit_code=$? echo "exit-code=$playwright_exit_code" >> "$GITHUB_OUTPUT" - echo "failure-kind=test" >> "$GITHUB_OUTPUT" + if [[ "$playwright_exit_code" =~ ^12[567]$ ]] || \ + [[ "$playwright_exit_code" -ge 128 ]]; then + echo "failure-kind=runner-container" >> "$GITHUB_OUTPUT" + else + echo "failure-kind=test" >> "$GITHUB_OUTPUT" + fi exit "$playwright_exit_code" - name: Capture Dockerized SUT logs @@ -216,6 +291,10 @@ runs: shell: bash env: E2E_STATUS_FILE: ${{ steps.initialize.outputs.status-file }} + E2E_RUNNER_DOWNLOAD_OUTCOME: ${{ inputs.runner-download-outcome }} + E2E_RUNNER_LOAD_OUTCOME: ${{ inputs.runner-load-outcome }} + E2E_RUNNER_EXIT_CODE: ${{ steps.runner.outputs.exit-code }} + E2E_ACTUAL_RUNNER_IMAGE_ID: ${{ steps.runner.outputs.actual-image-id }} E2E_STARTUP_EXIT_CODE: ${{ steps.start.outputs.exit-code }} E2E_PLAYWRIGHT_EXIT_CODE: ${{ steps.playwright.outputs.exit-code }} E2E_PLAYWRIGHT_FAILURE_KIND: ${{ steps.playwright.outputs.failure-kind }} @@ -225,6 +304,10 @@ runs: set +e node "$GITHUB_ACTION_PATH/complete-shard.mjs" \ "$E2E_STATUS_FILE" \ + "$E2E_RUNNER_DOWNLOAD_OUTCOME" \ + "$E2E_RUNNER_LOAD_OUTCOME" \ + "$E2E_RUNNER_EXIT_CODE" \ + "$E2E_ACTUAL_RUNNER_IMAGE_ID" \ "$E2E_STARTUP_EXIT_CODE" \ "$E2E_PLAYWRIGHT_EXIT_CODE" \ "$E2E_PLAYWRIGHT_FAILURE_KIND" \ diff --git a/.github/actions/run-e2e/complete-shard.mjs b/.github/actions/run-e2e/complete-shard.mjs index 2be8091..6b3a9ae 100644 --- a/.github/actions/run-e2e/complete-shard.mjs +++ b/.github/actions/run-e2e/complete-shard.mjs @@ -3,6 +3,10 @@ import { dirname } from "node:path"; const [ statusFile, + runnerDownloadOutcome, + runnerLoadOutcome, + requestedRunnerExitCode, + actualRunnerImageId, requestedStartupExitCode, requestedPlaywrightExitCode, requestedPlaywrightFailureKind, @@ -23,6 +27,10 @@ const startupExitCode = parseExitCode( requestedStartupExitCode, "startup", ); +const runnerExitCode = parseExitCode( + requestedRunnerExitCode, + "runner image verification", +); const playwrightExitCode = parseExitCode( requestedPlaywrightExitCode, "Playwright", @@ -38,20 +46,57 @@ const finishedAt = new Date(); const startedAt = new Date(status.startedAt); const failures = []; -if (startupExitCode === null) { +const runnerTransportSucceeded = + runnerDownloadOutcome === "success" && runnerLoadOutcome === "success"; +if (runnerDownloadOutcome !== "success") { + failures.push({ + kind: "infrastructure", + phase: "runner-image-download", + message: `E2E runner image download outcome was ${runnerDownloadOutcome}`, + }); +} +if (runnerLoadOutcome !== "success") { + failures.push({ + kind: "infrastructure", + phase: "runner-image-load", + message: `E2E runner image load outcome was ${runnerLoadOutcome}`, + }); +} +if (runnerTransportSucceeded && runnerExitCode === null) { + failures.push({ + kind: "infrastructure", + phase: "runner-image", + message: "E2E runner image verification did not report an exit code", + }); +} else if (runnerTransportSucceeded && runnerExitCode !== 0) { + failures.push({ + kind: "infrastructure", + phase: "runner-image", + message: `E2E runner image verification exited with ${runnerExitCode}`, + }); +} +if ( + runnerTransportSucceeded && + runnerExitCode === 0 && + startupExitCode === null +) { failures.push({ kind: "infrastructure", phase: "sut-startup", message: "Dockerized SUT startup did not report an exit code", }); -} else if (startupExitCode !== 0) { +} else if ( + runnerTransportSucceeded && + runnerExitCode === 0 && + startupExitCode !== 0 +) { failures.push({ kind: "infrastructure", phase: "sut-startup", message: `Dockerized SUT startup exited with ${startupExitCode}`, }); } -if (startupExitCode === 0 && playwrightExitCode === null) { +if (runnerExitCode === 0 && startupExitCode === 0 && playwrightExitCode === null) { failures.push({ kind: "infrastructure", phase: "playwright-execution", @@ -60,12 +105,18 @@ if (startupExitCode === 0 && playwrightExitCode === null) { } else if ( playwrightExitCode !== null && playwrightExitCode > 0 && - requestedPlaywrightFailureKind === "infrastructure" + requestedPlaywrightFailureKind !== "test" ) { failures.push({ kind: "infrastructure", - phase: "playwright-selection", - message: `Playwright selection preparation exited with ${playwrightExitCode}`, + phase: + requestedPlaywrightFailureKind === "selection" + ? "playwright-selection" + : "runner-container", + message: + requestedPlaywrightFailureKind === "selection" + ? `Playwright selection preparation exited with ${playwrightExitCode}` + : `E2E runner container exited with ${playwrightExitCode}`, }); } else if (playwrightExitCode !== null && playwrightExitCode > 0) { failures.push({ @@ -115,18 +166,28 @@ status.result = testFailure ? "infrastructure-error" : "passed"; -const effectiveExitCode = !status.primaryFailure - ? 0 - : status.primaryFailure.phase === "playwright" - ? (playwrightExitCode ?? 1) - : startupExitCode !== null && startupExitCode !== 0 - ? startupExitCode - : logExitCode !== null && logExitCode !== 0 - ? logExitCode - : teardownExitCode !== null && teardownExitCode !== 0 - ? teardownExitCode - : 1; +const exitCodeForFailure = (failure) => { + switch (failure?.phase) { + case "playwright": + case "playwright-selection": + case "runner-container": + return playwrightExitCode ?? 1; + case "runner-image": + return runnerExitCode ?? 1; + case "sut-startup": + return startupExitCode ?? 1; + case "evidence": + return logExitCode ?? 1; + case "cleanup": + return teardownExitCode ?? 1; + default: + return failure ? 1 : 0; + } +}; +const effectiveExitCode = exitCodeForFailure(status.primaryFailure); status.exitCode = effectiveExitCode; +status.lifecycle.runner.verificationExitCode = runnerExitCode; +status.lifecycle.runner.actualImageId = actualRunnerImageId || null; status.lifecycle.sut.startupExitCode = startupExitCode; status.lifecycle.playwright = { exitCode: playwrightExitCode, diff --git a/.github/actions/run-e2e/prepare-shard.mjs b/.github/actions/run-e2e/prepare-shard.mjs index 502ad5b..83ac925 100644 --- a/.github/actions/run-e2e/prepare-shard.mjs +++ b/.github/actions/run-e2e/prepare-shard.mjs @@ -14,6 +14,10 @@ const [ requestedConfig, requestedComposeFile, requestedBaseUrl, + requestedRunnerImageName, + requestedRunnerImageId, + requestedRunnerDownloadOutcome, + requestedRunnerLoadOutcome, requestedProjects, requestedLabels, requestedLabelMatch, @@ -56,6 +60,28 @@ try { if (!["http:", "https:"].includes(baseUrl.protocol)) { fail("base-url must use HTTP or HTTPS"); } +if (!requestedRunnerImageName || /[\s\0]/.test(requestedRunnerImageName)) { + fail("runner-image-name must be a non-empty Docker reference without whitespace"); +} +if (!/^sha256:[a-f0-9]{64}$/.test(requestedRunnerImageId)) { + fail(`runner-image-id must be a sha256 image ID; received ${requestedRunnerImageId}`); +} +const allowedStepOutcomes = new Set([ + "success", + "failure", + "cancelled", + "skipped", +]); +for (const [label, outcome] of [ + ["runner-download-outcome", requestedRunnerDownloadOutcome], + ["runner-load-outcome", requestedRunnerLoadOutcome], +]) { + if (!allowedStepOutcomes.has(outcome)) { + fail( + `${label} must be success, failure, cancelled, or skipped; received ${outcome}`, + ); + } +} const selection = createSelection( requestedProjects, requestedLabels, @@ -139,6 +165,14 @@ const status = { grep: selection.grep, }, lifecycle: { + runner: { + imageName: requestedRunnerImageName, + expectedImageId: requestedRunnerImageId, + actualImageId: null, + downloadOutcome: requestedRunnerDownloadOutcome, + loadOutcome: requestedRunnerLoadOutcome, + verificationExitCode: null, + }, sut: { composeFile: requestedComposeFile, baseUrl: baseUrl.toString(), @@ -175,9 +209,12 @@ const quoteForShell = (value) => `'${value.replaceAll("'", `'"'"'`)}'`; const variables = { E2E_ABSOLUTE_WORKING_DIRECTORY: resolvedWorkingDirectory, E2E_ABSOLUTE_CONFIG: configPath, + E2E_CONTAINER_CONFIG: requestedConfig, E2E_ABSOLUTE_COMPOSE_FILE: resolvedComposeFile, E2E_BASE_URL: baseUrl.toString(), E2E_COMPOSE_PROJECT: composeProject, + E2E_RUNNER_IMAGE_NAME: requestedRunnerImageName, + E2E_RUNNER_IMAGE_ID: requestedRunnerImageId, E2E_SELECTION_FILE: selectionFile, E2E_SHARD_DIRECTORY: shardDirectory, E2E_STATUS_FILE: statusFile, diff --git a/.github/workflows/reusable-e2e.yml b/.github/workflows/reusable-e2e.yml index e768305..937c628 100644 --- a/.github/workflows/reusable-e2e.yml +++ b/.github/workflows/reusable-e2e.yml @@ -8,16 +8,6 @@ on: required: false default: e2e type: string - node-version: - description: Node.js version used by the E2E package. - required: false - default: "22" - type: string - pnpm-version: - description: pnpm version used to install the E2E package. - required: false - default: 10.26.2 - type: string playwright-config: description: Playwright configuration path, relative to the working directory. required: false @@ -64,6 +54,24 @@ on: default: true type: boolean outputs: + runner-image-name: + description: Exact local E2E runner image reference used by every shard. + value: ${{ jobs.build_runner.outputs.image-name }} + runner-image-id: + description: Content-addressed E2E runner Docker image ID. + value: ${{ jobs.build_runner.outputs.image-id }} + runner-image-artifact: + description: Artifact containing the portable E2E runner Docker archive. + value: ${{ jobs.build_runner.outputs.artifact-name }} + report-image-name: + description: Local image reference for the packaged E2E HTML report. + value: ${{ jobs.merge.outputs.report-image-name }} + report-image-id: + description: Content-addressed E2E report Docker image ID. + value: ${{ jobs.merge.outputs.report-image-id }} + report-image-artifact: + description: Artifact containing the portable E2E report Docker archive. + value: ${{ jobs.merge.outputs.report-image-artifact }} verdict: description: Authoritative E2E verdict. value: ${{ jobs.e2e_gate.outputs.verdict }} @@ -85,7 +93,6 @@ on: permissions: contents: read - pull-requests: write jobs: plan: @@ -111,9 +118,45 @@ jobs: ')" echo "matrix=$matrix" >> "$GITHUB_OUTPUT" + build_runner: + name: Build E2E runner image + needs: plan + runs-on: ubuntu-latest + timeout-minutes: 30 + outputs: + image-name: ${{ steps.build.outputs.image-name }} + image-id: ${{ steps.build.outputs.image-id }} + artifact-name: ${{ steps.upload-runner.outcome == 'success' && 'e2e-runner-image' || '' }} + steps: + - name: Check out caller repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + + - id: build + name: Build E2E runner image + uses: $/.github/actions/build-e2e-runner + with: + working-directory: ${{ inputs.working-directory }} + dockerfile: Dockerfile + image-name: playwright-e2e-runner:${{ github.sha }} + archive-path: .e2e-runner/playwright-e2e-runner.tar.zst + + - id: upload-runner + name: Upload portable E2E runner image + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: e2e-runner-image + path: ${{ steps.build.outputs.archive-path }} + if-no-files-found: error + compression-level: 0 + retention-days: ${{ inputs.artifact-retention-days }} + shards: name: E2E shard ${{ matrix.shard }}/${{ inputs.shard-count }} - needs: plan + needs: + - plan + - build_runner runs-on: ubuntu-latest timeout-minutes: 20 strategy: @@ -122,14 +165,22 @@ jobs: steps: - name: Check out caller repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - - name: Prepare E2E environment - uses: $/.github/actions/prepare-e2e + - id: download-runner + name: Download E2E runner image + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: - working-directory: ${{ inputs.working-directory }} - node-version: ${{ inputs.node-version }} - pnpm-version: ${{ inputs.pnpm-version }} - playwright-browser: chromium + name: ${{ needs.build_runner.outputs.artifact-name }} + path: runner-image + + - id: load-runner + name: Load E2E runner image + continue-on-error: true + shell: bash + run: docker load --input runner-image/playwright-e2e-runner.tar.zst - id: run name: Run Playwright shard @@ -140,6 +191,10 @@ jobs: playwright-config: ${{ inputs.playwright-config }} compose-file: ${{ inputs.compose-file }} base-url: ${{ inputs.base-url }} + runner-image-name: ${{ needs.build_runner.outputs.image-name }} + runner-image-id: ${{ needs.build_runner.outputs.image-id }} + runner-download-outcome: ${{ steps.download-runner.outcome }} + runner-load-outcome: ${{ steps.load-runner.outcome }} projects: ${{ inputs.projects }} labels: ${{ inputs.labels }} label-match: ${{ inputs.label-match }} @@ -157,7 +212,10 @@ jobs: retention-days: ${{ inputs.artifact-retention-days }} - name: Preserve shard verdict - if: always() && steps.run.outcome != 'success' + if: >- + always() && (steps.download-runner.outcome != 'success' || + steps.load-runner.outcome != 'success' || + steps.run.outcome != 'success') shell: bash run: | echo "::error::E2E shard ${{ matrix.shard }} did not complete successfully" @@ -167,29 +225,58 @@ jobs: name: Merge E2E evidence needs: - plan + - build_runner - shards if: always() runs-on: ubuntu-latest timeout-minutes: 10 outputs: - verdict: ${{ steps.merge.outputs.verdict }} - artifact-name: ${{ steps.merge.outputs.artifact-name }} - total: ${{ steps.merge.outputs.total }} - passed: ${{ steps.merge.outputs.passed }} - failed: ${{ steps.merge.outputs.failed }} - skipped: ${{ steps.merge.outputs.skipped }} + verdict: ${{ steps.publish.outputs.verdict }} + artifact-name: ${{ steps.publish.outputs.artifact-name }} + total: ${{ steps.publish.outputs.total }} + passed: ${{ steps.publish.outputs.passed }} + failed: ${{ steps.publish.outputs.failed }} + skipped: ${{ steps.publish.outputs.skipped }} + report-image-name: ${{ steps.package-report.outcome == 'success' && steps.package-report.outputs.image-name || '' }} + report-image-id: ${{ steps.package-report.outcome == 'success' && steps.package-report.outputs.image-id || '' }} + report-image-artifact: ${{ steps.package-report.outcome == 'success' && steps.package-report.outputs.artifact-name || '' }} steps: - - name: Check out caller repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - - name: Prepare report tooling - uses: $/.github/actions/prepare-e2e + - id: download-runner + name: Download E2E runner image + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: - working-directory: ${{ inputs.working-directory }} - node-version: ${{ inputs.node-version }} - pnpm-version: ${{ inputs.pnpm-version }} - playwright-browser: chromium - install-browser: "false" + name: e2e-runner-image + path: runner-image + + - id: load-runner + name: Load E2E runner image + continue-on-error: true + shell: bash + run: docker load --input runner-image/playwright-e2e-runner.tar.zst + + - id: verify-runner + name: Verify immutable E2E runner image + continue-on-error: true + shell: bash + env: + E2E_RUNNER_IMAGE_NAME: ${{ needs.build_runner.outputs.image-name }} + E2E_RUNNER_IMAGE_ID: ${{ needs.build_runner.outputs.image-id }} + run: | + set -euo pipefail + if [[ -z "$E2E_RUNNER_IMAGE_NAME" ]]; then + echo "Expected E2E runner image name is empty" >&2 + exit 1 + fi + if [[ ! "$E2E_RUNNER_IMAGE_ID" =~ ^sha256:[a-f0-9]{64}$ ]]; then + echo "Invalid expected E2E runner image ID: $E2E_RUNNER_IMAGE_ID" >&2 + exit 1 + fi + actual_image_id="$(docker image inspect --format '{{.Id}}' "$E2E_RUNNER_IMAGE_NAME")" + if [[ "$actual_image_id" != "$E2E_RUNNER_IMAGE_ID" ]]; then + echo "E2E runner image ID mismatch: expected $E2E_RUNNER_IMAGE_ID, got $actual_image_id" >&2 + exit 1 + fi - id: download name: Download shard evidence @@ -202,20 +289,47 @@ jobs: merge-multiple: false - id: merge - name: Merge and publish evidence + name: Merge E2E evidence if: always() continue-on-error: true uses: $/.github/actions/merge-e2e-evidence with: - working-directory: ${{ inputs.working-directory }} download-directory: downloaded-shards + evidence-directory: e2e-evidence + expected-shards: ${{ inputs.shard-count }} + runner-image-id: ${{ needs.build_runner.outputs.image-id }} + + - id: package-report + name: Package E2E report image + if: always() + continue-on-error: true + uses: $/.github/actions/package-e2e-report + with: + report-directory: e2e-evidence/playwright-report + image-name: playwright-e2e-report:${{ github.run_id }}-${{ github.run_attempt }} + archive-directory: e2e-report-image + artifact-name: e2e-report-image + retention-days: ${{ inputs.artifact-retention-days }} + + - id: publish + name: Publish and enforce authoritative evidence + if: always() + continue-on-error: true + uses: $/.github/actions/publish-e2e-evidence + with: evidence-directory: e2e-evidence expected-shards: ${{ inputs.shard-count }} artifact-name: e2e-evidence retention-days: ${{ inputs.artifact-retention-days }} + runner-download-outcome: ${{ steps.download-runner.outcome }} + runner-load-outcome: ${{ steps.load-runner.outcome }} + runner-verification-outcome: ${{ steps.verify-runner.outcome }} + shard-download-outcome: ${{ steps.download.outcome }} + merge-outcome: ${{ steps.merge.outcome }} + report-publication-outcome: ${{ steps.package-report.outcome }} - name: Preserve merge verdict - if: always() && (steps.download.outcome != 'success' || steps.merge.outcome != 'success') + if: always() && steps.publish.outcome != 'success' shell: bash run: | echo "::error::E2E evidence could not be completed successfully" @@ -254,6 +368,7 @@ jobs: pr_comment: name: E2E PR report needs: + - build_runner - merge - e2e_gate if: >- @@ -261,6 +376,8 @@ jobs: github.event.pull_request.head.repo.full_name == github.repository }} runs-on: ubuntu-latest timeout-minutes: 2 + permissions: + pull-requests: write steps: - name: Create or update E2E report continue-on-error: true @@ -275,6 +392,12 @@ jobs: E2E_FAILED: ${{ needs.merge.outputs.failed }} E2E_SKIPPED: ${{ needs.merge.outputs.skipped }} E2E_ARTIFACT_NAME: ${{ needs.merge.outputs.artifact-name }} + E2E_RUNNER_IMAGE_NAME: ${{ needs.build_runner.outputs.image-name }} + E2E_RUNNER_IMAGE_ID: ${{ needs.build_runner.outputs.image-id }} + E2E_RUNNER_IMAGE_ARTIFACT: ${{ needs.build_runner.outputs.artifact-name }} + E2E_REPORT_IMAGE_NAME: ${{ needs.merge.outputs.report-image-name }} + E2E_REPORT_IMAGE_ID: ${{ needs.merge.outputs.report-image-id }} + E2E_REPORT_IMAGE_ARTIFACT: ${{ needs.merge.outputs.report-image-artifact }} E2E_PROJECTS: ${{ inputs.projects }} E2E_LABELS: ${{ inputs.labels }} E2E_LABEL_MATCH: ${{ inputs.label-match }} @@ -309,6 +432,12 @@ jobs: const labels = list(process.env.E2E_LABELS, 'All labels'); const labelMatch = clean(process.env.E2E_LABEL_MATCH, 'all'); const artifactName = clean(process.env.E2E_ARTIFACT_NAME, 'e2e-evidence'); + const runnerImageName = clean(process.env.E2E_RUNNER_IMAGE_NAME, 'unavailable'); + const runnerImageId = clean(process.env.E2E_RUNNER_IMAGE_ID, 'unavailable'); + const runnerImageArtifact = clean(process.env.E2E_RUNNER_IMAGE_ARTIFACT, 'unavailable'); + const reportImageName = clean(process.env.E2E_REPORT_IMAGE_NAME, 'unavailable'); + const reportImageId = clean(process.env.E2E_REPORT_IMAGE_ID, 'unavailable'); + const reportImageArtifact = clean(process.env.E2E_REPORT_IMAGE_ARTIFACT, 'unavailable'); const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; const lines = [ @@ -322,6 +451,8 @@ jobs: `| Projects | ${cell(projects)} |`, `| Labels | ${cell(`${labels} · match ${labelMatch}`)} |`, '| SUT | Docker Compose, isolated per shard |', + `| Runner image | ${cell(`\`${runnerImageName}\` · \`${runnerImageId}\` · \`${runnerImageArtifact}\``)} |`, + `| Report image | ${cell(`\`${reportImageName}\` · \`${reportImageId}\` · \`${reportImageArtifact}\``)} |`, `| Evidence | [Open workflow run](${runUrl}) · \`${cell(artifactName)}\` |`, ]; diff --git a/README.md b/README.md index 4f81e63..7c48367 100644 --- a/README.md +++ b/README.md @@ -7,18 +7,29 @@ This repository proves the contract with a root Docker Compose service. The serv ```text pull request / main / manual | + build runner image once + | + portable runner archive + | +------+------+ | | shard 1 shard 2 - compose up compose up - Playwright Playwright - logs + down logs + down + load runner load runner + compose SUT compose SUT + run tests run tests + evidence evidence | | +------+------+ | merge evidence | + package report image + | + portable report archive + | E2E Gate + | + sticky PR confidence report ``` ## Run the example locally @@ -68,7 +79,7 @@ jobs: shard-count: 2 ``` -Replace the marker with the immutable SHA you adopt. The workflow needs no inherited secrets and does not write pull-request comments. +Replace the marker with the immutable SHA you adopt. The workflow needs no inherited secrets. Pull-request comments use the explicitly granted permission shown above. | Input | Default | Purpose | | --- | --- | --- | @@ -79,13 +90,11 @@ Replace the marker with the immutable SHA you adopt. The workflow needs no inher | `projects` | empty | Project names, one per line; empty selects all | | `labels` | empty | Tags beginning with `@`, one per line; empty selects all | | `label-match` | `all` | Require all listed labels or any listed label | -| `node-version` | `22` | Node.js runtime | -| `pnpm-version` | `10.26.2` | pnpm runtime | | `shard-count` | `2` | Parallel shards, from 1 through 32 | | `artifact-retention-days` | `10` | Evidence retention period | | `comment-on-pr` | `true` | Update one E2E report comment on same-repository pull requests | -The outputs are `verdict`, `artifact-name`, `total`, `passed`, `failed`, and `skipped`. +The workflow exposes the verdict, merged test totals, and the exact names and IDs of its portable images. The image outputs are listed below. ## Select projects and labels @@ -160,7 +169,59 @@ e2e-evidence/ +-- verdict.json ``` -`E2E Gate` fails for a failed or cancelled shard, unhealthy SUT lifecycle, missing evidence, report merge failure, artifact failure, or a non-passing verdict. Evidence publication cannot turn a failed run green. +The same immutable runner image merges the shard blobs; pull-request dependencies are never installed directly on the GitHub host. `E2E Gate` fails for runner transport or identity problems, a failed or cancelled shard, unhealthy SUT lifecycle, missing evidence, report merge failure, report-image publication failure, evidence publication failure, or a non-passing verdict. Evidence publication cannot turn a failed run green. + +## Load the runner and report images + +A healthy run publishes two Docker archives in addition to the regular test evidence. If report packaging or publication fails, the report outputs are empty and the authoritative verdict is an infrastructure error. + +| Artifact | Archive inside it | Purpose | +| --- | --- | --- | +| `e2e-runner-image` | `playwright-e2e-runner.tar.zst` | Exact Playwright runner used by every shard | +| `e2e-report-image` | `playwright-e2e-report.tar.zst` | Merged HTML report served on port 8080 | + +The workflow exposes these values: + +| Output | Meaning | +| --- | --- | +| `runner-image-name` | Commit-tagged runner image name | +| `runner-image-id` | Content-addressed local runner image ID | +| `runner-image-artifact` | Runner archive artifact name | +| `report-image-name` | Run-tagged report image name; empty when unavailable | +| `report-image-id` | Content-addressed local report image ID; empty when unavailable | +| `report-image-artifact` | Report archive artifact name; empty when unavailable | + +After downloading the runner artifact, load it and run the exact test environment again. Replace the image variable with the corresponding workflow output: + +```bash +docker load --input e2e-runner-image/playwright-e2e-runner.tar.zst +mkdir -p e2e-output +docker run --rm --init --ipc=host \ + --read-only \ + --tmpfs /tmp:rw,nosuid,nodev,size=512m \ + --cap-drop=ALL \ + --security-opt=no-new-privileges:true \ + --pids-limit=1024 \ + --user "$(id -u):$(id -g)" \ + --add-host=host.docker.internal:host-gateway \ + -e BASE_URL=http://host.docker.internal:4173 \ + -e HOME=/tmp \ + -v "$PWD/e2e-output:/evidence" \ + "$RUNNER_IMAGE_ID" +``` + +Append normal Playwright arguments, such as `--project=chromium --grep=@smoke`, after the image name. + +Load and serve the merged report in the same way: + +```bash +docker load --input e2e-report-image/playwright-e2e-report.tar.zst +docker run --rm -p 8080:8080 "$REPORT_IMAGE_NAME" +``` + +Open `http://127.0.0.1:8080`. The report image contains the complete merged HTML directory. Retained Playwright traces and other report attachments remain available from the report when the test produced them. + +These images are currently GitHub artifacts, not registry publications. Download and `docker load` the archives before use; artifact retention still applies, and the run-tagged image names are not remotely pullable. For pull requests from the same repository, the workflow creates or updates one sticky `E2E confidence report` comment with the verdict, aggregate totals, active project and label filters, failure reason, and workflow evidence link. Grant `pull-requests: write` in the caller workflow, or set `comment-on-pr: false` when comments are not wanted. Comment publication is non-blocking and does not change the gate verdict. diff --git a/e2e/.dockerignore b/e2e/.dockerignore new file mode 100644 index 0000000..a928c46 --- /dev/null +++ b/e2e/.dockerignore @@ -0,0 +1,81 @@ +# Dependency and package-manager state +node_modules/ +**/node_modules/ +.pnpm-store/ +**/.pnpm-store/ +.npm/ +**/.npm/ +.npmrc +**/.npmrc + +# Generated test evidence and reports +blob-report/ +**/blob-report/ +ctrf/ +**/ctrf/ +playwright-report/ +**/playwright-report/ +test-results/ +**/test-results/ +e2e-evidence/ +**/e2e-evidence/ +downloaded-shards/ +**/downloaded-shards/ +coverage/ +**/coverage/ + +# Build caches and portable image archives +.cache/ +**/.cache/ +.turbo/ +**/.turbo/ +.eslintcache +**/.eslintcache +*.tsbuildinfo +.e2e-runner/ +e2e-report-image/ +runner-image/ +*.tar +*.tar.gz +*.tar.zst +*.tgz +*.zst + +# Version-control and editor metadata +.git/ +**/.git/ +.gitignore +**/.gitignore +.gitattributes +**/.gitattributes +.hg/ +**/.hg/ +.svn/ +**/.svn/ +.idea/ +**/.idea/ +.vscode/ +**/.vscode/ +.DS_Store +**/.DS_Store + +# Local secrets and private keys +.env +**/.env +.env.* +**/.env.* +!.env.example +!**/.env.example +*.pem +*.key +*.p12 +*.pfx +*.jks +id_rsa +id_rsa.* +id_ed25519 +id_ed25519.* + +# Build instructions are not runtime inputs +Dockerfile +.dockerignore diff --git a/e2e/Dockerfile b/e2e/Dockerfile new file mode 100644 index 0000000..0881e8a --- /dev/null +++ b/e2e/Dockerfile @@ -0,0 +1,21 @@ +FROM mcr.microsoft.com/playwright:v1.61.1-noble@sha256:5b8f294aff9041b7191c34a4bab3ac270157a28774d4b0660e9743297b697e48 + +ENV CI=true \ + E2E_RESULTS_DIR=/evidence/test-results \ + PLAYWRIGHT_BLOB_OUTPUT_DIR=/evidence/blob-report \ + CTRF_OUTPUT_FILE=/evidence/ctrf/ctrf-report.json + +RUN corepack enable && corepack prepare pnpm@10.26.2 --activate + +WORKDIR /workspace/e2e + +COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./ +RUN pnpm install --frozen-lockfile + +COPY --chown=pwuser:pwuser . . + +RUN mkdir -p /evidence && chown pwuser:pwuser /evidence + +USER pwuser + +ENTRYPOINT ["node", "./runner-entrypoint.mjs"] diff --git a/e2e/runner-entrypoint.mjs b/e2e/runner-entrypoint.mjs new file mode 100644 index 0000000..16ff077 --- /dev/null +++ b/e2e/runner-entrypoint.mjs @@ -0,0 +1,57 @@ +import { spawn } from "node:child_process"; +import { mkdirSync } from "node:fs"; +import { dirname } from "node:path"; + +const evidenceRoot = "/evidence"; +const environmentDefaults = { + CI: "true", + E2E_RESULTS_DIR: `${evidenceRoot}/test-results`, + PLAYWRIGHT_BLOB_OUTPUT_DIR: `${evidenceRoot}/blob-report`, + CTRF_OUTPUT_FILE: `${evidenceRoot}/ctrf/ctrf-report.json`, +}; + +for (const [name, value] of Object.entries(environmentDefaults)) { + process.env[name] ??= value; +} + +if (!process.env.BASE_URL) { + console.error("BASE_URL is required, for example http://host.docker.internal:4173"); + process.exit(2); +} + +for (const directory of [ + process.env.E2E_RESULTS_DIR, + process.env.PLAYWRIGHT_BLOB_OUTPUT_DIR, + dirname(process.env.CTRF_OUTPUT_FILE), +]) { + mkdirSync(directory, { recursive: true }); +} + +const playwright = spawn( + "./node_modules/.bin/playwright", + ["test", ...process.argv.slice(2)], + { + env: process.env, + stdio: "inherit", + }, +); + +for (const signal of ["SIGINT", "SIGTERM"]) { + process.once(signal, () => { + playwright.kill(signal); + }); +} + +playwright.once("error", (error) => { + console.error(`Unable to start Playwright: ${error.message}`); + process.exit(1); +}); + +playwright.once("exit", (code, signal) => { + if (signal) { + process.kill(process.pid, signal); + return; + } + + process.exit(code ?? 1); +}); From 55e5bf556d316eef0f369a7086981742604c39ef Mon Sep 17 00:00:00 2001 From: omermorad Date: Mon, 28 Sep 2026 00:12:12 +0300 Subject: [PATCH 04/11] feat: configure E2E execution profiles --- .github/actions/build-e2e-runner/action.yml | 44 +++- .../build-e2e-runner/runner-archive.mjs | 129 ++++++++++ .github/actions/plan-e2e/action.yml | 87 +++++++ .github/actions/plan-e2e/plan.rb | 226 ++++++++++++++++++ .github/workflows/e2e.yml | 26 +- .github/workflows/reusable-e2e.yml | 157 ++++++------ README.md | 79 ++++-- e2e/.dockerignore | 3 +- e2e/ci.yml | 39 +++ 9 files changed, 683 insertions(+), 107 deletions(-) create mode 100644 .github/actions/build-e2e-runner/runner-archive.mjs create mode 100644 .github/actions/plan-e2e/action.yml create mode 100644 .github/actions/plan-e2e/plan.rb create mode 100644 e2e/ci.yml diff --git a/.github/actions/build-e2e-runner/action.yml b/.github/actions/build-e2e-runner/action.yml index 5e5de4f..e474162 100644 --- a/.github/actions/build-e2e-runner/action.yml +++ b/.github/actions/build-e2e-runner/action.yml @@ -1,5 +1,5 @@ -name: Build E2E Runner -description: Build the Playwright runner once and save a portable zstd-compressed Docker archive. +name: Resolve E2E Runner +description: Reuse or build the content-addressed Playwright runner archive. inputs: working-directory: @@ -17,6 +17,13 @@ inputs: description: Repository-relative output path for the zstd-compressed Docker image archive. required: false default: .e2e-runner/playwright-e2e-runner.tar.zst + content-hash: + description: SHA-256 of the E2E source tree used as the cache identity. + required: true + reuse-archive: + description: Validate and reuse the archive already restored at archive-path. + required: false + default: "false" outputs: image-name: @@ -28,6 +35,9 @@ outputs: archive-path: description: Absolute path to the portable .tar.zst Docker image archive. value: ${{ steps.build.outputs.archive-path }} + archive-sha256: + description: SHA-256 digest of the compressed Docker archive. + value: ${{ steps.build.outputs.archive-sha256 }} runs: using: composite @@ -40,9 +50,26 @@ runs: E2E_DOCKERFILE: ${{ inputs.dockerfile }} E2E_RUNNER_IMAGE: ${{ inputs.image-name }} E2E_ARCHIVE_PATH: ${{ inputs.archive-path }} + E2E_CONTENT_HASH: ${{ inputs.content-hash }} + E2E_REUSE_ARCHIVE: ${{ inputs.reuse-archive }} run: | set -euo pipefail + if [[ "$E2E_REUSE_ARCHIVE" == "true" ]]; then + node "$GITHUB_ACTION_PATH/runner-archive.mjs" \ + verify \ + "$GITHUB_WORKSPACE" \ + "$E2E_ARCHIVE_PATH" \ + "$E2E_RUNNER_IMAGE" \ + "$E2E_CONTENT_HASH" \ + "$GITHUB_OUTPUT" + exit 0 + fi + if [[ "$E2E_REUSE_ARCHIVE" != "false" ]]; then + echo "reuse-archive must be true or false" >&2 + exit 1 + fi + eval "$(node "$GITHUB_ACTION_PATH/prepare-build.mjs" \ "$GITHUB_WORKSPACE" \ "$E2E_WORKING_DIRECTORY" \ @@ -66,8 +93,11 @@ runs: zstd -T0 -3 --force -o "$E2E_ABSOLUTE_ARCHIVE_PATH" test -s "$E2E_ABSOLUTE_ARCHIVE_PATH" - { - echo "image-name=$E2E_RUNNER_IMAGE" - echo "image-id=$image_id" - echo "archive-path=$E2E_ABSOLUTE_ARCHIVE_PATH" - } >> "$GITHUB_OUTPUT" + node "$GITHUB_ACTION_PATH/runner-archive.mjs" \ + write \ + "$GITHUB_WORKSPACE" \ + "$E2E_ARCHIVE_PATH" \ + "$E2E_RUNNER_IMAGE" \ + "$E2E_CONTENT_HASH" \ + "$GITHUB_OUTPUT" \ + "$image_id" diff --git a/.github/actions/build-e2e-runner/runner-archive.mjs b/.github/actions/build-e2e-runner/runner-archive.mjs new file mode 100644 index 0000000..7d612be --- /dev/null +++ b/.github/actions/build-e2e-runner/runner-archive.mjs @@ -0,0 +1,129 @@ +import { + appendFileSync, + createReadStream, + existsSync, + lstatSync, + readFileSync, + realpathSync, + statSync, + writeFileSync, +} from "node:fs"; +import { createHash } from "node:crypto"; +import { basename, dirname, isAbsolute, relative, resolve, sep } from "node:path"; + +const [ + command, + workspace, + requestedArchivePath, + expectedImageName, + expectedContentHash, + outputFile, + imageId, +] = process.argv.slice(2); + +const fail = (message) => { + throw new Error(message); +}; + +if (!new Set(["write", "verify"]).has(command)) { + fail(`runner-archive command must be write or verify; received ${command}`); +} +if (!/^[a-f0-9]{64}$/.test(expectedContentHash)) { + fail("content-hash must be a lowercase SHA-256 digest"); +} +if (!expectedImageName || /[\s\0]/.test(expectedImageName)) { + fail("image-name must be a non-empty Docker reference without whitespace"); +} +if (!requestedArchivePath || isAbsolute(requestedArchivePath)) { + fail("archive-path must be repository-relative"); +} + +const workspacePath = realpathSync(workspace); +const archivePath = resolve(workspacePath, requestedArchivePath); +if (basename(archivePath) !== "playwright-e2e-runner.tar.zst") { + fail("archive-path must end with playwright-e2e-runner.tar.zst"); +} +const relativeArchive = relative(workspacePath, archivePath); +if ( + relativeArchive === "" || + relativeArchive === ".." || + relativeArchive.startsWith(`..${sep}`) || + isAbsolute(relativeArchive) +) { + fail("archive-path must remain inside GITHUB_WORKSPACE"); +} +if (!existsSync(archivePath) || !lstatSync(archivePath).isFile()) { + fail(`runner archive does not exist: ${requestedArchivePath}`); +} +if (lstatSync(archivePath).isSymbolicLink()) { + fail("runner archive must not be a symbolic link"); +} + +const digestFile = async (path) => { + const hash = createHash("sha256"); + const stream = createReadStream(path); + for await (const chunk of stream) hash.update(chunk); + return hash.digest("hex"); +}; + +const archiveDigest = await digestFile(archivePath); +const archiveSize = statSync(archivePath).size; +const archiveDirectory = realpathSync(dirname(archivePath)); +const relativeArchiveDirectory = relative(workspacePath, archiveDirectory); +if ( + relativeArchiveDirectory === ".." || + relativeArchiveDirectory.startsWith(`..${sep}`) || + isAbsolute(relativeArchiveDirectory) +) { + fail("runner archive directory must remain inside GITHUB_WORKSPACE"); +} +const metadataPath = resolve(archiveDirectory, "metadata.json"); +if (existsSync(metadataPath) && lstatSync(metadataPath).isSymbolicLink()) { + fail("runner metadata must not be a symbolic link"); +} + +let metadata; +if (command === "write") { + if (!/^sha256:[a-f0-9]{64}$/.test(imageId ?? "")) { + fail("image-id must be a content-addressed Docker image ID"); + } + metadata = { + schemaVersion: 1, + contentHash: expectedContentHash, + imageName: expectedImageName, + imageId, + archive: { + file: "playwright-e2e-runner.tar.zst", + sha256: archiveDigest, + size: archiveSize, + }, + }; + writeFileSync(metadataPath, `${JSON.stringify(metadata, null, 2)}\n`, { + mode: 0o600, + }); +} else { + if (!existsSync(metadataPath) || !lstatSync(metadataPath).isFile()) { + fail("restored runner metadata.json is missing"); + } + try { + metadata = JSON.parse(readFileSync(metadataPath, "utf8")); + } catch (error) { + fail(`restored runner metadata is invalid JSON: ${error.message}`); + } + if (metadata.schemaVersion !== 1) fail("unsupported runner metadata schema"); + if (metadata.contentHash !== expectedContentHash) fail("restored runner content hash does not match"); + if (metadata.imageName !== expectedImageName) fail("restored runner image name does not match"); + if (!/^sha256:[a-f0-9]{64}$/.test(metadata.imageId ?? "")) fail("restored runner image ID is invalid"); + if (metadata.archive?.file !== "playwright-e2e-runner.tar.zst") fail("restored runner archive name is invalid"); + if (metadata.archive?.sha256 !== archiveDigest) fail("restored runner archive digest does not match"); + if (metadata.archive?.size !== archiveSize) fail("restored runner archive size does not match"); +} + +for (const [name, value] of [ + ["image-name", metadata.imageName], + ["image-id", metadata.imageId], + ["archive-path", archivePath], + ["archive-sha256", archiveDigest], +]) { + appendFileSync(outputFile, `${name}=${value}\n`); +} diff --git a/.github/actions/plan-e2e/action.yml b/.github/actions/plan-e2e/action.yml new file mode 100644 index 0000000..c47e9b1 --- /dev/null +++ b/.github/actions/plan-e2e/action.yml @@ -0,0 +1,87 @@ +name: Plan E2E +description: Validate the E2E execution manifest and produce one deterministic CI plan. + +inputs: + config-file: + description: Repository-relative E2E execution manifest. + required: false + default: e2e/ci.yml + profile: + description: Named execution profile from the manifest. + required: true + projects: + description: Optional newline-delimited project override. + required: false + default: "" + labels: + description: Optional newline-delimited label override. + required: false + default: "" + label-match: + description: Optional label matching override, either all or any. + required: false + default: "" + +outputs: + working-directory: + description: Repository-relative E2E package and runner context. + value: ${{ steps.plan.outputs.working-directory }} + dockerfile: + description: Dockerfile path relative to working-directory. + value: ${{ steps.plan.outputs.dockerfile }} + playwright-config: + description: Playwright config path relative to working-directory. + value: ${{ steps.plan.outputs.playwright-config }} + compose-file: + description: Docker Compose file relative to the repository root. + value: ${{ steps.plan.outputs.compose-file }} + base-url: + description: Ready SUT URL used by Playwright. + value: ${{ steps.plan.outputs.base-url }} + projects: + description: Effective newline-delimited Playwright projects. + value: ${{ steps.plan.outputs.projects }} + labels: + description: Effective newline-delimited Playwright labels. + value: ${{ steps.plan.outputs.labels }} + label-match: + description: Effective label matching mode. + value: ${{ steps.plan.outputs.label-match }} + shard-count: + description: Number of parallel Playwright shards. + value: ${{ steps.plan.outputs.shard-count }} + matrix: + description: GitHub Actions shard matrix JSON. + value: ${{ steps.plan.outputs.matrix }} + artifact-retention-days: + description: Evidence and image retention period. + value: ${{ steps.plan.outputs.artifact-retention-days }} + runner-content-hash: + description: SHA-256 of committed and non-ignored E2E runner inputs. + value: ${{ steps.plan.outputs.runner-content-hash }} + profile: + description: Effective execution profile name. + value: ${{ steps.plan.outputs.profile }} + +runs: + using: composite + steps: + - id: plan + name: Validate manifest and create plan + shell: bash + env: + E2E_CONFIG_FILE: ${{ inputs.config-file }} + E2E_PROFILE: ${{ inputs.profile }} + E2E_PROJECTS_OVERRIDE: ${{ inputs.projects }} + E2E_LABELS_OVERRIDE: ${{ inputs.labels }} + E2E_LABEL_MATCH_OVERRIDE: ${{ inputs.label-match }} + run: | + set -euo pipefail + ruby "$GITHUB_ACTION_PATH/plan.rb" \ + "$GITHUB_WORKSPACE" \ + "$E2E_CONFIG_FILE" \ + "$E2E_PROFILE" \ + "$E2E_PROJECTS_OVERRIDE" \ + "$E2E_LABELS_OVERRIDE" \ + "$E2E_LABEL_MATCH_OVERRIDE" \ + "$GITHUB_OUTPUT" diff --git a/.github/actions/plan-e2e/plan.rb b/.github/actions/plan-e2e/plan.rb new file mode 100644 index 0000000..40af4ed --- /dev/null +++ b/.github/actions/plan-e2e/plan.rb @@ -0,0 +1,226 @@ +require "digest" +require "json" +require "open3" +require "pathname" +require "securerandom" +require "uri" +require "yaml" + +workspace_argument, config_argument, profile_name, projects_override, + labels_override, label_match_override, output_file = ARGV + +def fail_plan(message) + raise ArgumentError, message +end + +def require_mapping(value, label) + fail_plan("#{label} must be a mapping") unless value.is_a?(Hash) + value +end + +def reject_unknown_keys(mapping, allowed, label) + unknown = mapping.keys.map(&:to_s) - allowed + return if unknown.empty? + + fail_plan("#{label} contains unknown keys: #{unknown.sort.join(', ')}") +end + +def require_relative_path(value, label) + fail_plan("#{label} must be a non-empty string") unless value.is_a?(String) && !value.empty? + + path = Pathname.new(value) + fail_plan("#{label} must be repository-relative") if path.absolute? + + clean = path.cleanpath.to_s + fail_plan("#{label} must remain below its parent directory") if clean == "." || clean == ".." || clean.start_with?("../") + clean +end + +def require_string_list(value, label, allow_empty: true) + fail_plan("#{label} must be a list") unless value.is_a?(Array) + fail_plan("#{label} must not be empty") if !allow_empty && value.empty? + + normalized = value.map do |entry| + fail_plan("#{label} entries must be non-empty strings") unless entry.is_a?(String) && !entry.strip.empty? + fail_plan("#{label} entries must not contain newlines") if entry.match?(/[\r\n]/) + fail_plan("#{label} entries must not contain null bytes") if entry.include?("\0") + entry.strip + end + fail_plan("#{label} entries must be unique") unless normalized.uniq.length == normalized.length + normalized +end + +def parse_override(value) + value.to_s.lines.map(&:strip).reject(&:empty?).uniq +end + +def normalize_profile(name, value) + profile = require_mapping(value, "profiles.#{name}") + reject_unknown_keys(profile, %w[projects labels labelMatch], "profiles.#{name}") + projects = require_string_list(profile["projects"], "profiles.#{name}.projects", allow_empty: false) + labels = require_string_list(profile["labels"], "profiles.#{name}.labels") + labels.each do |label| + fail_plan("labels must begin with @ and contain a name: #{label}") unless label.start_with?("@") && label.length > 1 + end + label_match = profile["labelMatch"] + fail_plan("profiles.#{name}.labelMatch must be all or any") unless %w[all any].include?(label_match) + { "projects" => projects, "labels" => labels, "labelMatch" => label_match } +end + +def append_output(file, name, value) + string = value.to_s + if string.include?("\n") + begin + delimiter = "E2E_#{SecureRandom.hex(16)}" + end while string.lines.map(&:chomp).include?(delimiter) + File.open(file, "a") do |output| + output.puts("#{name}<<#{delimiter}") + output.puts(string) + output.puts(delimiter) + end + else + File.open(file, "a") { |output| output.puts("#{name}=#{string}") } + end +end + +workspace = Pathname.new(workspace_argument).realpath +config_relative = require_relative_path(config_argument, "config-file") +config_path = workspace.join(config_relative) +fail_plan("config-file does not exist: #{config_relative}") unless config_path.file? +fail_plan("config-file must not be a symbolic link") if config_path.symlink? +fail_plan("config-file is too large") if config_path.size > 65_536 + +resolved_config = config_path.realpath +relative_config = resolved_config.relative_path_from(workspace).to_s +fail_plan("config-file must remain inside GITHUB_WORKSPACE") if relative_config == ".." || relative_config.start_with?("../") + +manifest = YAML.safe_load( + resolved_config.read, + permitted_classes: [], + permitted_symbols: [], + aliases: false, +) +manifest = require_mapping(manifest, "manifest") +reject_unknown_keys(manifest, %w[version runner playwright sut execution profiles], "manifest") +fail_plan("manifest version must be 1") unless manifest["version"] == 1 + +runner = require_mapping(manifest["runner"], "runner") +reject_unknown_keys(runner, %w[dockerfile], "runner") +dockerfile = require_relative_path(runner["dockerfile"], "runner.dockerfile") + +playwright = require_mapping(manifest["playwright"], "playwright") +reject_unknown_keys(playwright, %w[config], "playwright") +playwright_config = require_relative_path(playwright["config"], "playwright.config") + +sut = require_mapping(manifest["sut"], "sut") +reject_unknown_keys(sut, %w[composeFile baseUrl], "sut") +compose_file = require_relative_path(sut["composeFile"], "sut.composeFile") +begin + base_url = URI(sut["baseUrl"].to_s) +rescue URI::InvalidURIError + fail_plan("sut.baseUrl must be an HTTP or HTTPS URL") +end +fail_plan("sut.baseUrl must be an HTTP or HTTPS URL") unless %w[http https].include?(base_url.scheme) && base_url.host +fail_plan("sut.baseUrl must not contain credentials or a fragment") if base_url.userinfo || base_url.fragment + +execution = require_mapping(manifest["execution"], "execution") +reject_unknown_keys(execution, %w[shards artifactRetentionDays], "execution") +shard_count = execution["shards"] +fail_plan("execution.shards must be an integer from 1 through 32") unless shard_count.is_a?(Integer) && shard_count.between?(1, 32) +retention_days = execution["artifactRetentionDays"] +fail_plan("execution.artifactRetentionDays must be an integer from 1 through 90") unless retention_days.is_a?(Integer) && retention_days.between?(1, 90) + +profiles = require_mapping(manifest["profiles"], "profiles") +fail_plan("profiles must contain at least one profile") if profiles.empty? +normalized_profiles = profiles.each_with_object({}) do |(name, value), result| + fail_plan("profile names must be non-empty strings") unless name.is_a?(String) && !name.empty? + result[name] = normalize_profile(name, value) +end +fail_plan("profile must be a non-empty string") unless profile_name.is_a?(String) && !profile_name.empty? +unless normalized_profiles.key?(profile_name) + fail_plan("unknown profile #{profile_name}; available profiles: #{normalized_profiles.keys.sort.join(', ')}") +end +profile = normalized_profiles.fetch(profile_name) +projects = profile.fetch("projects") +labels = profile.fetch("labels") +label_match = profile.fetch("labelMatch") + +override_projects = parse_override(projects_override) +override_labels = parse_override(labels_override) +projects = override_projects unless override_projects.empty? +labels = override_labels unless override_labels.empty? +unless label_match_override.to_s.empty? + fail_plan("label-match override must be all or any") unless %w[all any].include?(label_match_override) + label_match = label_match_override +end +labels.each do |label| + fail_plan("labels must begin with @ and contain a name: #{label}") unless label.start_with?("@") && label.length > 1 +end + +working_directory_path = resolved_config.dirname.realpath +working_directory = working_directory_path.relative_path_from(workspace).to_s +fail_plan("config-file must live below the repository root") if working_directory == "." || working_directory.start_with?("../") + +dockerfile_path = working_directory_path.join(dockerfile).cleanpath +playwright_config_path = working_directory_path.join(playwright_config).cleanpath +compose_path = workspace.join(compose_file).cleanpath +[ + [dockerfile_path, "runner.dockerfile"], + [playwright_config_path, "playwright.config"], + [compose_path, "sut.composeFile"], +].each do |path, label| + fail_plan("#{label} does not exist: #{path}") unless path.file? +end +fail_plan("runner.dockerfile must remain inside the E2E directory") unless dockerfile_path.realpath.to_s.start_with?("#{working_directory_path}/") +fail_plan("playwright.config must remain inside the E2E directory") unless playwright_config_path.realpath.to_s.start_with?("#{working_directory_path}/") +fail_plan("sut.composeFile must remain inside the repository") unless compose_path.realpath.to_s.start_with?("#{workspace}/") + +files_output, files_error, files_status = Open3.capture3( + "git", + "-C", + workspace.to_s, + "ls-files", + "-z", + "--cached", + "--others", + "--exclude-standard", + "--", + working_directory, +) +fail_plan("cannot enumerate E2E sources: #{files_error.strip}") unless files_status.success? +files = files_output.split("\0").reject(&:empty?).sort +files.reject! { |relative_file| relative_file == relative_config } +fail_plan("E2E directory contains no source files") if files.empty? + +digest = Digest::SHA256.new +files.each do |relative_file| + path = workspace.join(relative_file) + digest.update(relative_file) + digest.update("\0") + digest.update(path.lstat.mode.to_s(8)) + digest.update("\0") + if path.symlink? + digest.update("symlink\0") + digest.update(path.readlink.to_s) + elsif path.file? + digest.update("file\0") + digest.update(path.binread) + else + fail_plan("unsupported E2E source type: #{relative_file}") + end + digest.update("\0") +end + +append_output(output_file, "working-directory", working_directory) +append_output(output_file, "dockerfile", dockerfile) +append_output(output_file, "playwright-config", playwright_config) +append_output(output_file, "compose-file", compose_file) +append_output(output_file, "base-url", base_url.to_s) +append_output(output_file, "projects", projects.join("\n")) +append_output(output_file, "labels", labels.join("\n")) +append_output(output_file, "label-match", label_match) +append_output(output_file, "shard-count", shard_count) +append_output(output_file, "matrix", { shard: (1..shard_count).to_a }.to_json) +append_output(output_file, "artifact-retention-days", retention_days) +append_output(output_file, "runner-content-hash", digest.hexdigest) +append_output(output_file, "profile", profile_name) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index f9baef3..774c69d 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -7,22 +7,28 @@ on: - main workflow_dispatch: inputs: + profile: + description: Execution profile from e2e/ci.yml. + required: true + default: regression + type: string projects: - description: Playwright project names, one per line; empty runs all projects. + description: Optional project override, one per line. required: false default: "" type: string labels: - description: Playwright tags beginning with @, one per line; empty runs all tags. + description: Optional label override, one per line. required: false default: "" type: string label-match: - description: Require all labels or any label. + description: Optional label matching override. required: false - default: all + default: profile type: choice options: + - profile - all - any @@ -39,10 +45,12 @@ jobs: name: E2E pipeline uses: $/.github/workflows/reusable-e2e.yml with: - working-directory: e2e - compose-file: compose.e2e.yml - base-url: http://127.0.0.1:4173 + config-file: e2e/ci.yml + profile: >- + ${{ github.event_name == 'pull_request' && 'pull-request' || + github.event_name == 'push' && 'main' || inputs.profile }} projects: ${{ github.event_name == 'workflow_dispatch' && inputs.projects || '' }} labels: ${{ github.event_name == 'workflow_dispatch' && inputs.labels || '' }} - label-match: ${{ github.event_name == 'workflow_dispatch' && inputs.label-match || 'all' }} - shard-count: 2 + label-match: >- + ${{ github.event_name == 'workflow_dispatch' && inputs.label-match != 'profile' && + inputs.label-match || '' }} diff --git a/.github/workflows/reusable-e2e.yml b/.github/workflows/reusable-e2e.yml index 937c628..1e7b1f1 100644 --- a/.github/workflows/reusable-e2e.yml +++ b/.github/workflows/reusable-e2e.yml @@ -3,51 +3,31 @@ name: Reusable E2E on: workflow_call: inputs: - working-directory: - description: Directory containing the Playwright package. + config-file: + description: Repository-relative E2E execution manifest. required: false - default: e2e + default: e2e/ci.yml type: string - playwright-config: - description: Playwright configuration path, relative to the working directory. + profile: + description: Named execution profile from the E2E manifest. required: false - default: playwright.config.ts - type: string - compose-file: - description: Docker Compose file for the SUT, relative to the repository root. - required: false - default: compose.e2e.yml - type: string - base-url: - description: Base URL exposed by the Dockerized SUT. - required: false - default: http://127.0.0.1:4173 + default: pull-request type: string projects: - description: Newline-delimited Playwright project names; empty selects all projects. + description: Optional newline-delimited project override; empty uses the profile. required: false default: "" type: string labels: - description: Newline-delimited Playwright tags beginning with @; empty selects all tags. + description: Optional newline-delimited label override; empty uses the profile. required: false default: "" type: string label-match: - description: Label matching mode, either all or any. + description: Optional label matching override, either all or any. required: false - default: all + default: "" type: string - shard-count: - description: Number of Playwright shards to run in parallel (1-32). - required: false - default: 2 - type: number - artifact-retention-days: - description: Number of days to retain shard and merged evidence artifacts. - required: false - default: 10 - type: number comment-on-pr: description: Create or update one E2E report comment on same-repository pull requests. required: false @@ -63,6 +43,15 @@ on: runner-image-artifact: description: Artifact containing the portable E2E runner Docker archive. value: ${{ jobs.build_runner.outputs.artifact-name }} + runner-cache-hit: + description: Whether the runner archive was restored by E2E content hash. + value: ${{ jobs.build_runner.outputs.cache-hit }} + runner-content-hash: + description: SHA-256 of the E2E runtime and build inputs used by the runner. + value: ${{ jobs.plan.outputs.runner-content-hash }} + profile: + description: Effective E2E execution profile. + value: ${{ jobs.plan.outputs.profile }} report-image-name: description: Local image reference for the packaged E2E HTML report. value: ${{ jobs.merge.outputs.report-image-name }} @@ -100,23 +89,34 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 2 outputs: - matrix: ${{ steps.matrix.outputs.matrix }} + working-directory: ${{ steps.plan.outputs.working-directory }} + dockerfile: ${{ steps.plan.outputs.dockerfile }} + playwright-config: ${{ steps.plan.outputs.playwright-config }} + compose-file: ${{ steps.plan.outputs.compose-file }} + base-url: ${{ steps.plan.outputs.base-url }} + projects: ${{ steps.plan.outputs.projects }} + labels: ${{ steps.plan.outputs.labels }} + label-match: ${{ steps.plan.outputs.label-match }} + shard-count: ${{ steps.plan.outputs.shard-count }} + matrix: ${{ steps.plan.outputs.matrix }} + artifact-retention-days: ${{ steps.plan.outputs.artifact-retention-days }} + runner-content-hash: ${{ steps.plan.outputs.runner-content-hash }} + profile: ${{ steps.plan.outputs.profile }} steps: - - id: matrix - name: Validate and create shard matrix - env: - SHARD_COUNT: ${{ inputs.shard-count }} - shell: bash - run: | - matrix="$(node -e ' - const count = Number(process.env.SHARD_COUNT); - if (!Number.isInteger(count) || count < 1 || count > 32) { - console.error("shard-count must be an integer from 1 through 32"); - process.exit(1); - } - process.stdout.write(JSON.stringify({ shard: Array.from({ length: count }, (_, index) => index + 1) })); - ')" - echo "matrix=$matrix" >> "$GITHUB_OUTPUT" + - name: Check out caller repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + + - id: plan + name: Validate E2E execution manifest + uses: $/.github/actions/plan-e2e + with: + config-file: ${{ inputs.config-file }} + profile: ${{ inputs.profile }} + projects: ${{ inputs.projects }} + labels: ${{ inputs.labels }} + label-match: ${{ inputs.label-match }} build_runner: name: Build E2E runner image @@ -127,20 +127,31 @@ jobs: image-name: ${{ steps.build.outputs.image-name }} image-id: ${{ steps.build.outputs.image-id }} artifact-name: ${{ steps.upload-runner.outcome == 'success' && 'e2e-runner-image' || '' }} + cache-hit: ${{ steps.runner-cache.outcome == 'success' && steps.runner-cache.outputs.cache-hit == 'true' }} steps: - name: Check out caller repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: persist-credentials: false + - id: runner-cache + name: Restore E2E runner archive + continue-on-error: true + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: .e2e-runner + key: e2e-runner-v1-${{ runner.os }}-${{ runner.arch }}-${{ needs.plan.outputs.runner-content-hash }} + - id: build - name: Build E2E runner image + name: Resolve E2E runner image uses: $/.github/actions/build-e2e-runner with: - working-directory: ${{ inputs.working-directory }} - dockerfile: Dockerfile - image-name: playwright-e2e-runner:${{ github.sha }} + working-directory: ${{ needs.plan.outputs.working-directory }} + dockerfile: ${{ needs.plan.outputs.dockerfile }} + image-name: playwright-e2e-runner:e2e-${{ needs.plan.outputs.runner-content-hash }} archive-path: .e2e-runner/playwright-e2e-runner.tar.zst + content-hash: ${{ needs.plan.outputs.runner-content-hash }} + reuse-archive: ${{ steps.runner-cache.outcome == 'success' && steps.runner-cache.outputs.cache-hit == 'true' }} - id: upload-runner name: Upload portable E2E runner image @@ -150,10 +161,10 @@ jobs: path: ${{ steps.build.outputs.archive-path }} if-no-files-found: error compression-level: 0 - retention-days: ${{ inputs.artifact-retention-days }} + retention-days: ${{ needs.plan.outputs.artifact-retention-days }} shards: - name: E2E shard ${{ matrix.shard }}/${{ inputs.shard-count }} + name: E2E shard ${{ matrix.shard }}/${{ needs.plan.outputs.shard-count }} needs: - plan - build_runner @@ -187,19 +198,19 @@ jobs: continue-on-error: true uses: $/.github/actions/run-e2e with: - working-directory: ${{ inputs.working-directory }} - playwright-config: ${{ inputs.playwright-config }} - compose-file: ${{ inputs.compose-file }} - base-url: ${{ inputs.base-url }} + working-directory: ${{ needs.plan.outputs.working-directory }} + playwright-config: ${{ needs.plan.outputs.playwright-config }} + compose-file: ${{ needs.plan.outputs.compose-file }} + base-url: ${{ needs.plan.outputs.base-url }} runner-image-name: ${{ needs.build_runner.outputs.image-name }} runner-image-id: ${{ needs.build_runner.outputs.image-id }} runner-download-outcome: ${{ steps.download-runner.outcome }} runner-load-outcome: ${{ steps.load-runner.outcome }} - projects: ${{ inputs.projects }} - labels: ${{ inputs.labels }} - label-match: ${{ inputs.label-match }} + projects: ${{ needs.plan.outputs.projects }} + labels: ${{ needs.plan.outputs.labels }} + label-match: ${{ needs.plan.outputs.label-match }} shard-index: ${{ matrix.shard }} - shard-total: ${{ inputs.shard-count }} + shard-total: ${{ needs.plan.outputs.shard-count }} results-directory: test-results - name: Upload shard evidence @@ -209,7 +220,7 @@ jobs: name: e2e-shard-${{ matrix.shard }} path: ${{ steps.run.outputs.shard-directory }} if-no-files-found: error - retention-days: ${{ inputs.artifact-retention-days }} + retention-days: ${{ needs.plan.outputs.artifact-retention-days }} - name: Preserve shard verdict if: >- @@ -296,7 +307,7 @@ jobs: with: download-directory: downloaded-shards evidence-directory: e2e-evidence - expected-shards: ${{ inputs.shard-count }} + expected-shards: ${{ needs.plan.outputs.shard-count }} runner-image-id: ${{ needs.build_runner.outputs.image-id }} - id: package-report @@ -309,7 +320,7 @@ jobs: image-name: playwright-e2e-report:${{ github.run_id }}-${{ github.run_attempt }} archive-directory: e2e-report-image artifact-name: e2e-report-image - retention-days: ${{ inputs.artifact-retention-days }} + retention-days: ${{ needs.plan.outputs.artifact-retention-days }} - id: publish name: Publish and enforce authoritative evidence @@ -318,9 +329,9 @@ jobs: uses: $/.github/actions/publish-e2e-evidence with: evidence-directory: e2e-evidence - expected-shards: ${{ inputs.shard-count }} + expected-shards: ${{ needs.plan.outputs.shard-count }} artifact-name: e2e-evidence - retention-days: ${{ inputs.artifact-retention-days }} + retention-days: ${{ needs.plan.outputs.artifact-retention-days }} runner-download-outcome: ${{ steps.download-runner.outcome }} runner-load-outcome: ${{ steps.load-runner.outcome }} runner-verification-outcome: ${{ steps.verify-runner.outcome }} @@ -368,6 +379,7 @@ jobs: pr_comment: name: E2E PR report needs: + - plan - build_runner - merge - e2e_gate @@ -395,12 +407,14 @@ jobs: E2E_RUNNER_IMAGE_NAME: ${{ needs.build_runner.outputs.image-name }} E2E_RUNNER_IMAGE_ID: ${{ needs.build_runner.outputs.image-id }} E2E_RUNNER_IMAGE_ARTIFACT: ${{ needs.build_runner.outputs.artifact-name }} + E2E_RUNNER_CACHE_HIT: ${{ needs.build_runner.outputs.cache-hit }} E2E_REPORT_IMAGE_NAME: ${{ needs.merge.outputs.report-image-name }} E2E_REPORT_IMAGE_ID: ${{ needs.merge.outputs.report-image-id }} E2E_REPORT_IMAGE_ARTIFACT: ${{ needs.merge.outputs.report-image-artifact }} - E2E_PROJECTS: ${{ inputs.projects }} - E2E_LABELS: ${{ inputs.labels }} - E2E_LABEL_MATCH: ${{ inputs.label-match }} + E2E_PROFILE: ${{ needs.plan.outputs.profile }} + E2E_PROJECTS: ${{ needs.plan.outputs.projects }} + E2E_LABELS: ${{ needs.plan.outputs.labels }} + E2E_LABEL_MATCH: ${{ needs.plan.outputs.label-match }} with: script: | const marker = ''; @@ -428,6 +442,7 @@ jobs: const passedCount = clean(process.env.E2E_PASSED, 'unknown'); const failedCount = clean(process.env.E2E_FAILED, 'unknown'); const skippedCount = clean(process.env.E2E_SKIPPED, 'unknown'); + const profile = clean(process.env.E2E_PROFILE, 'unknown'); const projects = list(process.env.E2E_PROJECTS, 'All configured projects'); const labels = list(process.env.E2E_LABELS, 'All labels'); const labelMatch = clean(process.env.E2E_LABEL_MATCH, 'all'); @@ -435,6 +450,9 @@ jobs: const runnerImageName = clean(process.env.E2E_RUNNER_IMAGE_NAME, 'unavailable'); const runnerImageId = clean(process.env.E2E_RUNNER_IMAGE_ID, 'unavailable'); const runnerImageArtifact = clean(process.env.E2E_RUNNER_IMAGE_ARTIFACT, 'unavailable'); + const runnerSource = runnerImageName === 'unavailable' + ? 'unavailable' + : clean(process.env.E2E_RUNNER_CACHE_HIT, '') === 'true' ? 'cache hit' : 'built'; const reportImageName = clean(process.env.E2E_REPORT_IMAGE_NAME, 'unavailable'); const reportImageId = clean(process.env.E2E_REPORT_IMAGE_ID, 'unavailable'); const reportImageArtifact = clean(process.env.E2E_REPORT_IMAGE_ARTIFACT, 'unavailable'); @@ -448,10 +466,11 @@ jobs: '| --- | --- |', `| Gate | ${status} |`, `| Tests | ${cell(`${passedCount} passed · ${failedCount} failed · ${skippedCount} skipped · ${total} total`)} |`, + `| Profile | ${cell(`\`${profile}\``)} |`, `| Projects | ${cell(projects)} |`, `| Labels | ${cell(`${labels} · match ${labelMatch}`)} |`, '| SUT | Docker Compose, isolated per shard |', - `| Runner image | ${cell(`\`${runnerImageName}\` · \`${runnerImageId}\` · \`${runnerImageArtifact}\``)} |`, + `| Runner image | ${cell(`\`${runnerImageName}\` · \`${runnerImageId}\` · \`${runnerImageArtifact}\` · ${runnerSource}`)} |`, `| Report image | ${cell(`\`${reportImageName}\` · \`${reportImageId}\` · \`${reportImageArtifact}\``)} |`, `| Evidence | [Open workflow run](${runUrl}) · \`${cell(artifactName)}\` |`, ]; diff --git a/README.md b/README.md index 7c48367..ad68c36 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,8 @@ This repository proves the contract with a root Docker Compose service. The serv ```text pull request / main / manual | - build runner image once + resolve runner by E2E content hash + cache hit / build once | portable runner archive | @@ -73,38 +74,71 @@ jobs: e2e: uses: codotech/playwright-e2e-ci-framework/.github/workflows/reusable-e2e.yml@<40-character-release-commit-sha> with: - working-directory: e2e - compose-file: compose.e2e.yml - base-url: http://127.0.0.1:4173 - shard-count: 2 + config-file: e2e/ci.yml + profile: pull-request ``` Replace the marker with the immutable SHA you adopt. The workflow needs no inherited secrets. Pull-request comments use the explicitly granted permission shown above. | Input | Default | Purpose | | --- | --- | --- | -| `working-directory` | `e2e` | Playwright package location | -| `playwright-config` | `playwright.config.ts` | Config path inside the package | -| `compose-file` | `compose.e2e.yml` | Compose file relative to the repository root | -| `base-url` | `http://127.0.0.1:4173` | URL Playwright uses after the SUT is healthy | -| `projects` | empty | Project names, one per line; empty selects all | -| `labels` | empty | Tags beginning with `@`, one per line; empty selects all | -| `label-match` | `all` | Require all listed labels or any listed label | -| `shard-count` | `2` | Parallel shards, from 1 through 32 | -| `artifact-retention-days` | `10` | Evidence retention period | +| `config-file` | `e2e/ci.yml` | Execution manifest and runner directory | +| `profile` | `pull-request` | Named selection and execution policy | +| `projects` | empty | Optional project override, one per line | +| `labels` | empty | Optional label override, one per line | +| `label-match` | empty | Optional `all` or `any` override | | `comment-on-pr` | `true` | Update one E2E report comment on same-repository pull requests | The workflow exposes the verdict, merged test totals, and the exact names and IDs of its portable images. The image outputs are listed below. +## Configure execution policy + +Keep CI policy in `e2e/ci.yml`: + +```yaml +version: 1 + +runner: + dockerfile: Dockerfile + +playwright: + config: playwright.config.ts + +sut: + composeFile: compose.e2e.yml + baseUrl: http://127.0.0.1:4173 + +execution: + shards: 2 + artifactRetentionDays: 10 + +profiles: + pull-request: + projects: [api, chromium] + labels: ["@smoke"] + labelMatch: any + + main: + projects: [api, chromium] + labels: [] + labelMatch: all +``` + +The manifest defines which configured tests CI selects. `playwright.config.ts` remains the source of truth for what each Playwright project means: browser, device, test matching, dependencies, and runtime behavior. The plan fails before building anything when the manifest has unknown keys, invalid paths, an unknown profile, invalid labels, or unsafe values. + +The runner archive is cached by a SHA-256 of the committed and non-ignored runner inputs under `e2e/`. A service-only change reuses the identical runner archive but still rebuilds the SUT and runs E2E. A test, fixture, dependency, Playwright config, runner, Dockerfile, or Docker-ignore change produces a new runner image. A manifest-only policy change selects a different test plan without rebuilding identical runner bytes. A missing or evicted cache safely falls back to a build. + ## Select projects and labels A [Playwright project](https://playwright.dev/docs/test-projects) is a configured execution variant, such as a browser, device, authentication state, or test group. Each `projects` line becomes an exact `--project` selection. A framework `label` means a [Playwright tag](https://playwright.dev/docs/test-annotations#tag-tests), not a project. Every label must start with `@`; the workflow converts the labels into one `--grep` expression. -For example, a manual run can use multiline values: +For example, a manual run can select `regression` and optionally override it with multiline values: ```text +profile: regression + projects: api chromium @@ -116,9 +150,9 @@ labels: label-match: all ``` -The repository's pull-request and `main` push runs leave both filters empty. Only a manual run supplies the form values above. +Pull requests use the `pull-request` profile, pushes to `main` use the `main` profile, and manual runs choose a profile explicitly. Manual project, label, and match values override only the selected profile fields that are provided. -`all` requires a test to carry both `@smoke` and `@browser`. `any` accepts a test carrying either tag. Blank project or label lines are ignored, and leaving a field empty disables that filter. +`all` requires a test to carry both `@smoke` and `@browser`. `any` accepts a test carrying either tag. Blank lines are ignored. An empty manual field keeps the profile value; `labels: []` in a profile disables label filtering for that profile. Projects and labels are independent. The selected tests are the intersection of both filters, and Playwright shards that result afterward: @@ -184,9 +218,12 @@ The workflow exposes these values: | Output | Meaning | | --- | --- | -| `runner-image-name` | Commit-tagged runner image name | +| `runner-image-name` | E2E-content-hash-tagged runner image name | | `runner-image-id` | Content-addressed local runner image ID | | `runner-image-artifact` | Runner archive artifact name | +| `runner-cache-hit` | Whether the content-addressed runner archive was reused | +| `runner-content-hash` | SHA-256 identity of the E2E runner inputs | +| `profile` | Effective execution profile | | `report-image-name` | Run-tagged report image name; empty when unavailable | | `report-image-id` | Content-addressed local report image ID; empty when unavailable | | `report-image-artifact` | Report archive artifact name; empty when unavailable | @@ -210,7 +247,7 @@ docker run --rm --init --ipc=host \ "$RUNNER_IMAGE_ID" ``` -Append normal Playwright arguments, such as `--project=chromium --grep=@smoke`, after the image name. +Append normal Playwright arguments, such as `--project=chromium --grep=@smoke`, after the image reference. Load and serve the merged report in the same way: @@ -221,8 +258,8 @@ docker run --rm -p 8080:8080 "$REPORT_IMAGE_NAME" Open `http://127.0.0.1:8080`. The report image contains the complete merged HTML directory. Retained Playwright traces and other report attachments remain available from the report when the test produced them. -These images are currently GitHub artifacts, not registry publications. Download and `docker load` the archives before use; artifact retention still applies, and the run-tagged image names are not remotely pullable. +These images are currently GitHub artifacts, not registry publications. Download and `docker load` the archives before use; artifact retention still applies, and the local image names are not remotely pullable. -For pull requests from the same repository, the workflow creates or updates one sticky `E2E confidence report` comment with the verdict, aggregate totals, active project and label filters, failure reason, and workflow evidence link. Grant `pull-requests: write` in the caller workflow, or set `comment-on-pr: false` when comments are not wanted. Comment publication is non-blocking and does not change the gate verdict. +For pull requests from the same repository, the workflow creates or updates one sticky `E2E confidence report` comment with the verdict, selected profile, aggregate totals, active project and label filters, image identities, failure reason, and workflow evidence link. Grant `pull-requests: write` in the caller workflow, or set `comment-on-pr: false` when comments are not wanted. Comment publication is non-blocking and does not change the gate verdict. The framework targets GitHub.com and uses `$/` references so the reusable workflow and its composite actions come from the same commit. That syntax is not available on GitHub Enterprise Server. diff --git a/e2e/.dockerignore b/e2e/.dockerignore index a928c46..c0ca984 100644 --- a/e2e/.dockerignore +++ b/e2e/.dockerignore @@ -76,6 +76,7 @@ id_rsa.* id_ed25519 id_ed25519.* -# Build instructions are not runtime inputs +# CI and build instructions are not runtime inputs +ci.yml Dockerfile .dockerignore diff --git a/e2e/ci.yml b/e2e/ci.yml new file mode 100644 index 0000000..3fba989 --- /dev/null +++ b/e2e/ci.yml @@ -0,0 +1,39 @@ +version: 1 + +runner: + dockerfile: Dockerfile + +playwright: + config: playwright.config.ts + +sut: + composeFile: compose.e2e.yml + baseUrl: http://127.0.0.1:4173 + +execution: + shards: 2 + artifactRetentionDays: 10 + +profiles: + pull-request: + projects: + - api + - chromium + labels: + - "@smoke" + labelMatch: any + + main: + projects: + - api + - chromium + labels: [] + labelMatch: all + + regression: + projects: + - api + - chromium + labels: + - "@regression" + labelMatch: any From da5b30dfef9665c9054329a196f613c4ca43b7b5 Mon Sep 17 00:00:00 2001 From: omermorad Date: Mon, 28 Sep 2026 00:20:13 +0300 Subject: [PATCH 05/11] refactor: separate runner build from cache reuse --- .github/actions/build-e2e-runner/action.yml | 25 +---- .../build-e2e-runner/runner-archive.mjs | 57 +++------- .github/actions/plan-e2e/plan.rb | 4 + .../actions/validate-e2e-runner/action.yml | 47 ++++++++ .../validate-e2e-runner/validate-archive.mjs | 104 ++++++++++++++++++ .github/workflows/reusable-e2e.yml | 26 +++-- 6 files changed, 193 insertions(+), 70 deletions(-) create mode 100644 .github/actions/validate-e2e-runner/action.yml create mode 100644 .github/actions/validate-e2e-runner/validate-archive.mjs diff --git a/.github/actions/build-e2e-runner/action.yml b/.github/actions/build-e2e-runner/action.yml index e474162..c094ec2 100644 --- a/.github/actions/build-e2e-runner/action.yml +++ b/.github/actions/build-e2e-runner/action.yml @@ -1,5 +1,5 @@ -name: Resolve E2E Runner -description: Reuse or build the content-addressed Playwright runner archive. +name: Build E2E Runner +description: Build and archive a content-addressed Playwright runner image. inputs: working-directory: @@ -20,10 +20,6 @@ inputs: content-hash: description: SHA-256 of the E2E source tree used as the cache identity. required: true - reuse-archive: - description: Validate and reuse the archive already restored at archive-path. - required: false - default: "false" outputs: image-name: @@ -51,25 +47,9 @@ runs: E2E_RUNNER_IMAGE: ${{ inputs.image-name }} E2E_ARCHIVE_PATH: ${{ inputs.archive-path }} E2E_CONTENT_HASH: ${{ inputs.content-hash }} - E2E_REUSE_ARCHIVE: ${{ inputs.reuse-archive }} run: | set -euo pipefail - if [[ "$E2E_REUSE_ARCHIVE" == "true" ]]; then - node "$GITHUB_ACTION_PATH/runner-archive.mjs" \ - verify \ - "$GITHUB_WORKSPACE" \ - "$E2E_ARCHIVE_PATH" \ - "$E2E_RUNNER_IMAGE" \ - "$E2E_CONTENT_HASH" \ - "$GITHUB_OUTPUT" - exit 0 - fi - if [[ "$E2E_REUSE_ARCHIVE" != "false" ]]; then - echo "reuse-archive must be true or false" >&2 - exit 1 - fi - eval "$(node "$GITHUB_ACTION_PATH/prepare-build.mjs" \ "$GITHUB_WORKSPACE" \ "$E2E_WORKING_DIRECTORY" \ @@ -94,7 +74,6 @@ runs: test -s "$E2E_ABSOLUTE_ARCHIVE_PATH" node "$GITHUB_ACTION_PATH/runner-archive.mjs" \ - write \ "$GITHUB_WORKSPACE" \ "$E2E_ARCHIVE_PATH" \ "$E2E_RUNNER_IMAGE" \ diff --git a/.github/actions/build-e2e-runner/runner-archive.mjs b/.github/actions/build-e2e-runner/runner-archive.mjs index 7d612be..c63334e 100644 --- a/.github/actions/build-e2e-runner/runner-archive.mjs +++ b/.github/actions/build-e2e-runner/runner-archive.mjs @@ -3,7 +3,6 @@ import { createReadStream, existsSync, lstatSync, - readFileSync, realpathSync, statSync, writeFileSync, @@ -12,7 +11,6 @@ import { createHash } from "node:crypto"; import { basename, dirname, isAbsolute, relative, resolve, sep } from "node:path"; const [ - command, workspace, requestedArchivePath, expectedImageName, @@ -25,9 +23,6 @@ const fail = (message) => { throw new Error(message); }; -if (!new Set(["write", "verify"]).has(command)) { - fail(`runner-archive command must be write or verify; received ${command}`); -} if (!/^[a-f0-9]{64}$/.test(expectedContentHash)) { fail("content-hash must be a lowercase SHA-256 digest"); } @@ -82,43 +77,25 @@ if (existsSync(metadataPath) && lstatSync(metadataPath).isSymbolicLink()) { fail("runner metadata must not be a symbolic link"); } -let metadata; -if (command === "write") { - if (!/^sha256:[a-f0-9]{64}$/.test(imageId ?? "")) { - fail("image-id must be a content-addressed Docker image ID"); - } - metadata = { - schemaVersion: 1, - contentHash: expectedContentHash, - imageName: expectedImageName, - imageId, - archive: { - file: "playwright-e2e-runner.tar.zst", - sha256: archiveDigest, - size: archiveSize, - }, - }; - writeFileSync(metadataPath, `${JSON.stringify(metadata, null, 2)}\n`, { - mode: 0o600, - }); -} else { - if (!existsSync(metadataPath) || !lstatSync(metadataPath).isFile()) { - fail("restored runner metadata.json is missing"); - } - try { - metadata = JSON.parse(readFileSync(metadataPath, "utf8")); - } catch (error) { - fail(`restored runner metadata is invalid JSON: ${error.message}`); - } - if (metadata.schemaVersion !== 1) fail("unsupported runner metadata schema"); - if (metadata.contentHash !== expectedContentHash) fail("restored runner content hash does not match"); - if (metadata.imageName !== expectedImageName) fail("restored runner image name does not match"); - if (!/^sha256:[a-f0-9]{64}$/.test(metadata.imageId ?? "")) fail("restored runner image ID is invalid"); - if (metadata.archive?.file !== "playwright-e2e-runner.tar.zst") fail("restored runner archive name is invalid"); - if (metadata.archive?.sha256 !== archiveDigest) fail("restored runner archive digest does not match"); - if (metadata.archive?.size !== archiveSize) fail("restored runner archive size does not match"); +if (!/^sha256:[a-f0-9]{64}$/.test(imageId ?? "")) { + fail("image-id must be a content-addressed Docker image ID"); } +const metadata = { + schemaVersion: 1, + contentHash: expectedContentHash, + imageName: expectedImageName, + imageId, + archive: { + file: "playwright-e2e-runner.tar.zst", + sha256: archiveDigest, + size: archiveSize, + }, +}; +writeFileSync(metadataPath, `${JSON.stringify(metadata, null, 2)}\n`, { + mode: 0o600, +}); + for (const [name, value] of [ ["image-name", metadata.imageName], ["image-id", metadata.imageId], diff --git a/.github/actions/plan-e2e/plan.rb b/.github/actions/plan-e2e/plan.rb index 40af4ed..39210f2 100644 --- a/.github/actions/plan-e2e/plan.rb +++ b/.github/actions/plan-e2e/plan.rb @@ -193,6 +193,10 @@ def append_output(file, name, value) fail_plan("E2E directory contains no source files") if files.empty? digest = Digest::SHA256.new +digest.update("e2e-runner-content-v1\0") +digest.update("runner.dockerfile\0") +digest.update(dockerfile) +digest.update("\0") files.each do |relative_file| path = workspace.join(relative_file) digest.update(relative_file) diff --git a/.github/actions/validate-e2e-runner/action.yml b/.github/actions/validate-e2e-runner/action.yml new file mode 100644 index 0000000..cc178f9 --- /dev/null +++ b/.github/actions/validate-e2e-runner/action.yml @@ -0,0 +1,47 @@ +name: Validate E2E Runner Archive +description: Validate a cached runner archive and expose its immutable identity. + +inputs: + archive-path: + description: Repository-relative cached Docker archive path. + required: false + default: .e2e-runner/playwright-e2e-runner.tar.zst + image-name: + description: Expected content-addressed local image reference. + required: true + content-hash: + description: Expected SHA-256 identity of the E2E runner inputs. + required: true + +outputs: + image-name: + description: Local Docker image reference stored in the archive. + value: ${{ steps.validate.outputs.image-name }} + image-id: + description: Content-addressed Docker image ID stored in metadata. + value: ${{ steps.validate.outputs.image-id }} + archive-path: + description: Absolute path to the validated archive. + value: ${{ steps.validate.outputs.archive-path }} + archive-sha256: + description: Verified SHA-256 digest of the compressed archive. + value: ${{ steps.validate.outputs.archive-sha256 }} + +runs: + using: composite + steps: + - id: validate + name: Validate cached runner archive + shell: bash + env: + E2E_ARCHIVE_PATH: ${{ inputs.archive-path }} + E2E_RUNNER_IMAGE: ${{ inputs.image-name }} + E2E_CONTENT_HASH: ${{ inputs.content-hash }} + run: | + set -euo pipefail + node "$GITHUB_ACTION_PATH/validate-archive.mjs" \ + "$GITHUB_WORKSPACE" \ + "$E2E_ARCHIVE_PATH" \ + "$E2E_RUNNER_IMAGE" \ + "$E2E_CONTENT_HASH" \ + "$GITHUB_OUTPUT" diff --git a/.github/actions/validate-e2e-runner/validate-archive.mjs b/.github/actions/validate-e2e-runner/validate-archive.mjs new file mode 100644 index 0000000..da7ead8 --- /dev/null +++ b/.github/actions/validate-e2e-runner/validate-archive.mjs @@ -0,0 +1,104 @@ +import { + appendFileSync, + createReadStream, + existsSync, + lstatSync, + readFileSync, + realpathSync, + statSync, +} from "node:fs"; +import { createHash } from "node:crypto"; +import { basename, dirname, isAbsolute, relative, resolve, sep } from "node:path"; + +const [ + workspace, + requestedArchivePath, + expectedImageName, + expectedContentHash, + outputFile, +] = process.argv.slice(2); + +const fail = (message) => { + throw new Error(message); +}; + +if (!/^[a-f0-9]{64}$/.test(expectedContentHash)) { + fail("content-hash must be a lowercase SHA-256 digest"); +} +if (!expectedImageName || /[\s\0]/.test(expectedImageName)) { + fail("image-name must be a non-empty Docker reference without whitespace"); +} +if (!requestedArchivePath || isAbsolute(requestedArchivePath)) { + fail("archive-path must be repository-relative"); +} + +const workspacePath = realpathSync(workspace); +const archivePath = resolve(workspacePath, requestedArchivePath); +if (basename(archivePath) !== "playwright-e2e-runner.tar.zst") { + fail("archive-path must end with playwright-e2e-runner.tar.zst"); +} +const relativeArchive = relative(workspacePath, archivePath); +if ( + relativeArchive === "" || + relativeArchive === ".." || + relativeArchive.startsWith(`..${sep}`) || + isAbsolute(relativeArchive) +) { + fail("archive-path must remain inside GITHUB_WORKSPACE"); +} +if (!existsSync(archivePath) || !lstatSync(archivePath).isFile()) { + fail(`runner archive does not exist: ${requestedArchivePath}`); +} +if (lstatSync(archivePath).isSymbolicLink()) { + fail("runner archive must not be a symbolic link"); +} + +const archiveDirectory = realpathSync(dirname(archivePath)); +const relativeArchiveDirectory = relative(workspacePath, archiveDirectory); +if ( + relativeArchiveDirectory === ".." || + relativeArchiveDirectory.startsWith(`..${sep}`) || + isAbsolute(relativeArchiveDirectory) +) { + fail("runner archive directory must remain inside GITHUB_WORKSPACE"); +} +const metadataPath = resolve(archiveDirectory, "metadata.json"); +if (!existsSync(metadataPath) || !lstatSync(metadataPath).isFile()) { + fail("cached runner metadata.json is missing"); +} +if (lstatSync(metadataPath).isSymbolicLink()) { + fail("cached runner metadata must not be a symbolic link"); +} + +const digestFile = async (path) => { + const hash = createHash("sha256"); + const stream = createReadStream(path); + for await (const chunk of stream) hash.update(chunk); + return hash.digest("hex"); +}; + +let metadata; +try { + metadata = JSON.parse(readFileSync(metadataPath, "utf8")); +} catch (error) { + fail(`cached runner metadata is invalid JSON: ${error.message}`); +} + +const archiveDigest = await digestFile(archivePath); +const archiveSize = statSync(archivePath).size; +if (metadata.schemaVersion !== 1) fail("unsupported runner metadata schema"); +if (metadata.contentHash !== expectedContentHash) fail("cached runner content hash does not match"); +if (metadata.imageName !== expectedImageName) fail("cached runner image name does not match"); +if (!/^sha256:[a-f0-9]{64}$/.test(metadata.imageId ?? "")) fail("cached runner image ID is invalid"); +if (metadata.archive?.file !== "playwright-e2e-runner.tar.zst") fail("cached runner archive name is invalid"); +if (metadata.archive?.sha256 !== archiveDigest) fail("cached runner archive digest does not match"); +if (metadata.archive?.size !== archiveSize) fail("cached runner archive size does not match"); + +for (const [name, value] of [ + ["image-name", metadata.imageName], + ["image-id", metadata.imageId], + ["archive-path", archivePath], + ["archive-sha256", archiveDigest], +]) { + appendFileSync(outputFile, `${name}=${value}\n`); +} diff --git a/.github/workflows/reusable-e2e.yml b/.github/workflows/reusable-e2e.yml index 1e7b1f1..f09e0cb 100644 --- a/.github/workflows/reusable-e2e.yml +++ b/.github/workflows/reusable-e2e.yml @@ -119,15 +119,18 @@ jobs: label-match: ${{ inputs.label-match }} build_runner: - name: Build E2E runner image + name: Resolve E2E runner image needs: plan runs-on: ubuntu-latest timeout-minutes: 30 + concurrency: + group: e2e-runner-${{ needs.plan.outputs.runner-content-hash }} + cancel-in-progress: false outputs: - image-name: ${{ steps.build.outputs.image-name }} - image-id: ${{ steps.build.outputs.image-id }} + image-name: ${{ steps.reuse.outputs.image-name || steps.build.outputs.image-name }} + image-id: ${{ steps.reuse.outputs.image-id || steps.build.outputs.image-id }} artifact-name: ${{ steps.upload-runner.outcome == 'success' && 'e2e-runner-image' || '' }} - cache-hit: ${{ steps.runner-cache.outcome == 'success' && steps.runner-cache.outputs.cache-hit == 'true' }} + cache-hit: ${{ steps.reuse.outcome == 'success' }} steps: - name: Check out caller repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -142,8 +145,18 @@ jobs: path: .e2e-runner key: e2e-runner-v1-${{ runner.os }}-${{ runner.arch }}-${{ needs.plan.outputs.runner-content-hash }} + - id: reuse + name: Validate cached E2E runner image + if: steps.runner-cache.outcome == 'success' && steps.runner-cache.outputs.cache-hit == 'true' + uses: $/.github/actions/validate-e2e-runner + with: + archive-path: .e2e-runner/playwright-e2e-runner.tar.zst + image-name: playwright-e2e-runner:e2e-${{ needs.plan.outputs.runner-content-hash }} + content-hash: ${{ needs.plan.outputs.runner-content-hash }} + - id: build - name: Resolve E2E runner image + name: Build E2E runner image + if: steps.runner-cache.outcome != 'success' || steps.runner-cache.outputs.cache-hit != 'true' uses: $/.github/actions/build-e2e-runner with: working-directory: ${{ needs.plan.outputs.working-directory }} @@ -151,14 +164,13 @@ jobs: image-name: playwright-e2e-runner:e2e-${{ needs.plan.outputs.runner-content-hash }} archive-path: .e2e-runner/playwright-e2e-runner.tar.zst content-hash: ${{ needs.plan.outputs.runner-content-hash }} - reuse-archive: ${{ steps.runner-cache.outcome == 'success' && steps.runner-cache.outputs.cache-hit == 'true' }} - id: upload-runner name: Upload portable E2E runner image uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: e2e-runner-image - path: ${{ steps.build.outputs.archive-path }} + path: ${{ steps.reuse.outputs.archive-path || steps.build.outputs.archive-path }} if-no-files-found: error compression-level: 0 retention-days: ${{ needs.plan.outputs.artifact-retention-days }} From 1aa752c9ae23675b346b97f88b9651d5014b582e Mon Sep 17 00:00:00 2001 From: omermorad Date: Mon, 28 Sep 2026 00:24:26 +0300 Subject: [PATCH 06/11] docs: define smart runner cache contract --- .../build-e2e-runner/prepare-build.mjs | 15 +++++--- .github/actions/plan-e2e/action.yml | 6 --- .github/actions/plan-e2e/plan.rb | 14 +++---- .github/workflows/e2e.yml | 1 - .github/workflows/reusable-e2e.yml | 13 +++---- README.md | 37 ++++++++++++++++--- 6 files changed, 53 insertions(+), 33 deletions(-) diff --git a/.github/actions/build-e2e-runner/prepare-build.mjs b/.github/actions/build-e2e-runner/prepare-build.mjs index a2a5439..a091e13 100644 --- a/.github/actions/build-e2e-runner/prepare-build.mjs +++ b/.github/actions/build-e2e-runner/prepare-build.mjs @@ -5,7 +5,7 @@ import { realpathSync, rmSync, } from "node:fs"; -import { dirname, isAbsolute, relative, resolve, sep } from "node:path"; +import { basename, dirname, isAbsolute, relative, resolve, sep } from "node:path"; const [ workspace, @@ -54,6 +54,9 @@ if (dockerfileRelativePath.startsWith("..") || isAbsolute(dockerfileRelativePath } const archivePath = resolve(workspacePath, requestedArchivePath); +if (basename(archivePath) !== "playwright-e2e-runner.tar.zst") { + throw new Error("archive-path must end with playwright-e2e-runner.tar.zst"); +} const archiveRelativePath = relative(workspacePath, archivePath); if (archiveRelativePath.startsWith("..") || isAbsolute(archiveRelativePath)) { throw new Error("archive-path must remain inside GITHUB_WORKSPACE"); @@ -77,11 +80,13 @@ if ( ) { throw new Error("archive-path must not traverse a symlink outside GITHUB_WORKSPACE"); } -if (existsSync(archivePath) && lstatSync(archivePath).isSymbolicLink()) { - throw new Error("archive-path must not be a symbolic link"); -} mkdirSync(dirname(archivePath), { recursive: true }); -rmSync(archivePath, { force: true }); +for (const outputPath of [archivePath, resolve(dirname(archivePath), "metadata.json")]) { + if (existsSync(outputPath) && lstatSync(outputPath).isDirectory()) { + throw new Error(`runner output path must not be a directory: ${outputPath}`); + } + rmSync(outputPath, { force: true }); +} const quoteForShell = (value) => `'${value.replaceAll("'", `'"'"'`)}'`; for (const [name, value] of Object.entries({ diff --git a/.github/actions/plan-e2e/action.yml b/.github/actions/plan-e2e/action.yml index c47e9b1..79a5419 100644 --- a/.github/actions/plan-e2e/action.yml +++ b/.github/actions/plan-e2e/action.yml @@ -2,10 +2,6 @@ name: Plan E2E description: Validate the E2E execution manifest and produce one deterministic CI plan. inputs: - config-file: - description: Repository-relative E2E execution manifest. - required: false - default: e2e/ci.yml profile: description: Named execution profile from the manifest. required: true @@ -70,7 +66,6 @@ runs: name: Validate manifest and create plan shell: bash env: - E2E_CONFIG_FILE: ${{ inputs.config-file }} E2E_PROFILE: ${{ inputs.profile }} E2E_PROJECTS_OVERRIDE: ${{ inputs.projects }} E2E_LABELS_OVERRIDE: ${{ inputs.labels }} @@ -79,7 +74,6 @@ runs: set -euo pipefail ruby "$GITHUB_ACTION_PATH/plan.rb" \ "$GITHUB_WORKSPACE" \ - "$E2E_CONFIG_FILE" \ "$E2E_PROFILE" \ "$E2E_PROJECTS_OVERRIDE" \ "$E2E_LABELS_OVERRIDE" \ diff --git a/.github/actions/plan-e2e/plan.rb b/.github/actions/plan-e2e/plan.rb index 39210f2..d10b4e3 100644 --- a/.github/actions/plan-e2e/plan.rb +++ b/.github/actions/plan-e2e/plan.rb @@ -6,7 +6,7 @@ require "uri" require "yaml" -workspace_argument, config_argument, profile_name, projects_override, +workspace_argument, profile_name, projects_override, labels_override, label_match_override, output_file = ARGV def fail_plan(message) @@ -84,15 +84,15 @@ def append_output(file, name, value) end workspace = Pathname.new(workspace_argument).realpath -config_relative = require_relative_path(config_argument, "config-file") +config_relative = "e2e/ci.yml" config_path = workspace.join(config_relative) -fail_plan("config-file does not exist: #{config_relative}") unless config_path.file? -fail_plan("config-file must not be a symbolic link") if config_path.symlink? -fail_plan("config-file is too large") if config_path.size > 65_536 +fail_plan("e2e/ci.yml does not exist") unless config_path.file? +fail_plan("e2e/ci.yml must not be a symbolic link") if config_path.symlink? +fail_plan("e2e/ci.yml is too large") if config_path.size > 65_536 resolved_config = config_path.realpath relative_config = resolved_config.relative_path_from(workspace).to_s -fail_plan("config-file must remain inside GITHUB_WORKSPACE") if relative_config == ".." || relative_config.start_with?("../") +fail_plan("e2e/ci.yml must remain inside GITHUB_WORKSPACE") if relative_config == ".." || relative_config.start_with?("../") manifest = YAML.safe_load( resolved_config.read, @@ -159,7 +159,7 @@ def append_output(file, name, value) working_directory_path = resolved_config.dirname.realpath working_directory = working_directory_path.relative_path_from(workspace).to_s -fail_plan("config-file must live below the repository root") if working_directory == "." || working_directory.start_with?("../") +fail_plan("e2e/ci.yml must live below the repository root") if working_directory == "." || working_directory.start_with?("../") dockerfile_path = working_directory_path.join(dockerfile).cleanpath playwright_config_path = working_directory_path.join(playwright_config).cleanpath diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 774c69d..4fa0887 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -45,7 +45,6 @@ jobs: name: E2E pipeline uses: $/.github/workflows/reusable-e2e.yml with: - config-file: e2e/ci.yml profile: >- ${{ github.event_name == 'pull_request' && 'pull-request' || github.event_name == 'push' && 'main' || inputs.profile }} diff --git a/.github/workflows/reusable-e2e.yml b/.github/workflows/reusable-e2e.yml index f09e0cb..021f0ea 100644 --- a/.github/workflows/reusable-e2e.yml +++ b/.github/workflows/reusable-e2e.yml @@ -3,11 +3,6 @@ name: Reusable E2E on: workflow_call: inputs: - config-file: - description: Repository-relative E2E execution manifest. - required: false - default: e2e/ci.yml - type: string profile: description: Named execution profile from the E2E manifest. required: false @@ -112,7 +107,6 @@ jobs: name: Validate E2E execution manifest uses: $/.github/actions/plan-e2e with: - config-file: ${{ inputs.config-file }} profile: ${{ inputs.profile }} projects: ${{ inputs.projects }} labels: ${{ inputs.labels }} @@ -124,7 +118,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 concurrency: - group: e2e-runner-${{ needs.plan.outputs.runner-content-hash }} + group: e2e-runner-${{ github.event.pull_request.number || github.ref }}-${{ needs.plan.outputs.runner-content-hash }} cancel-in-progress: false outputs: image-name: ${{ steps.reuse.outputs.image-name || steps.build.outputs.image-name }} @@ -148,6 +142,7 @@ jobs: - id: reuse name: Validate cached E2E runner image if: steps.runner-cache.outcome == 'success' && steps.runner-cache.outputs.cache-hit == 'true' + continue-on-error: true uses: $/.github/actions/validate-e2e-runner with: archive-path: .e2e-runner/playwright-e2e-runner.tar.zst @@ -156,7 +151,9 @@ jobs: - id: build name: Build E2E runner image - if: steps.runner-cache.outcome != 'success' || steps.runner-cache.outputs.cache-hit != 'true' + if: >- + always() && !cancelled() && steps.runner-cache.outcome != 'skipped' && + steps.reuse.outcome != 'success' uses: $/.github/actions/build-e2e-runner with: working-directory: ${{ needs.plan.outputs.working-directory }} diff --git a/README.md b/README.md index ad68c36..85c0c36 100644 --- a/README.md +++ b/README.md @@ -7,10 +7,17 @@ This repository proves the contract with a root Docker Compose service. The serv ```text pull request / main / manual | - resolve runner by E2E content hash - cache hit / build once + plan + hash runner inputs | - portable runner archive + serialize identical hashes + | + +------+------+ + | | + validated cache cache miss + | pure runner build + +------+------+ + | + portable runner archive | +------+------+ | | @@ -74,7 +81,6 @@ jobs: e2e: uses: codotech/playwright-e2e-ci-framework/.github/workflows/reusable-e2e.yml@<40-character-release-commit-sha> with: - config-file: e2e/ci.yml profile: pull-request ``` @@ -82,7 +88,6 @@ Replace the marker with the immutable SHA you adopt. The workflow needs no inher | Input | Default | Purpose | | --- | --- | --- | -| `config-file` | `e2e/ci.yml` | Execution manifest and runner directory | | `profile` | `pull-request` | Named selection and execution policy | | `projects` | empty | Optional project override, one per line | | `labels` | empty | Optional label override, one per line | @@ -95,6 +100,8 @@ The workflow exposes the verdict, merged test totals, and the exact names and ID Keep CI policy in `e2e/ci.yml`: +The manifest path is intentionally fixed to `e2e/ci.yml`, matching the runner build context exclusion and keeping execution-only policy outside the runner image. + ```yaml version: 1 @@ -126,7 +133,25 @@ profiles: The manifest defines which configured tests CI selects. `playwright.config.ts` remains the source of truth for what each Playwright project means: browser, device, test matching, dependencies, and runtime behavior. The plan fails before building anything when the manifest has unknown keys, invalid paths, an unknown profile, invalid labels, or unsafe values. -The runner archive is cached by a SHA-256 of the committed and non-ignored runner inputs under `e2e/`. A service-only change reuses the identical runner archive but still rebuilds the SUT and runs E2E. A test, fixture, dependency, Playwright config, runner, Dockerfile, or Docker-ignore change produces a new runner image. A manifest-only policy change selects a different test plan without rebuilding identical runner bytes. A missing or evicted cache safely falls back to a build. +## Reuse identical runner images + +The runner cache key is: + +```text +e2e-runner-v1--- +``` + +The content hash covers the normalized `runner.dockerfile` selection plus every Git-tracked or non-ignored untracked file under the E2E directory, except the execution manifest itself. Paths, file types, Unix modes, file bytes, and symlink targets contribute to the hash. + +The identity is deliberately conservative around runner inputs: + +- A test, fixture, dependency, Playwright configuration file, runner entrypoint, Dockerfile, Docker-ignore rule, or selected runner Dockerfile change creates a new identity. +- Profiles, project and label selection, Playwright config selection, shard count, retention, and SUT policy remain outside the runner identity. They can change what runs without rebuilding identical image bytes; the selected config file's contents are still covered by the E2E tree hash. +- A service-only change reuses the runner but still rebuilds the SUT and runs the E2E suite. + +The workflow owns cache reuse. On an exact cache hit, a separate validation action checks the metadata content hash, image reference and ID, archive name, byte size, and SHA-256 before reuse. `runner-cache-hit` is `true` only after that validation succeeds. A missing, evicted, unavailable, or invalid cache runs the build action, which removes stale output files, performs a fresh Docker build, and writes a new archive plus metadata; it contains no cache-reuse branch. Invalid cached bytes are never trusted. + +Runner resolution uses the concurrency group `e2e-runner--` with cancellation disabled. Identical runner requests in the same change stream resolve one at a time, while unrelated pull requests cannot cancel each other's pending runner job. Different content hashes resolve independently. Every run still uploads its resolved archive as `e2e-runner-image`, so shards consume a run-scoped artifact rather than the shared cache directly. ## Select projects and labels From b3694bbd724847bb2c519dd50bcf48a51f5405f1 Mon Sep 17 00:00:00 2001 From: omermorad Date: Mon, 28 Sep 2026 08:47:26 +0300 Subject: [PATCH 07/11] refactor: use test results terminology --- .../action.yml | 22 ++--- .../finalize.mjs | 40 +++++----- .../action.yml | 40 +++++----- .../assemble-results.mjs} | 30 +++---- .../stage-results.mjs} | 22 ++--- .github/actions/plan-e2e/action.yml | 2 +- .../action.yml | 50 ++++++------ .../evaluate-publication.mjs | 24 +++--- .../prepare-publication.mjs | 80 +++++++++---------- .github/actions/run-e2e/action.yml | 20 ++--- .github/actions/run-e2e/complete-shard.mjs | 8 +- .github/actions/run-e2e/prepare-shard.mjs | 4 +- .github/workflows/reusable-e2e.yml | 72 ++++++++--------- e2e/.dockerignore | 6 +- e2e/Dockerfile | 8 +- e2e/runner-entrypoint.mjs | 8 +- 16 files changed, 218 insertions(+), 218 deletions(-) rename .github/actions/{finalize-e2e-verdict => finalize-e2e-result}/action.yml (66%) rename .github/actions/{finalize-e2e-verdict => finalize-e2e-result}/finalize.mjs (68%) rename .github/actions/{merge-e2e-evidence => merge-e2e-results}/action.yml (76%) rename .github/actions/{merge-e2e-evidence/assemble-evidence.mjs => merge-e2e-results/assemble-results.mjs} (90%) rename .github/actions/{merge-e2e-evidence/stage-evidence.mjs => merge-e2e-results/stage-results.mjs} (89%) rename .github/actions/{publish-e2e-evidence => publish-e2e-results}/action.yml (72%) rename .github/actions/{publish-e2e-evidence => publish-e2e-results}/evaluate-publication.mjs (67%) rename .github/actions/{publish-e2e-evidence => publish-e2e-results}/prepare-publication.mjs (51%) diff --git a/.github/actions/finalize-e2e-verdict/action.yml b/.github/actions/finalize-e2e-result/action.yml similarity index 66% rename from .github/actions/finalize-e2e-verdict/action.yml rename to .github/actions/finalize-e2e-result/action.yml index 304c73e..f1f3113 100644 --- a/.github/actions/finalize-e2e-verdict/action.yml +++ b/.github/actions/finalize-e2e-result/action.yml @@ -1,9 +1,9 @@ -name: Finalize E2E Verdict -description: Enforce the evidence verdict and all upstream GitHub Actions job results. +name: Finalize E2E Result +description: Enforce the merged E2E result and all upstream GitHub Actions job results. inputs: - verdict-file: - description: Path to the downloaded verdict.json, absolute or relative to GITHUB_WORKSPACE. + result-file: + description: Path to the downloaded result.json, absolute or relative to GITHUB_WORKSPACE. required: true shard-job-result: description: GitHub Actions result of the shard matrix job. @@ -17,21 +17,21 @@ inputs: default: success outputs: - verdict: - description: Final enforced verdict. - value: ${{ steps.finalize.outputs.verdict }} + result: + description: Final enforced result. + value: ${{ steps.finalize.outputs.result }} primary-failure: - description: Primary failure message, empty when the verdict passed. + description: Primary failure message, empty when the result passed. value: ${{ steps.finalize.outputs.primary-failure }} runs: using: composite steps: - - name: Enforce final E2E verdict + - name: Enforce final E2E result id: finalize shell: bash env: - E2E_VERDICT_FILE: ${{ inputs.verdict-file }} + E2E_RESULT_FILE: ${{ inputs.result-file }} E2E_SHARD_JOB_RESULT: ${{ inputs.shard-job-result }} E2E_MERGE_JOB_RESULT: ${{ inputs.merge-job-result }} E2E_PUBLICATION_RESULT: ${{ inputs.publication-result }} @@ -39,7 +39,7 @@ runs: set -euo pipefail node "$GITHUB_ACTION_PATH/finalize.mjs" \ "$GITHUB_WORKSPACE" \ - "$E2E_VERDICT_FILE" \ + "$E2E_RESULT_FILE" \ "$E2E_SHARD_JOB_RESULT" \ "$E2E_MERGE_JOB_RESULT" \ "$E2E_PUBLICATION_RESULT" \ diff --git a/.github/actions/finalize-e2e-verdict/finalize.mjs b/.github/actions/finalize-e2e-result/finalize.mjs similarity index 68% rename from .github/actions/finalize-e2e-verdict/finalize.mjs rename to .github/actions/finalize-e2e-result/finalize.mjs index e216454..294dfb1 100644 --- a/.github/actions/finalize-e2e-verdict/finalize.mjs +++ b/.github/actions/finalize-e2e-result/finalize.mjs @@ -8,7 +8,7 @@ import { isAbsolute, relative, resolve } from "node:path"; const [ workspace, - requestedVerdictFile, + requestedResultFile, shardJobResult, mergeJobResult, publicationResult, @@ -17,31 +17,31 @@ const [ ] = process.argv.slice(2); const workspacePath = realpathSync(workspace); -const verdictFile = isAbsolute(requestedVerdictFile) - ? requestedVerdictFile - : resolve(workspacePath, requestedVerdictFile); -const verdictRelativePath = relative(workspacePath, verdictFile); +const resultFile = isAbsolute(requestedResultFile) + ? requestedResultFile + : resolve(workspacePath, requestedResultFile); +const resultRelativePath = relative(workspacePath, resultFile); const orchestrationFailures = []; if ( - verdictRelativePath === ".." || - verdictRelativePath.startsWith( + resultRelativePath === ".." || + resultRelativePath.startsWith( `..${process.platform === "win32" ? "\\" : "/"}`, ) || - isAbsolute(verdictRelativePath) + isAbsolute(resultRelativePath) ) { - orchestrationFailures.push("verdict-file must be inside GITHUB_WORKSPACE"); + orchestrationFailures.push("result-file must be inside GITHUB_WORKSPACE"); } -let verdict = null; -if (orchestrationFailures.length === 0 && existsSync(verdictFile)) { +let result = null; +if (orchestrationFailures.length === 0 && existsSync(resultFile)) { try { - verdict = JSON.parse(readFileSync(verdictFile, "utf8")); + result = JSON.parse(readFileSync(resultFile, "utf8")); } catch (error) { - orchestrationFailures.push(`Cannot parse verdict.json: ${error.message}`); + orchestrationFailures.push(`Cannot parse result.json: ${error.message}`); } } else if (orchestrationFailures.length === 0) { - orchestrationFailures.push(`Missing verdict.json at ${verdictRelativePath}`); + orchestrationFailures.push(`Missing result.json at ${resultRelativePath}`); } const upstreamResults = [ @@ -55,8 +55,8 @@ for (const [label, result] of upstreamResults) { } } -let primaryFailure = verdict?.primaryFailure ?? null; -const secondaryFailures = [...(verdict?.secondaryFailures ?? [])]; +let primaryFailure = result?.primaryFailure ?? null; +const secondaryFailures = [...(result?.secondaryFailures ?? [])]; for (const message of orchestrationFailures) { const failure = { kind: "infrastructure", message }; if (!primaryFailure) primaryFailure = failure; @@ -64,7 +64,7 @@ for (const message of orchestrationFailures) { secondaryFailures.push(failure); } -const recordedResult = verdict?.result ?? "infrastructure-error"; +const recordedResult = result?.result ?? "infrastructure-error"; const finalResult = recordedResult === "failed" ? "failed" @@ -78,7 +78,7 @@ appendFileSync( [ "## Final E2E Gate", "", - `**Verdict:** ${finalResult}`, + `**Result:** ${finalResult}`, "", `- Shard job: ${shardJobResult}`, `- Merge job: ${mergeJobResult}`, @@ -100,11 +100,11 @@ const escapeOutput = (value) => .replaceAll("%", "%25") .replaceAll("\r", "%0D") .replaceAll("\n", "%0A"); -appendFileSync(outputFile, `verdict=${finalResult}\n`); +appendFileSync(outputFile, `result=${finalResult}\n`); appendFileSync(outputFile, `primary-failure=${escapeOutput(primaryMessage)}\n`); if (finalResult !== "passed") { - console.error(`Final E2E verdict: ${finalResult}`); + console.error(`Final E2E result: ${finalResult}`); if (primaryMessage) console.error(`Primary failure: ${primaryMessage}`); for (const failure of secondaryFailures) console.error(`Additional failure: ${failure.message}`); diff --git a/.github/actions/merge-e2e-evidence/action.yml b/.github/actions/merge-e2e-results/action.yml similarity index 76% rename from .github/actions/merge-e2e-evidence/action.yml rename to .github/actions/merge-e2e-results/action.yml index cfdc95a..7b406c6 100644 --- a/.github/actions/merge-e2e-evidence/action.yml +++ b/.github/actions/merge-e2e-results/action.yml @@ -1,15 +1,15 @@ -name: Merge E2E Evidence -description: Validate shard evidence and merge reports inside the immutable E2E runner image. +name: Merge E2E Results +description: Validate shard results and merge reports inside the immutable E2E runner image. inputs: download-directory: description: Repository-relative directory containing downloaded shard artifacts. required: false default: downloaded-shards - evidence-directory: - description: Repository-relative directory where merged evidence is assembled. + results-directory: + description: Repository-relative directory where merged results are assembled. required: false - default: e2e-evidence + default: e2e-results expected-shards: description: Number of shard status files and report sets required. required: true @@ -18,9 +18,9 @@ inputs: required: true outputs: - verdict: - description: "Authoritative verdict: passed, failed, or infrastructure-error." - value: ${{ steps.assemble.outputs.verdict }} + result: + description: "Final result: passed, failed, or infrastructure-error." + value: ${{ steps.assemble.outputs.result }} total: description: Aggregated test count. value: ${{ steps.assemble.outputs.total }} @@ -33,26 +33,26 @@ outputs: skipped: description: Aggregated skipped test count. value: ${{ steps.assemble.outputs.skipped }} - verdict-file: - description: Absolute path to verdict.json. - value: ${{ steps.assemble.outputs.verdict-file }} + result-file: + description: Absolute path to result.json. + value: ${{ steps.assemble.outputs.result-file }} runs: using: composite steps: - - name: Stage and validate shard evidence + - name: Stage and validate shard results id: stage shell: bash env: E2E_DOWNLOAD_DIRECTORY: ${{ inputs.download-directory }} - E2E_EVIDENCE_DIRECTORY: ${{ inputs.evidence-directory }} + E2E_RESULTS_DIRECTORY: ${{ inputs.results-directory }} E2E_EXPECTED_SHARDS: ${{ inputs.expected-shards }} run: | set -euo pipefail - node "$GITHUB_ACTION_PATH/stage-evidence.mjs" \ + node "$GITHUB_ACTION_PATH/stage-results.mjs" \ "$GITHUB_WORKSPACE" \ "$E2E_DOWNLOAD_DIRECTORY" \ - "$E2E_EVIDENCE_DIRECTORY" \ + "$E2E_RESULTS_DIRECTORY" \ "$E2E_EXPECTED_SHARDS" \ "$GITHUB_OUTPUT" @@ -70,7 +70,7 @@ runs: exit_code=0 if [[ "$E2E_CAN_MERGE" != "true" ]]; then - echo "Blob report merge cannot run because shard evidence is incomplete." + echo "Blob report merge cannot run because shard results are incomplete." exit_code=42 elif [[ ! "$E2E_RUNNER_IMAGE_ID" =~ ^sha256:[a-f0-9]{64}$ ]]; then echo "Blob report merge cannot run because the runner image ID is invalid." @@ -100,18 +100,18 @@ runs: echo "exit-code=$exit_code" >> "$GITHUB_OUTPUT" exit "$exit_code" - - name: Assemble deterministic evidence + - name: Assemble deterministic results id: assemble if: always() && steps.stage.outcome == 'success' shell: bash env: - E2E_EVIDENCE_DIRECTORY: ${{ steps.stage.outputs.evidence-directory }} + E2E_RESULTS_DIRECTORY: ${{ steps.stage.outputs.results-directory }} E2E_EXPECTED_SHARDS: ${{ inputs.expected-shards }} E2E_HTML_MERGE_EXIT_CODE: ${{ steps.merge-html.outputs.exit-code }} run: | set -euo pipefail - node "$GITHUB_ACTION_PATH/assemble-evidence.mjs" \ - "$E2E_EVIDENCE_DIRECTORY" \ + node "$GITHUB_ACTION_PATH/assemble-results.mjs" \ + "$E2E_RESULTS_DIRECTORY" \ "$E2E_EXPECTED_SHARDS" \ "${E2E_HTML_MERGE_EXIT_CODE:-99}" \ "$GITHUB_OUTPUT" diff --git a/.github/actions/merge-e2e-evidence/assemble-evidence.mjs b/.github/actions/merge-e2e-results/assemble-results.mjs similarity index 90% rename from .github/actions/merge-e2e-evidence/assemble-evidence.mjs rename to .github/actions/merge-e2e-results/assemble-results.mjs index a5931be..d0d2c86 100644 --- a/.github/actions/merge-e2e-evidence/assemble-evidence.mjs +++ b/.github/actions/merge-e2e-results/assemble-results.mjs @@ -8,21 +8,21 @@ import { import { resolve } from "node:path"; const [ - evidenceDirectory, + resultsDirectory, requestedExpectedShards, requestedHtmlMergeExitCode, outputFile, ] = process.argv.slice(2); const expectedShards = Number(requestedExpectedShards); const htmlMergeExitCode = Number(requestedHtmlMergeExitCode); -const stagingDirectory = resolve(evidenceDirectory, ".merge-input"); +const stagingDirectory = resolve(resultsDirectory, ".merge-input"); const scanFile = resolve(stagingDirectory, "scan.json"); const infrastructureFailures = []; let scan = { expectedShards, observedShards: [], statuses: [], errors: [] }; if (!existsSync(scanFile)) { infrastructureFailures.push( - "Shard evidence staging did not produce scan.json", + "Shard results staging did not produce scan.json", ); } else { try { @@ -30,7 +30,7 @@ if (!existsSync(scanFile)) { infrastructureFailures.push(...(scan.errors ?? [])); } catch (error) { infrastructureFailures.push( - `Cannot parse shard evidence scan: ${error.message}`, + `Cannot parse shard results scan: ${error.message}`, ); } } @@ -118,7 +118,7 @@ const ctrfReport = { }, }; writeFileSync( - resolve(evidenceDirectory, "ctrf-report.json"), + resolve(resultsDirectory, "ctrf-report.json"), `${JSON.stringify(ctrfReport, null, 2)}\n`, ); @@ -180,8 +180,8 @@ if (hasTestFailure) { ); } -const verdict = { - schemaVersion: 1, +const result = { + schemaVersion: 2, result: hasTestFailure ? "failed" : infrastructureFailures.length > 0 @@ -214,18 +214,18 @@ const verdict = { generatedAt: new Date().toISOString(), }; writeFileSync( - resolve(evidenceDirectory, "verdict.json"), - `${JSON.stringify(verdict, null, 2)}\n`, + resolve(resultsDirectory, "result.json"), + `${JSON.stringify(result, null, 2)}\n`, ); if (outputFile) { for (const [name, value] of [ - ["verdict", verdict.result], - ["total", verdict.summary.total], - ["passed", verdict.summary.passed], - ["failed", verdict.summary.failed], - ["skipped", verdict.summary.skipped], - ["verdict-file", resolve(evidenceDirectory, "verdict.json")], + ["result", result.result], + ["total", result.summary.total], + ["passed", result.summary.passed], + ["failed", result.summary.failed], + ["skipped", result.summary.skipped], + ["result-file", resolve(resultsDirectory, "result.json")], ]) { writeFileSync(outputFile, `${name}=${value}\n`, { flag: "a" }); } diff --git a/.github/actions/merge-e2e-evidence/stage-evidence.mjs b/.github/actions/merge-e2e-results/stage-results.mjs similarity index 89% rename from .github/actions/merge-e2e-evidence/stage-evidence.mjs rename to .github/actions/merge-e2e-results/stage-results.mjs index 121563e..925b577 100644 --- a/.github/actions/merge-e2e-evidence/stage-evidence.mjs +++ b/.github/actions/merge-e2e-results/stage-results.mjs @@ -14,7 +14,7 @@ import { basename, isAbsolute, relative, resolve, sep } from "node:path"; const [ workspace, requestedDownloadDirectory, - requestedEvidenceDirectory, + requestedResultsDirectory, requestedExpectedShards, outputFile, ] = process.argv.slice(2); @@ -47,28 +47,28 @@ const assertRelative = (value, label) => { for (const [value, label] of [ [requestedDownloadDirectory, "download-directory"], - [requestedEvidenceDirectory, "evidence-directory"], + [requestedResultsDirectory, "results-directory"], ]) { assertRelative(value, label); } const workspacePath = realpathSync(workspace); const downloadDirectory = resolve(workspacePath, requestedDownloadDirectory); -const evidenceDirectory = resolve(workspacePath, requestedEvidenceDirectory); +const resultsDirectory = resolve(workspacePath, requestedResultsDirectory); if ( - existsSync(evidenceDirectory) && - lstatSync(evidenceDirectory).isSymbolicLink() + existsSync(resultsDirectory) && + lstatSync(resultsDirectory).isSymbolicLink() ) { - throw new Error("evidence-directory must not be a symbolic link"); + throw new Error("results-directory must not be a symbolic link"); } -rmSync(evidenceDirectory, { recursive: true, force: true }); +rmSync(resultsDirectory, { recursive: true, force: true }); -const stagingDirectory = resolve(evidenceDirectory, ".merge-input"); +const stagingDirectory = resolve(resultsDirectory, ".merge-input"); const blobDirectory = resolve(stagingDirectory, "blob-report"); const ctrfDirectory = resolve(stagingDirectory, "ctrf"); -const statusDirectory = resolve(evidenceDirectory, "shard-status"); -const htmlDirectory = resolve(evidenceDirectory, "playwright-report"); +const statusDirectory = resolve(resultsDirectory, "shard-status"); +const htmlDirectory = resolve(resultsDirectory, "playwright-report"); mkdirSync(blobDirectory, { recursive: true }); mkdirSync(ctrfDirectory, { recursive: true }); mkdirSync(statusDirectory, { recursive: true }); @@ -200,7 +200,7 @@ const appendOutput = (name, value) => { writeFileSync(outputFile, `${name}=${escaped}\n`, { flag: "a" }); }; -appendOutput("evidence-directory", evidenceDirectory); +appendOutput("results-directory", resultsDirectory); appendOutput("blob-directory", blobDirectory); appendOutput("html-directory", htmlDirectory); appendOutput("can-merge", errors.length === 0 ? "true" : "false"); diff --git a/.github/actions/plan-e2e/action.yml b/.github/actions/plan-e2e/action.yml index 79a5419..e8d3e97 100644 --- a/.github/actions/plan-e2e/action.yml +++ b/.github/actions/plan-e2e/action.yml @@ -50,7 +50,7 @@ outputs: description: GitHub Actions shard matrix JSON. value: ${{ steps.plan.outputs.matrix }} artifact-retention-days: - description: Evidence and image retention period. + description: Results and image retention period. value: ${{ steps.plan.outputs.artifact-retention-days }} runner-content-hash: description: SHA-256 of committed and non-ignored E2E runner inputs. diff --git a/.github/actions/publish-e2e-evidence/action.yml b/.github/actions/publish-e2e-results/action.yml similarity index 72% rename from .github/actions/publish-e2e-evidence/action.yml rename to .github/actions/publish-e2e-results/action.yml index 4479685..9e69ea9 100644 --- a/.github/actions/publish-e2e-evidence/action.yml +++ b/.github/actions/publish-e2e-results/action.yml @@ -1,20 +1,20 @@ -name: Publish E2E Evidence -description: Finalize, publish, summarize, and enforce the authoritative E2E verdict. +name: Publish E2E Results +description: Finalize, publish, summarize, and enforce the final E2E result. inputs: - evidence-directory: - description: Repository-relative directory containing merged E2E evidence. + results-directory: + description: Repository-relative directory containing merged E2E results. required: false - default: e2e-evidence + default: e2e-results expected-shards: description: Number of shard results required by the run. required: true artifact-name: - description: Name of the durable merged evidence artifact. + description: Name of the durable merged results artifact. required: false - default: e2e-evidence + default: e2e-results retention-days: - description: Number of days GitHub retains the merged evidence artifact. + description: Number of days GitHub retains the merged results artifact. required: false default: "10" runner-download-outcome: @@ -27,21 +27,21 @@ inputs: description: Outcome of verifying the loaded runner image ID. required: true shard-download-outcome: - description: Outcome of downloading shard evidence. + description: Outcome of downloading shard results. required: true merge-outcome: - description: Outcome of deterministic evidence assembly. + description: Outcome of deterministic results assembly. required: true report-publication-outcome: description: Outcome of building and publishing the report image archive. required: true outputs: - verdict: - description: "Authoritative verdict: passed, failed, or infrastructure-error." - value: ${{ steps.evaluate.outputs.verdict }} + result: + description: "Final result: passed, failed, or infrastructure-error." + value: ${{ steps.evaluate.outputs.result }} artifact-name: - description: Merged evidence artifact name. + description: Merged results artifact name. value: ${{ inputs.artifact-name }} total: description: Aggregated test count. @@ -55,18 +55,18 @@ outputs: skipped: description: Aggregated skipped test count. value: ${{ steps.evaluate.outputs.skipped }} - verdict-file: - description: Absolute path to the finalized verdict.json. - value: ${{ steps.prepare.outputs.verdict-file }} + result-file: + description: Absolute path to the finalized result.json. + value: ${{ steps.prepare.outputs.result-file }} runs: using: composite steps: - id: prepare - name: Finalize authoritative verdict + name: Finalize final result shell: bash env: - E2E_EVIDENCE_DIRECTORY: ${{ inputs.evidence-directory }} + E2E_RESULTS_DIRECTORY: ${{ inputs.results-directory }} E2E_EXPECTED_SHARDS: ${{ inputs.expected-shards }} E2E_RUNNER_DOWNLOAD_OUTCOME: ${{ inputs.runner-download-outcome }} E2E_RUNNER_LOAD_OUTCOME: ${{ inputs.runner-load-outcome }} @@ -78,7 +78,7 @@ runs: set -euo pipefail node "$GITHUB_ACTION_PATH/prepare-publication.mjs" \ "$GITHUB_WORKSPACE" \ - "$E2E_EVIDENCE_DIRECTORY" \ + "$E2E_RESULTS_DIRECTORY" \ "$E2E_EXPECTED_SHARDS" \ "$E2E_RUNNER_DOWNLOAD_OUTCOME" \ "$E2E_RUNNER_LOAD_OUTCOME" \ @@ -89,27 +89,27 @@ runs: "$GITHUB_OUTPUT" - id: publish - name: Upload authoritative E2E evidence + name: Upload final E2E results continue-on-error: true uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: ${{ inputs.artifact-name }} - path: ${{ steps.prepare.outputs.evidence-directory }}/ + path: ${{ steps.prepare.outputs.results-directory }}/ if-no-files-found: error retention-days: ${{ inputs.retention-days }} - id: evaluate - name: Publish summary and enforce verdict + name: Publish summary and enforce result if: always() && steps.prepare.outcome == 'success' shell: bash env: - E2E_VERDICT_FILE: ${{ steps.prepare.outputs.verdict-file }} + E2E_RESULT_FILE: ${{ steps.prepare.outputs.result-file }} E2E_ARTIFACT_NAME: ${{ inputs.artifact-name }} E2E_PUBLICATION_OUTCOME: ${{ steps.publish.outcome }} run: | set -euo pipefail node "$GITHUB_ACTION_PATH/evaluate-publication.mjs" \ - "$E2E_VERDICT_FILE" \ + "$E2E_RESULT_FILE" \ "$E2E_ARTIFACT_NAME" \ "${E2E_PUBLICATION_OUTCOME:-failure}" \ "$GITHUB_STEP_SUMMARY" \ diff --git a/.github/actions/publish-e2e-evidence/evaluate-publication.mjs b/.github/actions/publish-e2e-results/evaluate-publication.mjs similarity index 67% rename from .github/actions/publish-e2e-evidence/evaluate-publication.mjs rename to .github/actions/publish-e2e-results/evaluate-publication.mjs index a745e81..429b42d 100644 --- a/.github/actions/publish-e2e-evidence/evaluate-publication.mjs +++ b/.github/actions/publish-e2e-results/evaluate-publication.mjs @@ -1,25 +1,25 @@ import { appendFileSync, readFileSync } from "node:fs"; -const [verdictFile, artifactName, publicationOutcome, summaryFile, outputFile] = +const [resultFile, artifactName, publicationOutcome, summaryFile, outputFile] = process.argv.slice(2); -const verdict = JSON.parse(readFileSync(verdictFile, "utf8")); -const summary = verdict.summary ?? {}; +const result = JSON.parse(readFileSync(resultFile, "utf8")); +const summary = result.summary ?? {}; const enforcedResult = - verdict.result === "passed" && publicationOutcome !== "success" + result.result === "passed" && publicationOutcome !== "success" ? "infrastructure-error" - : verdict.result; + : result.result; const publicationFailure = publicationOutcome === "success" ? null - : `Evidence artifact publication finished with ${publicationOutcome || "unknown"}`; -const primaryFailure = verdict.primaryFailure?.message ?? publicationFailure; + : `Results artifact publication finished with ${publicationOutcome || "unknown"}`; +const primaryFailure = result.primaryFailure?.message ?? publicationFailure; appendFileSync( summaryFile, [ "## E2E Gate", "", - `**Verdict:** ${enforcedResult}`, + `**Result:** ${enforcedResult}`, "", "| Result | Count |", "| --- | ---: |", @@ -28,9 +28,9 @@ appendFileSync( `| Failed | ${summary.failed ?? 0} |`, `| Skipped | ${summary.skipped ?? 0} |`, "", - `**Shards:** ${(verdict.observedShards ?? []).length}/${verdict.expectedShards ?? 0}`, + `**Shards:** ${(result.observedShards ?? []).length}/${result.expectedShards ?? 0}`, "", - `**Evidence artifact:** \`${artifactName}\``, + `**Results artifact:** \`${artifactName}\``, ...(primaryFailure ? ["", `**Primary failure:** ${primaryFailure}`] : []), ...(publicationFailure && publicationFailure !== primaryFailure ? ["", `**Publication failure:** ${publicationFailure}`] @@ -46,7 +46,7 @@ const appendOutput = (name, value) => { .replaceAll("\n", "%0A"); appendFileSync(outputFile, `${name}=${escaped}\n`); }; -appendOutput("verdict", enforcedResult); +appendOutput("result", enforcedResult); appendOutput("total", Number(summary.total ?? 0)); appendOutput("passed", Number(summary.passed ?? 0)); appendOutput("failed", Number(summary.failed ?? 0)); @@ -54,7 +54,7 @@ appendOutput("skipped", Number(summary.skipped ?? 0)); appendOutput("primary-failure", primaryFailure ?? ""); if (enforcedResult !== "passed" || publicationOutcome !== "success") { - console.error(`E2E evidence is not healthy: ${enforcedResult}`); + console.error(`E2E results did not complete successfully: ${enforcedResult}`); if (primaryFailure) console.error(primaryFailure); if (publicationFailure && publicationFailure !== primaryFailure) { console.error(publicationFailure); diff --git a/.github/actions/publish-e2e-evidence/prepare-publication.mjs b/.github/actions/publish-e2e-results/prepare-publication.mjs similarity index 51% rename from .github/actions/publish-e2e-evidence/prepare-publication.mjs rename to .github/actions/publish-e2e-results/prepare-publication.mjs index e9229c0..c8cd5c7 100644 --- a/.github/actions/publish-e2e-evidence/prepare-publication.mjs +++ b/.github/actions/publish-e2e-results/prepare-publication.mjs @@ -10,7 +10,7 @@ import { isAbsolute, relative, resolve, sep } from "node:path"; const [ workspace, - requestedEvidenceDirectory, + requestedResultsDirectory, requestedExpectedShards, runnerDownloadOutcome, runnerLoadOutcome, @@ -29,46 +29,46 @@ if ( ) { throw new Error("expected-shards must be an integer between 1 and 100"); } -if (!requestedEvidenceDirectory || isAbsolute(requestedEvidenceDirectory)) { - throw new Error("evidence-directory must be a non-empty repository-relative path"); +if (!requestedResultsDirectory || isAbsolute(requestedResultsDirectory)) { + throw new Error("results-directory must be a non-empty repository-relative path"); } const workspacePath = realpathSync(workspace); -const evidenceDirectory = resolve(workspacePath, requestedEvidenceDirectory); -const relativeEvidenceDirectory = relative(workspacePath, evidenceDirectory); +const resultsDirectory = resolve(workspacePath, requestedResultsDirectory); +const relativeResultsDirectory = relative(workspacePath, resultsDirectory); if ( - relativeEvidenceDirectory === "" || - relativeEvidenceDirectory === ".." || - relativeEvidenceDirectory.startsWith(`..${sep}`) || - isAbsolute(relativeEvidenceDirectory) + relativeResultsDirectory === "" || + relativeResultsDirectory === ".." || + relativeResultsDirectory.startsWith(`..${sep}`) || + isAbsolute(relativeResultsDirectory) ) { - throw new Error("evidence-directory must resolve below GITHUB_WORKSPACE"); + throw new Error("results-directory must resolve below GITHUB_WORKSPACE"); } if ( - existsSync(evidenceDirectory) && - lstatSync(evidenceDirectory).isSymbolicLink() + existsSync(resultsDirectory) && + lstatSync(resultsDirectory).isSymbolicLink() ) { - throw new Error("evidence-directory must not be a symbolic link"); + throw new Error("results-directory must not be a symbolic link"); } -mkdirSync(evidenceDirectory, { recursive: true }); +mkdirSync(resultsDirectory, { recursive: true }); -const verdictFile = resolve(evidenceDirectory, "verdict.json"); -let verdict = null; -if (existsSync(verdictFile)) { +const resultFile = resolve(resultsDirectory, "result.json"); +let result = null; +if (existsSync(resultFile)) { try { - verdict = JSON.parse(readFileSync(verdictFile, "utf8")); + result = JSON.parse(readFileSync(resultFile, "utf8")); } catch { - verdict = null; + result = null; } } -if (!verdict || typeof verdict !== "object") { - verdict = { - schemaVersion: 1, +if (!result || typeof result !== "object") { + result = { + schemaVersion: 2, result: "infrastructure-error", primaryFailure: { kind: "infrastructure", - phase: "evidence-assembly", - message: "Evidence assembly did not produce a readable verdict.json", + phase: "results-assembly", + message: "Results assembly did not produce a readable result.json", }, secondaryFailures: [], expectedShards, @@ -91,8 +91,8 @@ const checks = [ ["runner-archive-download", runnerDownloadOutcome], ["runner-image-load", runnerLoadOutcome], ["runner-image-verification", runnerVerificationOutcome], - ["shard-evidence-download", shardDownloadOutcome], - ["evidence-merge", mergeOutcome], + ["shard-results-download", shardDownloadOutcome], + ["results-merge", mergeOutcome], ["report-image-publication", reportPublicationOutcome], ]; const newFailures = checks @@ -103,28 +103,28 @@ const newFailures = checks message: `${phase} finished with ${outcome || "unknown"}`, })); -verdict.secondaryFailures = Array.isArray(verdict.secondaryFailures) - ? verdict.secondaryFailures +result.secondaryFailures = Array.isArray(result.secondaryFailures) + ? result.secondaryFailures : []; if (newFailures.length > 0) { - if (!verdict.primaryFailure) { - verdict.primaryFailure = newFailures.shift(); + if (!result.primaryFailure) { + result.primaryFailure = newFailures.shift(); } - verdict.secondaryFailures.push(...newFailures); - if (verdict.result === "passed" || !verdict.result) { - verdict.result = "infrastructure-error"; + result.secondaryFailures.push(...newFailures); + if (result.result === "passed" || !result.result) { + result.result = "infrastructure-error"; } } -verdict.expectedShards = expectedShards; -verdict.artifacts = { - ...(verdict.artifacts ?? {}), +result.expectedShards = expectedShards; +result.artifacts = { + ...(result.artifacts ?? {}), reportImage: reportPublicationOutcome === "success" ? { published: true, name: "e2e-report-image" } : { published: false, name: null }, }; -verdict.finalizedAt = new Date().toISOString(); -writeFileSync(verdictFile, `${JSON.stringify(verdict, null, 2)}\n`); +result.finalizedAt = new Date().toISOString(); +writeFileSync(resultFile, `${JSON.stringify(result, null, 2)}\n`); const appendOutput = (name, value) => { const escaped = String(value) @@ -133,5 +133,5 @@ const appendOutput = (name, value) => { .replaceAll("\n", "%0A"); writeFileSync(outputFile, `${name}=${escaped}\n`, { flag: "a" }); }; -appendOutput("evidence-directory", evidenceDirectory); -appendOutput("verdict-file", verdictFile); +appendOutput("results-directory", resultsDirectory); +appendOutput("result-file", resultFile); diff --git a/.github/actions/run-e2e/action.yml b/.github/actions/run-e2e/action.yml index 0949ee6..c83ecbe 100644 --- a/.github/actions/run-e2e/action.yml +++ b/.github/actions/run-e2e/action.yml @@ -1,5 +1,5 @@ name: Run E2E Shard -description: Run one Playwright shard and always leave machine-readable shard evidence. +description: Run one Playwright shard and always leave machine-readable shard results. inputs: working-directory: @@ -58,10 +58,10 @@ outputs: description: "Shard result: passed, failed, or infrastructure-error." value: ${{ steps.finalize.outputs.result }} exit-code: - description: Effective shard exit code after SUT, Playwright, evidence, and cleanup checks. + description: Effective shard exit code after SUT, Playwright, results, and cleanup checks. value: ${{ steps.finalize.outputs.exit-code }} shard-directory: - description: Absolute path containing this shard's evidence. + description: Absolute path containing this shard's results. value: ${{ steps.initialize.outputs.shard-directory }} status-file: description: Absolute path to this shard's status JSON. @@ -70,7 +70,7 @@ outputs: runs: using: composite steps: - - name: Initialize shard evidence + - name: Initialize shard results id: initialize shell: bash env: @@ -223,11 +223,11 @@ runs: --env BASE_URL="$E2E_BASE_URL" \ --env CI=true \ --env HOME=/tmp \ - --env E2E_RESULTS_DIR=/evidence/test-results \ - --env PLAYWRIGHT_BLOB_OUTPUT_DIR=/evidence/blob-report \ - --env PLAYWRIGHT_OUTPUT_DIR=/evidence/test-results \ - --env CTRF_OUTPUT_FILE=/evidence/ctrf/ctrf-report.json \ - --volume "$E2E_SHARD_DIRECTORY:/evidence" \ + --env E2E_RESULTS_DIR=/test-output/test-results \ + --env PLAYWRIGHT_BLOB_OUTPUT_DIR=/test-output/blob-report \ + --env PLAYWRIGHT_OUTPUT_DIR=/test-output/test-results \ + --env CTRF_OUTPUT_FILE=/test-output/ctrf/ctrf-report.json \ + --volume "$E2E_SHARD_DIRECTORY:/test-output" \ "$E2E_RUNNER_IMAGE_ID" \ --config "$E2E_CONFIG" \ --shard="$E2E_SHARD_INDEX/$E2E_SHARD_TOTAL" \ @@ -285,7 +285,7 @@ runs: echo "exit-code=$teardown_exit_code" >> "$GITHUB_OUTPUT" exit "$teardown_exit_code" - - name: Finalize shard evidence + - name: Finalize shard results id: finalize if: always() && steps.initialize.outcome == 'success' shell: bash diff --git a/.github/actions/run-e2e/complete-shard.mjs b/.github/actions/run-e2e/complete-shard.mjs index 6b3a9ae..77534b8 100644 --- a/.github/actions/run-e2e/complete-shard.mjs +++ b/.github/actions/run-e2e/complete-shard.mjs @@ -128,13 +128,13 @@ if (runnerExitCode === 0 && startupExitCode === 0 && playwrightExitCode === null if (logExitCode === null) { failures.push({ kind: "infrastructure", - phase: "evidence", + phase: "results", message: "Docker Compose log capture did not report an exit code", }); } else if (logExitCode !== 0) { failures.push({ kind: "infrastructure", - phase: "evidence", + phase: "results", message: `Docker Compose log capture exited with ${logExitCode}`, }); } @@ -176,7 +176,7 @@ const exitCodeForFailure = (failure) => { return runnerExitCode ?? 1; case "sut-startup": return startupExitCode ?? 1; - case "evidence": + case "results": return logExitCode ?? 1; case "cleanup": return teardownExitCode ?? 1; @@ -193,7 +193,7 @@ status.lifecycle.playwright = { exitCode: playwrightExitCode, started: playwrightExitCode !== null, }; -status.lifecycle.evidence.logCaptureExitCode = logExitCode; +status.lifecycle.results.logCaptureExitCode = logExitCode; status.lifecycle.cleanup.teardownExitCode = teardownExitCode; status.finishedAt = finishedAt.toISOString(); status.durationMs = Math.max(0, finishedAt.getTime() - startedAt.getTime()); diff --git a/.github/actions/run-e2e/prepare-shard.mjs b/.github/actions/run-e2e/prepare-shard.mjs index 83ac925..2813ccc 100644 --- a/.github/actions/run-e2e/prepare-shard.mjs +++ b/.github/actions/run-e2e/prepare-shard.mjs @@ -149,7 +149,7 @@ writeFileSync(selectionFile, `${JSON.stringify(selection, null, 2)}\n`); const statusFile = resolve(shardDirectory, "shard-status.json"); const status = { - schemaVersion: 1, + schemaVersion: 2, shard: { index: shardIndex, total: shardTotal }, result: "running", exitCode: null, @@ -180,7 +180,7 @@ const status = { startupExitCode: null, }, playwright: { exitCode: null, started: false }, - evidence: { logCaptureExitCode: null }, + results: { logCaptureExitCode: null }, cleanup: { teardownExitCode: null }, }, github: { diff --git a/.github/workflows/reusable-e2e.yml b/.github/workflows/reusable-e2e.yml index 021f0ea..c0c7b9e 100644 --- a/.github/workflows/reusable-e2e.yml +++ b/.github/workflows/reusable-e2e.yml @@ -56,11 +56,11 @@ on: report-image-artifact: description: Artifact containing the portable E2E report Docker archive. value: ${{ jobs.merge.outputs.report-image-artifact }} - verdict: - description: Authoritative E2E verdict. - value: ${{ jobs.e2e_gate.outputs.verdict }} + result: + description: Final E2E result. + value: ${{ jobs.e2e_gate.outputs.result }} artifact-name: - description: Name of the merged evidence artifact. + description: Name of the merged results artifact. value: ${{ jobs.merge.outputs.artifact-name }} total: description: Total number of tests in the merged report. @@ -222,7 +222,7 @@ jobs: shard-total: ${{ needs.plan.outputs.shard-count }} results-directory: test-results - - name: Upload shard evidence + - name: Upload shard results if: always() && steps.run.outputs.shard-directory != '' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: @@ -231,7 +231,7 @@ jobs: if-no-files-found: error retention-days: ${{ needs.plan.outputs.artifact-retention-days }} - - name: Preserve shard verdict + - name: Preserve shard result if: >- always() && (steps.download-runner.outcome != 'success' || steps.load-runner.outcome != 'success' || @@ -242,7 +242,7 @@ jobs: exit 1 merge: - name: Merge E2E evidence + name: Merge E2E results needs: - plan - build_runner @@ -251,7 +251,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 outputs: - verdict: ${{ steps.publish.outputs.verdict }} + result: ${{ steps.publish.outputs.result }} artifact-name: ${{ steps.publish.outputs.artifact-name }} total: ${{ steps.publish.outputs.total }} passed: ${{ steps.publish.outputs.passed }} @@ -299,7 +299,7 @@ jobs: fi - id: download - name: Download shard evidence + name: Download shard results if: always() continue-on-error: true uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 @@ -309,13 +309,13 @@ jobs: merge-multiple: false - id: merge - name: Merge E2E evidence + name: Merge E2E results if: always() continue-on-error: true - uses: $/.github/actions/merge-e2e-evidence + uses: $/.github/actions/merge-e2e-results with: download-directory: downloaded-shards - evidence-directory: e2e-evidence + results-directory: e2e-results expected-shards: ${{ needs.plan.outputs.shard-count }} runner-image-id: ${{ needs.build_runner.outputs.image-id }} @@ -325,21 +325,21 @@ jobs: continue-on-error: true uses: $/.github/actions/package-e2e-report with: - report-directory: e2e-evidence/playwright-report + report-directory: e2e-results/playwright-report image-name: playwright-e2e-report:${{ github.run_id }}-${{ github.run_attempt }} archive-directory: e2e-report-image artifact-name: e2e-report-image retention-days: ${{ needs.plan.outputs.artifact-retention-days }} - id: publish - name: Publish and enforce authoritative evidence + name: Publish and enforce final results if: always() continue-on-error: true - uses: $/.github/actions/publish-e2e-evidence + uses: $/.github/actions/publish-e2e-results with: - evidence-directory: e2e-evidence + results-directory: e2e-results expected-shards: ${{ needs.plan.outputs.shard-count }} - artifact-name: e2e-evidence + artifact-name: e2e-results retention-days: ${{ needs.plan.outputs.artifact-retention-days }} runner-download-outcome: ${{ steps.download-runner.outcome }} runner-load-outcome: ${{ steps.load-runner.outcome }} @@ -348,11 +348,11 @@ jobs: merge-outcome: ${{ steps.merge.outcome }} report-publication-outcome: ${{ steps.package-report.outcome }} - - name: Preserve merge verdict + - name: Preserve merge result if: always() && steps.publish.outcome != 'success' shell: bash run: | - echo "::error::E2E evidence could not be completed successfully" + echo "::error::E2E results could not be completed successfully" exit 1 e2e_gate: @@ -365,22 +365,22 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 outputs: - verdict: ${{ steps.finalize.outputs.verdict }} + result: ${{ steps.finalize.outputs.result }} primary-failure: ${{ steps.finalize.outputs.primary-failure }} steps: - id: download - name: Download authoritative evidence + name: Download final results continue-on-error: true uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: - name: e2e-evidence - path: final-evidence + name: e2e-results + path: final-results - id: finalize - name: Enforce authoritative verdict - uses: $/.github/actions/finalize-e2e-verdict + name: Enforce final result + uses: $/.github/actions/finalize-e2e-result with: - verdict-file: final-evidence/verdict.json + result-file: final-results/result.json shard-job-result: ${{ needs.shards.result }} merge-job-result: ${{ needs.merge.result }} publication-result: ${{ steps.download.outcome }} @@ -405,8 +405,8 @@ jobs: uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 env: E2E_GATE_JOB_RESULT: ${{ needs.e2e_gate.result }} - E2E_GATE_VERDICT: ${{ needs.e2e_gate.outputs.verdict }} - E2E_MERGE_VERDICT: ${{ needs.merge.outputs.verdict }} + E2E_GATE_RESULT: ${{ needs.e2e_gate.outputs.result }} + E2E_MERGE_RESULT: ${{ needs.merge.outputs.result }} E2E_PRIMARY_FAILURE: ${{ needs.e2e_gate.outputs.primary-failure }} E2E_TOTAL: ${{ needs.merge.outputs.total }} E2E_PASSED: ${{ needs.merge.outputs.passed }} @@ -441,12 +441,12 @@ jobs: }; const gateResult = clean(process.env.E2E_GATE_JOB_RESULT, 'failure'); - const verdict = clean( - process.env.E2E_GATE_VERDICT, - clean(process.env.E2E_MERGE_VERDICT, gateResult === 'success' ? 'passed' : 'infrastructure-error'), + const result = clean( + process.env.E2E_GATE_RESULT, + clean(process.env.E2E_MERGE_RESULT, gateResult === 'success' ? 'passed' : 'infrastructure-error'), ); - const passed = verdict === 'passed'; - const status = passed ? '✅ Passed' : verdict === 'failed' ? '❌ Tests failed' : '⚠️ Infrastructure error'; + const passed = result === 'passed'; + const status = passed ? '✅ Passed' : result === 'failed' ? '❌ Tests failed' : '⚠️ Infrastructure error'; const total = clean(process.env.E2E_TOTAL, 'unknown'); const passedCount = clean(process.env.E2E_PASSED, 'unknown'); const failedCount = clean(process.env.E2E_FAILED, 'unknown'); @@ -455,7 +455,7 @@ jobs: const projects = list(process.env.E2E_PROJECTS, 'All configured projects'); const labels = list(process.env.E2E_LABELS, 'All labels'); const labelMatch = clean(process.env.E2E_LABEL_MATCH, 'all'); - const artifactName = clean(process.env.E2E_ARTIFACT_NAME, 'e2e-evidence'); + const artifactName = clean(process.env.E2E_ARTIFACT_NAME, 'e2e-results'); const runnerImageName = clean(process.env.E2E_RUNNER_IMAGE_NAME, 'unavailable'); const runnerImageId = clean(process.env.E2E_RUNNER_IMAGE_ID, 'unavailable'); const runnerImageArtifact = clean(process.env.E2E_RUNNER_IMAGE_ARTIFACT, 'unavailable'); @@ -469,7 +469,7 @@ jobs: const lines = [ marker, - '## E2E confidence report', + '## E2E test report', '', '| Signal | Result |', '| --- | --- |', @@ -481,7 +481,7 @@ jobs: '| SUT | Docker Compose, isolated per shard |', `| Runner image | ${cell(`\`${runnerImageName}\` · \`${runnerImageId}\` · \`${runnerImageArtifact}\` · ${runnerSource}`)} |`, `| Report image | ${cell(`\`${reportImageName}\` · \`${reportImageId}\` · \`${reportImageArtifact}\``)} |`, - `| Evidence | [Open workflow run](${runUrl}) · \`${cell(artifactName)}\` |`, + `| Results | [Open workflow run](${runUrl}) · \`${cell(artifactName)}\` |`, ]; const primaryFailure = clean(process.env.E2E_PRIMARY_FAILURE, ''); diff --git a/e2e/.dockerignore b/e2e/.dockerignore index c0ca984..987a4b7 100644 --- a/e2e/.dockerignore +++ b/e2e/.dockerignore @@ -8,7 +8,7 @@ node_modules/ .npmrc **/.npmrc -# Generated test evidence and reports +# Generated test results and reports blob-report/ **/blob-report/ ctrf/ @@ -17,8 +17,8 @@ playwright-report/ **/playwright-report/ test-results/ **/test-results/ -e2e-evidence/ -**/e2e-evidence/ +e2e-results/ +**/e2e-results/ downloaded-shards/ **/downloaded-shards/ coverage/ diff --git a/e2e/Dockerfile b/e2e/Dockerfile index 0881e8a..91e244b 100644 --- a/e2e/Dockerfile +++ b/e2e/Dockerfile @@ -1,9 +1,9 @@ FROM mcr.microsoft.com/playwright:v1.61.1-noble@sha256:5b8f294aff9041b7191c34a4bab3ac270157a28774d4b0660e9743297b697e48 ENV CI=true \ - E2E_RESULTS_DIR=/evidence/test-results \ - PLAYWRIGHT_BLOB_OUTPUT_DIR=/evidence/blob-report \ - CTRF_OUTPUT_FILE=/evidence/ctrf/ctrf-report.json + E2E_RESULTS_DIR=/test-output/test-results \ + PLAYWRIGHT_BLOB_OUTPUT_DIR=/test-output/blob-report \ + CTRF_OUTPUT_FILE=/test-output/ctrf/ctrf-report.json RUN corepack enable && corepack prepare pnpm@10.26.2 --activate @@ -14,7 +14,7 @@ RUN pnpm install --frozen-lockfile COPY --chown=pwuser:pwuser . . -RUN mkdir -p /evidence && chown pwuser:pwuser /evidence +RUN mkdir -p /test-output && chown pwuser:pwuser /test-output USER pwuser diff --git a/e2e/runner-entrypoint.mjs b/e2e/runner-entrypoint.mjs index 16ff077..25971c8 100644 --- a/e2e/runner-entrypoint.mjs +++ b/e2e/runner-entrypoint.mjs @@ -2,12 +2,12 @@ import { spawn } from "node:child_process"; import { mkdirSync } from "node:fs"; import { dirname } from "node:path"; -const evidenceRoot = "/evidence"; +const testOutputRoot = "/test-output"; const environmentDefaults = { CI: "true", - E2E_RESULTS_DIR: `${evidenceRoot}/test-results`, - PLAYWRIGHT_BLOB_OUTPUT_DIR: `${evidenceRoot}/blob-report`, - CTRF_OUTPUT_FILE: `${evidenceRoot}/ctrf/ctrf-report.json`, + E2E_RESULTS_DIR: `${testOutputRoot}/test-results`, + PLAYWRIGHT_BLOB_OUTPUT_DIR: `${testOutputRoot}/blob-report`, + CTRF_OUTPUT_FILE: `${testOutputRoot}/ctrf/ctrf-report.json`, }; for (const [name, value] of Object.entries(environmentDefaults)) { From adfd5c912255f705692726eeb37fff60187c50ac Mon Sep 17 00:00:00 2001 From: omermorad Date: Mon, 28 Sep 2026 08:51:36 +0300 Subject: [PATCH 08/11] docs: describe the lightweight E2E framework --- README.md | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 85c0c36..961bb40 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # Playwright E2E CI Framework -A reusable GitHub Actions pipeline for Playwright tests against a Dockerized system under test (SUT), with durable evidence and one fail-closed gate. +A reusable GitHub Actions pipeline for Playwright tests against a Dockerized system under test (SUT), with retained test results and one fail-closed gate. -This repository proves the contract with a root Docker Compose service. The service exposes a health endpoint and echoes requests; Playwright tests it through API and Chromium clients. +This repository demonstrates the framework with a root Docker Compose service. The service exposes a health endpoint and echoes requests; Playwright tests it through API and Chromium clients. ```text pull request / main / manual @@ -25,11 +25,11 @@ pull request / main / manual load runner load runner compose SUT compose SUT run tests run tests - evidence evidence + test results test results | | +------+------+ | - merge evidence + merge test reports | package report image | @@ -37,7 +37,7 @@ pull request / main / manual | E2E Gate | - sticky PR confidence report + sticky PR test report ``` ## Run the example locally @@ -94,7 +94,7 @@ Replace the marker with the immutable SHA you adopt. The workflow needs no inher | `label-match` | empty | Optional `all` or `any` override | | `comment-on-pr` | `true` | Update one E2E report comment on same-repository pull requests | -The workflow exposes the verdict, merged test totals, and the exact names and IDs of its portable images. The image outputs are listed below. +The workflow exposes the final result, merged test totals, and the exact names and IDs of its portable images. The image outputs are listed below. ## Configure execution policy @@ -216,23 +216,23 @@ Each shard runs on its own runner and performs the full lifecycle: A startup or cleanup failure is an infrastructure error. A Playwright failure remains the primary failure when log collection or cleanup also fails. -## Evidence and gate +## Test results and gate -Every shard uploads its status, Compose logs, blob report, CTRF result, and Playwright test results. The merged `e2e-evidence` artifact contains: +Every shard uploads its status, Compose logs, blob report, CTRF result, and Playwright test results. The merged test-results artifact is named `e2e-results` and contains: ```text -e2e-evidence/ +e2e-results/ +-- playwright-report/ +-- ctrf-report.json +-- shard-status/ -+-- verdict.json ++-- result.json ``` -The same immutable runner image merges the shard blobs; pull-request dependencies are never installed directly on the GitHub host. `E2E Gate` fails for runner transport or identity problems, a failed or cancelled shard, unhealthy SUT lifecycle, missing evidence, report merge failure, report-image publication failure, evidence publication failure, or a non-passing verdict. Evidence publication cannot turn a failed run green. +The same immutable runner image merges the shard blobs; pull-request dependencies are never installed directly on the GitHub host. `E2E Gate` fails for runner transport or identity problems, a failed or cancelled shard, unhealthy SUT lifecycle, missing test results, report merge failure, report-image publication failure, test-results artifact publication failure, or a non-passing result. Publishing reports cannot turn a failed run green. ## Load the runner and report images -A healthy run publishes two Docker archives in addition to the regular test evidence. If report packaging or publication fails, the report outputs are empty and the authoritative verdict is an infrastructure error. +A healthy run publishes two Docker archives in addition to the merged test-results artifact. If report packaging or publication fails, the report outputs are empty and the final result is an infrastructure error. | Artifact | Archive inside it | Purpose | | --- | --- | --- | @@ -268,7 +268,7 @@ docker run --rm --init --ipc=host \ --add-host=host.docker.internal:host-gateway \ -e BASE_URL=http://host.docker.internal:4173 \ -e HOME=/tmp \ - -v "$PWD/e2e-output:/evidence" \ + -v "$PWD/e2e-output:/test-output" \ "$RUNNER_IMAGE_ID" ``` @@ -285,6 +285,6 @@ Open `http://127.0.0.1:8080`. The report image contains the complete merged HTML These images are currently GitHub artifacts, not registry publications. Download and `docker load` the archives before use; artifact retention still applies, and the local image names are not remotely pullable. -For pull requests from the same repository, the workflow creates or updates one sticky `E2E confidence report` comment with the verdict, selected profile, aggregate totals, active project and label filters, image identities, failure reason, and workflow evidence link. Grant `pull-requests: write` in the caller workflow, or set `comment-on-pr: false` when comments are not wanted. Comment publication is non-blocking and does not change the gate verdict. +For pull requests from the same repository, the workflow creates or updates one sticky E2E result comment with the final result, selected profile, aggregate totals, active project and label filters, image identities, failure reason, and workflow-run link. Grant `pull-requests: write` in the caller workflow, or set `comment-on-pr: false` when comments are not wanted. Comment publication is non-blocking and does not change the gate result. The framework targets GitHub.com and uses `$/` references so the reusable workflow and its composite actions come from the same commit. That syntax is not available on GitHub Enterprise Server. From 79b4e395c4dfb7cf0fe1e9d1f1c9745e8ee33327 Mon Sep 17 00:00:00 2001 From: omermorad Date: Mon, 28 Sep 2026 16:30:56 +0300 Subject: [PATCH 09/11] feat: expose the E2E engine as a root action --- .github/actions/merge-e2e-results/action.yml | 6 + .github/workflows/e2e.yml | 117 ++++- .github/workflows/reusable-e2e.yml | 516 ------------------- README.md | 302 ++++------- action.yml | 246 +++++++++ e2e/ci.yml | 2 +- 6 files changed, 466 insertions(+), 723 deletions(-) delete mode 100644 .github/workflows/reusable-e2e.yml create mode 100644 action.yml diff --git a/.github/actions/merge-e2e-results/action.yml b/.github/actions/merge-e2e-results/action.yml index 7b406c6..b47902e 100644 --- a/.github/actions/merge-e2e-results/action.yml +++ b/.github/actions/merge-e2e-results/action.yml @@ -18,6 +18,12 @@ inputs: required: true outputs: + results-directory: + description: Absolute directory containing merged E2E results. + value: ${{ steps.stage.outputs.results-directory }} + report-directory: + description: Absolute directory containing the merged Playwright HTML report. + value: ${{ steps.stage.outputs.html-directory }} result: description: "Final result: passed, failed, or infrastructure-error." value: ${{ steps.assemble.outputs.result }} diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 4fa0887..b08967d 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -18,12 +18,12 @@ on: default: "" type: string labels: - description: Optional label override, one per line. + description: Optional tag override, one per line. required: false default: "" type: string label-match: - description: Optional label matching override. + description: Optional tag matching override. required: false default: profile type: choice @@ -42,14 +42,105 @@ concurrency: jobs: e2e: - name: E2E pipeline - uses: $/.github/workflows/reusable-e2e.yml - with: - profile: >- - ${{ github.event_name == 'pull_request' && 'pull-request' || - github.event_name == 'push' && 'main' || inputs.profile }} - projects: ${{ github.event_name == 'workflow_dispatch' && inputs.projects || '' }} - labels: ${{ github.event_name == 'workflow_dispatch' && inputs.labels || '' }} - label-match: >- - ${{ github.event_name == 'workflow_dispatch' && inputs.label-match != 'profile' && - inputs.label-match || '' }} + name: E2E Gate + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e912b2c3a1b1f917ef8b # v4 + + - id: e2e + name: Run E2E + continue-on-error: true + uses: $/ + with: + profile: >- + ${{ github.event_name == 'pull_request' && 'pull-request' || + github.event_name == 'push' && 'main' || inputs.profile }} + projects: ${{ github.event_name == 'workflow_dispatch' && inputs.projects || '' }} + labels: ${{ github.event_name == 'workflow_dispatch' && inputs.labels || '' }} + label-match: >- + ${{ github.event_name == 'workflow_dispatch' && inputs.label-match != 'profile' && + inputs.label-match || '' }} + + - name: Update pull request + if: always() && github.event_name == 'pull_request' + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + E2E_RESULT: ${{ steps.e2e.outputs.result }} + E2E_PRIMARY_FAILURE: ${{ steps.e2e.outputs.primary-failure }} + E2E_TOTAL: ${{ steps.e2e.outputs.total }} + E2E_PASSED: ${{ steps.e2e.outputs.passed }} + E2E_FAILED: ${{ steps.e2e.outputs.failed }} + E2E_SKIPPED: ${{ steps.e2e.outputs.skipped }} + E2E_PROFILE: ${{ steps.e2e.outputs.profile }} + E2E_PROJECTS: ${{ steps.e2e.outputs.projects }} + E2E_LABELS: ${{ steps.e2e.outputs.labels }} + E2E_LABEL_MATCH: ${{ steps.e2e.outputs.label-match }} + E2E_RUNNER_CACHE_HIT: ${{ steps.e2e.outputs.runner-cache-hit }} + E2E_RUNNER_IMAGE: ${{ steps.e2e.outputs.runner-image-name }} + E2E_RESULTS_ARTIFACT: ${{ steps.e2e.outputs.results-artifact }} + E2E_REPORT_IMAGE: ${{ steps.e2e.outputs.report-image-name }} + E2E_REPORT_ARTIFACT: ${{ steps.e2e.outputs.report-image-artifact }} + with: + script: | + const marker = ''; + const value = (name, fallback = 'n/a') => process.env[name] || fallback; + const list = (name) => value(name).split('\n').filter(Boolean).join(', '); + const result = value('E2E_RESULT', 'infrastructure-error'); + const icon = result === 'passed' ? '✅' : result === 'failed' ? '❌' : '⚠️'; + const cache = value('E2E_RUNNER_CACHE_HIT', 'false') === 'true' ? 'reused' : 'built'; + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const rows = [ + ['Gate', `${icon} ${result}`], + ['Tests', `${value('E2E_PASSED', '0')} passed · ${value('E2E_FAILED', '0')} failed · ${value('E2E_SKIPPED', '0')} skipped · ${value('E2E_TOTAL', '0')} total`], + ['Profile', value('E2E_PROFILE')], + ['Projects', list('E2E_PROJECTS')], + ['Tags', `${list('E2E_LABELS')} (${value('E2E_LABEL_MATCH')})`], + ['Runner', `${value('E2E_RUNNER_IMAGE')} (${cache})`], + ['Results', value('E2E_RESULTS_ARTIFACT')], + ['Report image', `${value('E2E_REPORT_IMAGE')} · ${value('E2E_REPORT_ARTIFACT')}`], + ['Run', `[Open workflow run](${runUrl})`], + ]; + const failure = process.env.E2E_PRIMARY_FAILURE + ? `\n> ${process.env.E2E_PRIMARY_FAILURE}\n` + : ''; + const body = [ + marker, + '## E2E test report', + '', + '| | |', + '|---|---|', + ...rows.map(([name, detail]) => `| ${name} | ${detail} |`), + failure, + ].join('\n'); + const comments = await github.paginate(github.rest.issues.listComments, { + ...context.repo, + issue_number: context.issue.number, + per_page: 100, + }); + const previous = comments.find((comment) => + comment.user?.type === 'Bot' && comment.body?.includes(marker)); + if (previous) { + await github.rest.issues.updateComment({ + ...context.repo, + comment_id: previous.id, + body, + }); + } else { + await github.rest.issues.createComment({ + ...context.repo, + issue_number: context.issue.number, + body, + }); + } + + - name: Preserve E2E result + if: always() + shell: bash + env: + E2E_STEP_OUTCOME: ${{ steps.e2e.outcome }} + run: | + if [[ "$E2E_STEP_OUTCOME" != "success" ]]; then + exit 1 + fi diff --git a/.github/workflows/reusable-e2e.yml b/.github/workflows/reusable-e2e.yml deleted file mode 100644 index c0c7b9e..0000000 --- a/.github/workflows/reusable-e2e.yml +++ /dev/null @@ -1,516 +0,0 @@ -name: Reusable E2E - -on: - workflow_call: - inputs: - profile: - description: Named execution profile from the E2E manifest. - required: false - default: pull-request - type: string - projects: - description: Optional newline-delimited project override; empty uses the profile. - required: false - default: "" - type: string - labels: - description: Optional newline-delimited label override; empty uses the profile. - required: false - default: "" - type: string - label-match: - description: Optional label matching override, either all or any. - required: false - default: "" - type: string - comment-on-pr: - description: Create or update one E2E report comment on same-repository pull requests. - required: false - default: true - type: boolean - outputs: - runner-image-name: - description: Exact local E2E runner image reference used by every shard. - value: ${{ jobs.build_runner.outputs.image-name }} - runner-image-id: - description: Content-addressed E2E runner Docker image ID. - value: ${{ jobs.build_runner.outputs.image-id }} - runner-image-artifact: - description: Artifact containing the portable E2E runner Docker archive. - value: ${{ jobs.build_runner.outputs.artifact-name }} - runner-cache-hit: - description: Whether the runner archive was restored by E2E content hash. - value: ${{ jobs.build_runner.outputs.cache-hit }} - runner-content-hash: - description: SHA-256 of the E2E runtime and build inputs used by the runner. - value: ${{ jobs.plan.outputs.runner-content-hash }} - profile: - description: Effective E2E execution profile. - value: ${{ jobs.plan.outputs.profile }} - report-image-name: - description: Local image reference for the packaged E2E HTML report. - value: ${{ jobs.merge.outputs.report-image-name }} - report-image-id: - description: Content-addressed E2E report Docker image ID. - value: ${{ jobs.merge.outputs.report-image-id }} - report-image-artifact: - description: Artifact containing the portable E2E report Docker archive. - value: ${{ jobs.merge.outputs.report-image-artifact }} - result: - description: Final E2E result. - value: ${{ jobs.e2e_gate.outputs.result }} - artifact-name: - description: Name of the merged results artifact. - value: ${{ jobs.merge.outputs.artifact-name }} - total: - description: Total number of tests in the merged report. - value: ${{ jobs.merge.outputs.total }} - passed: - description: Number of passing tests in the merged report. - value: ${{ jobs.merge.outputs.passed }} - failed: - description: Number of failing tests in the merged report. - value: ${{ jobs.merge.outputs.failed }} - skipped: - description: Number of skipped tests in the merged report. - value: ${{ jobs.merge.outputs.skipped }} - -permissions: - contents: read - -jobs: - plan: - name: Plan E2E shards - runs-on: ubuntu-latest - timeout-minutes: 2 - outputs: - working-directory: ${{ steps.plan.outputs.working-directory }} - dockerfile: ${{ steps.plan.outputs.dockerfile }} - playwright-config: ${{ steps.plan.outputs.playwright-config }} - compose-file: ${{ steps.plan.outputs.compose-file }} - base-url: ${{ steps.plan.outputs.base-url }} - projects: ${{ steps.plan.outputs.projects }} - labels: ${{ steps.plan.outputs.labels }} - label-match: ${{ steps.plan.outputs.label-match }} - shard-count: ${{ steps.plan.outputs.shard-count }} - matrix: ${{ steps.plan.outputs.matrix }} - artifact-retention-days: ${{ steps.plan.outputs.artifact-retention-days }} - runner-content-hash: ${{ steps.plan.outputs.runner-content-hash }} - profile: ${{ steps.plan.outputs.profile }} - steps: - - name: Check out caller repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - persist-credentials: false - - - id: plan - name: Validate E2E execution manifest - uses: $/.github/actions/plan-e2e - with: - profile: ${{ inputs.profile }} - projects: ${{ inputs.projects }} - labels: ${{ inputs.labels }} - label-match: ${{ inputs.label-match }} - - build_runner: - name: Resolve E2E runner image - needs: plan - runs-on: ubuntu-latest - timeout-minutes: 30 - concurrency: - group: e2e-runner-${{ github.event.pull_request.number || github.ref }}-${{ needs.plan.outputs.runner-content-hash }} - cancel-in-progress: false - outputs: - image-name: ${{ steps.reuse.outputs.image-name || steps.build.outputs.image-name }} - image-id: ${{ steps.reuse.outputs.image-id || steps.build.outputs.image-id }} - artifact-name: ${{ steps.upload-runner.outcome == 'success' && 'e2e-runner-image' || '' }} - cache-hit: ${{ steps.reuse.outcome == 'success' }} - steps: - - name: Check out caller repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - persist-credentials: false - - - id: runner-cache - name: Restore E2E runner archive - continue-on-error: true - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: .e2e-runner - key: e2e-runner-v1-${{ runner.os }}-${{ runner.arch }}-${{ needs.plan.outputs.runner-content-hash }} - - - id: reuse - name: Validate cached E2E runner image - if: steps.runner-cache.outcome == 'success' && steps.runner-cache.outputs.cache-hit == 'true' - continue-on-error: true - uses: $/.github/actions/validate-e2e-runner - with: - archive-path: .e2e-runner/playwright-e2e-runner.tar.zst - image-name: playwright-e2e-runner:e2e-${{ needs.plan.outputs.runner-content-hash }} - content-hash: ${{ needs.plan.outputs.runner-content-hash }} - - - id: build - name: Build E2E runner image - if: >- - always() && !cancelled() && steps.runner-cache.outcome != 'skipped' && - steps.reuse.outcome != 'success' - uses: $/.github/actions/build-e2e-runner - with: - working-directory: ${{ needs.plan.outputs.working-directory }} - dockerfile: ${{ needs.plan.outputs.dockerfile }} - image-name: playwright-e2e-runner:e2e-${{ needs.plan.outputs.runner-content-hash }} - archive-path: .e2e-runner/playwright-e2e-runner.tar.zst - content-hash: ${{ needs.plan.outputs.runner-content-hash }} - - - id: upload-runner - name: Upload portable E2E runner image - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: e2e-runner-image - path: ${{ steps.reuse.outputs.archive-path || steps.build.outputs.archive-path }} - if-no-files-found: error - compression-level: 0 - retention-days: ${{ needs.plan.outputs.artifact-retention-days }} - - shards: - name: E2E shard ${{ matrix.shard }}/${{ needs.plan.outputs.shard-count }} - needs: - - plan - - build_runner - runs-on: ubuntu-latest - timeout-minutes: 20 - strategy: - fail-fast: false - matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} - steps: - - name: Check out caller repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - persist-credentials: false - - - id: download-runner - name: Download E2E runner image - continue-on-error: true - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - name: ${{ needs.build_runner.outputs.artifact-name }} - path: runner-image - - - id: load-runner - name: Load E2E runner image - continue-on-error: true - shell: bash - run: docker load --input runner-image/playwright-e2e-runner.tar.zst - - - id: run - name: Run Playwright shard - continue-on-error: true - uses: $/.github/actions/run-e2e - with: - working-directory: ${{ needs.plan.outputs.working-directory }} - playwright-config: ${{ needs.plan.outputs.playwright-config }} - compose-file: ${{ needs.plan.outputs.compose-file }} - base-url: ${{ needs.plan.outputs.base-url }} - runner-image-name: ${{ needs.build_runner.outputs.image-name }} - runner-image-id: ${{ needs.build_runner.outputs.image-id }} - runner-download-outcome: ${{ steps.download-runner.outcome }} - runner-load-outcome: ${{ steps.load-runner.outcome }} - projects: ${{ needs.plan.outputs.projects }} - labels: ${{ needs.plan.outputs.labels }} - label-match: ${{ needs.plan.outputs.label-match }} - shard-index: ${{ matrix.shard }} - shard-total: ${{ needs.plan.outputs.shard-count }} - results-directory: test-results - - - name: Upload shard results - if: always() && steps.run.outputs.shard-directory != '' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: e2e-shard-${{ matrix.shard }} - path: ${{ steps.run.outputs.shard-directory }} - if-no-files-found: error - retention-days: ${{ needs.plan.outputs.artifact-retention-days }} - - - name: Preserve shard result - if: >- - always() && (steps.download-runner.outcome != 'success' || - steps.load-runner.outcome != 'success' || - steps.run.outcome != 'success') - shell: bash - run: | - echo "::error::E2E shard ${{ matrix.shard }} did not complete successfully" - exit 1 - - merge: - name: Merge E2E results - needs: - - plan - - build_runner - - shards - if: always() - runs-on: ubuntu-latest - timeout-minutes: 10 - outputs: - result: ${{ steps.publish.outputs.result }} - artifact-name: ${{ steps.publish.outputs.artifact-name }} - total: ${{ steps.publish.outputs.total }} - passed: ${{ steps.publish.outputs.passed }} - failed: ${{ steps.publish.outputs.failed }} - skipped: ${{ steps.publish.outputs.skipped }} - report-image-name: ${{ steps.package-report.outcome == 'success' && steps.package-report.outputs.image-name || '' }} - report-image-id: ${{ steps.package-report.outcome == 'success' && steps.package-report.outputs.image-id || '' }} - report-image-artifact: ${{ steps.package-report.outcome == 'success' && steps.package-report.outputs.artifact-name || '' }} - steps: - - id: download-runner - name: Download E2E runner image - continue-on-error: true - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - name: e2e-runner-image - path: runner-image - - - id: load-runner - name: Load E2E runner image - continue-on-error: true - shell: bash - run: docker load --input runner-image/playwright-e2e-runner.tar.zst - - - id: verify-runner - name: Verify immutable E2E runner image - continue-on-error: true - shell: bash - env: - E2E_RUNNER_IMAGE_NAME: ${{ needs.build_runner.outputs.image-name }} - E2E_RUNNER_IMAGE_ID: ${{ needs.build_runner.outputs.image-id }} - run: | - set -euo pipefail - if [[ -z "$E2E_RUNNER_IMAGE_NAME" ]]; then - echo "Expected E2E runner image name is empty" >&2 - exit 1 - fi - if [[ ! "$E2E_RUNNER_IMAGE_ID" =~ ^sha256:[a-f0-9]{64}$ ]]; then - echo "Invalid expected E2E runner image ID: $E2E_RUNNER_IMAGE_ID" >&2 - exit 1 - fi - actual_image_id="$(docker image inspect --format '{{.Id}}' "$E2E_RUNNER_IMAGE_NAME")" - if [[ "$actual_image_id" != "$E2E_RUNNER_IMAGE_ID" ]]; then - echo "E2E runner image ID mismatch: expected $E2E_RUNNER_IMAGE_ID, got $actual_image_id" >&2 - exit 1 - fi - - - id: download - name: Download shard results - if: always() - continue-on-error: true - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - pattern: e2e-shard-* - path: downloaded-shards - merge-multiple: false - - - id: merge - name: Merge E2E results - if: always() - continue-on-error: true - uses: $/.github/actions/merge-e2e-results - with: - download-directory: downloaded-shards - results-directory: e2e-results - expected-shards: ${{ needs.plan.outputs.shard-count }} - runner-image-id: ${{ needs.build_runner.outputs.image-id }} - - - id: package-report - name: Package E2E report image - if: always() - continue-on-error: true - uses: $/.github/actions/package-e2e-report - with: - report-directory: e2e-results/playwright-report - image-name: playwright-e2e-report:${{ github.run_id }}-${{ github.run_attempt }} - archive-directory: e2e-report-image - artifact-name: e2e-report-image - retention-days: ${{ needs.plan.outputs.artifact-retention-days }} - - - id: publish - name: Publish and enforce final results - if: always() - continue-on-error: true - uses: $/.github/actions/publish-e2e-results - with: - results-directory: e2e-results - expected-shards: ${{ needs.plan.outputs.shard-count }} - artifact-name: e2e-results - retention-days: ${{ needs.plan.outputs.artifact-retention-days }} - runner-download-outcome: ${{ steps.download-runner.outcome }} - runner-load-outcome: ${{ steps.load-runner.outcome }} - runner-verification-outcome: ${{ steps.verify-runner.outcome }} - shard-download-outcome: ${{ steps.download.outcome }} - merge-outcome: ${{ steps.merge.outcome }} - report-publication-outcome: ${{ steps.package-report.outcome }} - - - name: Preserve merge result - if: always() && steps.publish.outcome != 'success' - shell: bash - run: | - echo "::error::E2E results could not be completed successfully" - exit 1 - - e2e_gate: - name: E2E Gate - needs: - - plan - - shards - - merge - if: always() - runs-on: ubuntu-latest - timeout-minutes: 5 - outputs: - result: ${{ steps.finalize.outputs.result }} - primary-failure: ${{ steps.finalize.outputs.primary-failure }} - steps: - - id: download - name: Download final results - continue-on-error: true - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - name: e2e-results - path: final-results - - - id: finalize - name: Enforce final result - uses: $/.github/actions/finalize-e2e-result - with: - result-file: final-results/result.json - shard-job-result: ${{ needs.shards.result }} - merge-job-result: ${{ needs.merge.result }} - publication-result: ${{ steps.download.outcome }} - - pr_comment: - name: E2E PR report - needs: - - plan - - build_runner - - merge - - e2e_gate - if: >- - ${{ always() && inputs.comment-on-pr && github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name == github.repository }} - runs-on: ubuntu-latest - timeout-minutes: 2 - permissions: - pull-requests: write - steps: - - name: Create or update E2E report - continue-on-error: true - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 - env: - E2E_GATE_JOB_RESULT: ${{ needs.e2e_gate.result }} - E2E_GATE_RESULT: ${{ needs.e2e_gate.outputs.result }} - E2E_MERGE_RESULT: ${{ needs.merge.outputs.result }} - E2E_PRIMARY_FAILURE: ${{ needs.e2e_gate.outputs.primary-failure }} - E2E_TOTAL: ${{ needs.merge.outputs.total }} - E2E_PASSED: ${{ needs.merge.outputs.passed }} - E2E_FAILED: ${{ needs.merge.outputs.failed }} - E2E_SKIPPED: ${{ needs.merge.outputs.skipped }} - E2E_ARTIFACT_NAME: ${{ needs.merge.outputs.artifact-name }} - E2E_RUNNER_IMAGE_NAME: ${{ needs.build_runner.outputs.image-name }} - E2E_RUNNER_IMAGE_ID: ${{ needs.build_runner.outputs.image-id }} - E2E_RUNNER_IMAGE_ARTIFACT: ${{ needs.build_runner.outputs.artifact-name }} - E2E_RUNNER_CACHE_HIT: ${{ needs.build_runner.outputs.cache-hit }} - E2E_REPORT_IMAGE_NAME: ${{ needs.merge.outputs.report-image-name }} - E2E_REPORT_IMAGE_ID: ${{ needs.merge.outputs.report-image-id }} - E2E_REPORT_IMAGE_ARTIFACT: ${{ needs.merge.outputs.report-image-artifact }} - E2E_PROFILE: ${{ needs.plan.outputs.profile }} - E2E_PROJECTS: ${{ needs.plan.outputs.projects }} - E2E_LABELS: ${{ needs.plan.outputs.labels }} - E2E_LABEL_MATCH: ${{ needs.plan.outputs.label-match }} - with: - script: | - const marker = ''; - const clean = (value, fallback) => { - const normalized = String(value ?? '').trim(); - return normalized || fallback; - }; - const cell = (value) => String(value).replaceAll('|', '\\|').replaceAll('\n', '
'); - const list = (value, fallback) => { - const entries = String(value ?? '') - .split(/\r?\n/) - .map((entry) => entry.trim()) - .filter(Boolean); - return entries.length > 0 ? entries.map((entry) => `\`${entry}\``).join(', ') : fallback; - }; - - const gateResult = clean(process.env.E2E_GATE_JOB_RESULT, 'failure'); - const result = clean( - process.env.E2E_GATE_RESULT, - clean(process.env.E2E_MERGE_RESULT, gateResult === 'success' ? 'passed' : 'infrastructure-error'), - ); - const passed = result === 'passed'; - const status = passed ? '✅ Passed' : result === 'failed' ? '❌ Tests failed' : '⚠️ Infrastructure error'; - const total = clean(process.env.E2E_TOTAL, 'unknown'); - const passedCount = clean(process.env.E2E_PASSED, 'unknown'); - const failedCount = clean(process.env.E2E_FAILED, 'unknown'); - const skippedCount = clean(process.env.E2E_SKIPPED, 'unknown'); - const profile = clean(process.env.E2E_PROFILE, 'unknown'); - const projects = list(process.env.E2E_PROJECTS, 'All configured projects'); - const labels = list(process.env.E2E_LABELS, 'All labels'); - const labelMatch = clean(process.env.E2E_LABEL_MATCH, 'all'); - const artifactName = clean(process.env.E2E_ARTIFACT_NAME, 'e2e-results'); - const runnerImageName = clean(process.env.E2E_RUNNER_IMAGE_NAME, 'unavailable'); - const runnerImageId = clean(process.env.E2E_RUNNER_IMAGE_ID, 'unavailable'); - const runnerImageArtifact = clean(process.env.E2E_RUNNER_IMAGE_ARTIFACT, 'unavailable'); - const runnerSource = runnerImageName === 'unavailable' - ? 'unavailable' - : clean(process.env.E2E_RUNNER_CACHE_HIT, '') === 'true' ? 'cache hit' : 'built'; - const reportImageName = clean(process.env.E2E_REPORT_IMAGE_NAME, 'unavailable'); - const reportImageId = clean(process.env.E2E_REPORT_IMAGE_ID, 'unavailable'); - const reportImageArtifact = clean(process.env.E2E_REPORT_IMAGE_ARTIFACT, 'unavailable'); - const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; - - const lines = [ - marker, - '## E2E test report', - '', - '| Signal | Result |', - '| --- | --- |', - `| Gate | ${status} |`, - `| Tests | ${cell(`${passedCount} passed · ${failedCount} failed · ${skippedCount} skipped · ${total} total`)} |`, - `| Profile | ${cell(`\`${profile}\``)} |`, - `| Projects | ${cell(projects)} |`, - `| Labels | ${cell(`${labels} · match ${labelMatch}`)} |`, - '| SUT | Docker Compose, isolated per shard |', - `| Runner image | ${cell(`\`${runnerImageName}\` · \`${runnerImageId}\` · \`${runnerImageArtifact}\` · ${runnerSource}`)} |`, - `| Report image | ${cell(`\`${reportImageName}\` · \`${reportImageId}\` · \`${reportImageArtifact}\``)} |`, - `| Results | [Open workflow run](${runUrl}) · \`${cell(artifactName)}\` |`, - ]; - - const primaryFailure = clean(process.env.E2E_PRIMARY_FAILURE, ''); - if (!passed && primaryFailure) { - lines.push('', `> ${primaryFailure.replaceAll('\n', '\n> ')}`); - } - lines.push('', `_Updated from run ${context.runId}._`); - const body = lines.join('\n'); - - const comments = await github.paginate(github.rest.issues.listComments, { - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - per_page: 100, - }); - const existing = comments.find((comment) => comment.body?.includes(marker)); - - if (existing) { - await github.rest.issues.updateComment({ - owner: context.repo.owner, - repo: context.repo.repo, - comment_id: existing.id, - body, - }); - } else { - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - body, - }); - } diff --git a/README.md b/README.md index 961bb40..8410cba 100644 --- a/README.md +++ b/README.md @@ -1,106 +1,106 @@ -# Playwright E2E CI Framework +# Playwright E2E -A reusable GitHub Actions pipeline for Playwright tests against a Dockerized system under test (SUT), with retained test results and one fail-closed gate. +A portable GitHub Action and lightweight framework for running Playwright against a Dockerized system under test (SUT). -This repository demonstrates the framework with a root Docker Compose service. The service exposes a health endpoint and echoes requests; Playwright tests it through API and Chromium clients. +It keeps the application repository in control of its tests and services while providing a consistent CI engine: deterministic runner images, smart reuse, SUT lifecycle management, test filtering, Playwright reports, traces, portable artifacts, and a fail-closed result. ```text pull request / main / manual | - plan + hash runner inputs + validate e2e/ci.yml | - serialize identical hashes + hash E2E runner inputs | +------+------+ | | validated cache cache miss - | pure runner build + | build runner +------+------+ | - portable runner archive + start Dockerized SUT | - +------+------+ - | | - shard 1 shard 2 - load runner load runner - compose SUT compose SUT - run tests run tests - test results test results - | | - +------+------+ - | - merge test reports + Playwright projects + tags | - package report image + results + traces | - portable report archive + HTML report + report image | E2E Gate - | - sticky PR test report -``` - -## Run the example locally - -Prerequisites: Docker, Node.js 22, and Corepack. - -Install the E2E dependencies and Chromium once: - -```bash -corepack enable -corepack prepare pnpm@10.26.2 --activate -pnpm --dir e2e install --frozen-lockfile -pnpm --dir e2e install:browsers ``` -Start the SUT, run Playwright, and then remove the Compose resources: +## Start from the template -```bash -docker compose -f compose.e2e.yml up --detach --build --wait -BASE_URL=http://127.0.0.1:4173 pnpm --dir e2e test -docker compose -f compose.e2e.yml down --volumes --remove-orphans -``` +The quickest path is [codotech/playwright-e2e-starter](https://github.com/codotech/playwright-e2e-starter). Create a repository from that template, replace its example SUT and tests, then adapt the profiles in `e2e/ci.yml`. -Always run the final command, including after a failed test. Use `pnpm --dir e2e test:ci` for the CI reporters and `pnpm --dir e2e test:static` for a type check. +The template owns repository policy: triggers, permissions, concurrency, checkout, the sticky pull-request comment, and preservation of the action result. This repository owns the portable execution engine. -## Use the reusable workflow +## Use the action -Keep the Playwright package in `e2e/` and the Compose file at the repository root. Pin the reusable workflow to a full release commit SHA: +Check out the caller repository first, then invoke the root action. Pin it to a full commit SHA: ```yaml -name: E2E - -on: - pull_request: - permissions: contents: read pull-requests: write -jobs: - e2e: - uses: codotech/playwright-e2e-ci-framework/.github/workflows/reusable-e2e.yml@<40-character-release-commit-sha> +steps: + - uses: actions/checkout@de0fac2e4500dabe0009e912b2c3a1b1f917ef8b + + - id: e2e + uses: codotech/playwright-e2e@<40-character-commit-sha> with: profile: pull-request ``` -Replace the marker with the immutable SHA you adopt. The workflow needs no inherited secrets. Pull-request comments use the explicitly granted permission shown above. +There is intentionally no release tag yet. Adopt an immutable commit only after reviewing it; the starter contains a working pinned example. + +### Inputs | Input | Default | Purpose | | --- | --- | --- | -| `profile` | `pull-request` | Named selection and execution policy | -| `projects` | empty | Optional project override, one per line | -| `labels` | empty | Optional label override, one per line | -| `label-match` | empty | Optional `all` or `any` override | -| `comment-on-pr` | `true` | Update one E2E report comment on same-repository pull requests | +| `profile` | `pull-request` | Named execution profile from `e2e/ci.yml` | +| `projects` | empty | Optional Playwright project override, one per line | +| `labels` | empty | Optional Playwright tag override, one per line | +| `label-match` | empty | Optional `all` or `any` tag matching override | + +Projects and tags are independent filters. Projects select configured Playwright variants. Tags select tests through Playwright's `--grep`. The action runs the intersection. + +### Outputs + +| Output | Meaning | +| --- | --- | +| `result` | `passed`, `failed`, or `infrastructure-error` | +| `primary-failure` | Main failure message, empty on success | +| `total`, `passed`, `failed`, `skipped` | Aggregate test counts | +| `profile`, `projects`, `labels`, `label-match` | Effective selection | +| `runner-cache-hit` | Whether a validated runner archive was reused | +| `runner-content-hash` | SHA-256 identity of runner inputs | +| `runner-image-name`, `runner-image-id`, `runner-image-artifact` | Exact portable runner image | +| `results-directory`, `results-artifact` | Merged machine-readable and Playwright results | +| `report-directory` | Generated Playwright HTML report | +| `report-image-name`, `report-image-id`, `report-image-artifact` | Portable image serving the HTML report | -The workflow exposes the final result, merged test totals, and the exact names and IDs of its portable images. The image outputs are listed below. +The action fails at the end when tests fail or infrastructure is incomplete. A caller that needs to publish a pull-request comment first can set `continue-on-error: true`, publish the comment from the outputs, and add a final step that fails when the action step outcome is not `success`. The starter demonstrates this pattern. -## Configure execution policy +## Repository contract -Keep CI policy in `e2e/ci.yml`: +The caller keeps these files: -The manifest path is intentionally fixed to `e2e/ci.yml`, matching the runner build context exclusion and keeping execution-only policy outside the runner image. +```text +. ++-- compose.e2e.yml ++-- e2e/ +| +-- ci.yml +| +-- Dockerfile +| +-- runner-entrypoint.mjs +| +-- package.json +| +-- pnpm-lock.yaml +| +-- playwright.config.ts +| +-- tests/ ++-- sut/ # example only; use your real services +``` + +`e2e/ci.yml` is the CI contract: ```yaml version: 1 @@ -116,7 +116,7 @@ sut: baseUrl: http://127.0.0.1:4173 execution: - shards: 2 + shards: 1 artifactRetentionDays: 10 profiles: @@ -131,160 +131,76 @@ profiles: labelMatch: all ``` -The manifest defines which configured tests CI selects. `playwright.config.ts` remains the source of truth for what each Playwright project means: browser, device, test matching, dependencies, and runtime behavior. The plan fails before building anything when the manifest has unknown keys, invalid paths, an unknown profile, invalid labels, or unsafe values. - -## Reuse identical runner images - -The runner cache key is: - -```text -e2e-runner-v1--- -``` - -The content hash covers the normalized `runner.dockerfile` selection plus every Git-tracked or non-ignored untracked file under the E2E directory, except the execution manifest itself. Paths, file types, Unix modes, file bytes, and symlink targets contribute to the hash. +The root action is deliberately a single GitHub job. Playwright can still use multiple workers inside the runner container. Keep `execution.shards: 1`; repository-level job matrices can be added later by the caller without changing the portable engine. -The identity is deliberately conservative around runner inputs: +`playwright.config.ts` defines projects, browser/device settings, matching, dependencies, reporters, and runtime behavior. `e2e/ci.yml` selects what CI runs. Unknown keys, invalid paths, unsafe values, nonexistent projects, or malformed tags fail before the SUT starts. -- A test, fixture, dependency, Playwright configuration file, runner entrypoint, Dockerfile, Docker-ignore rule, or selected runner Dockerfile change creates a new identity. -- Profiles, project and label selection, Playwright config selection, shard count, retention, and SUT policy remain outside the runner identity. They can change what runs without rebuilding identical image bytes; the selected config file's contents are still covered by the E2E tree hash. -- A service-only change reuses the runner but still rebuilds the SUT and runs the E2E suite. +## What rebuilds the runner -The workflow owns cache reuse. On an exact cache hit, a separate validation action checks the metadata content hash, image reference and ID, archive name, byte size, and SHA-256 before reuse. `runner-cache-hit` is `true` only after that validation succeeds. A missing, evicted, unavailable, or invalid cache runs the build action, which removes stale output files, performs a fresh Docker build, and writes a new archive plus metadata; it contains no cache-reuse branch. Invalid cached bytes are never trusted. +The runner identity covers every Git-tracked or non-ignored untracked entry below `e2e/`, except `e2e/ci.yml`. Paths, file types, Unix modes, file bytes, and symlink targets all contribute. -Runner resolution uses the concurrency group `e2e-runner--` with cancellation disabled. Identical runner requests in the same change stream resolve one at a time, while unrelated pull requests cannot cancel each other's pending runner job. Different content hashes resolve independently. Every run still uploads its resolved archive as `e2e-runner-image`, so shards consume a run-scoped artifact rather than the shared cache directly. +A new runner is built when tests, fixtures, dependencies, the lockfile, Playwright configuration, runner entrypoint, Dockerfile, or Docker-ignore rules change. Profiles, filters, retention, and service-only changes reuse the same runner bytes while still running the suite against the newly built SUT. -## Select projects and labels +On an exact cache hit, the action validates the archive metadata, image reference, content hash, byte size, SHA-256, and image ID before using it. A missing, evicted, or invalid cache causes a clean rebuild. -A [Playwright project](https://playwright.dev/docs/test-projects) is a configured execution variant, such as a browser, device, authentication state, or test group. Each `projects` line becomes an exact `--project` selection. +## Test lifecycle -A framework `label` means a [Playwright tag](https://playwright.dev/docs/test-annotations#tag-tests), not a project. Every label must start with `@`; the workflow converts the labels into one `--grep` expression. +The action performs one fail-closed sequence: -For example, a manual run can select `regression` and optionally override it with multiline values: +1. Validate configuration and resolve the requested profile. +2. Reuse or build the content-addressed Playwright runner. +3. Build and start the SUT with Docker Compose and wait for health checks. +4. Run the selected Playwright projects and tags. +5. Capture results, traces, screenshots, SUT logs, and teardown logs. +6. Tear down containers, volumes, and networks even after failure. +7. Build the HTML report and portable report image. +8. Upload artifacts and enforce the final result. -```text -profile: regression +A Playwright failure remains the primary failure even if later log collection or cleanup also fails. Missing results, runner identity problems, unhealthy services, or publication failures produce `infrastructure-error`; they never turn a run green. -projects: -api -chromium +## Artifacts -labels: -@smoke -@browser - -label-match: all -``` - -Pull requests use the `pull-request` profile, pushes to `main` use the `main` profile, and manual runs choose a profile explicitly. Manual project, label, and match values override only the selected profile fields that are provided. - -`all` requires a test to carry both `@smoke` and `@browser`. `any` accepts a test carrying either tag. Blank lines are ignored. An empty manual field keeps the profile value; `labels: []` in a profile disables label filtering for that profile. - -Projects and labels are independent. The selected tests are the intersection of both filters, and Playwright shards that result afterward: - -```text -configured tests - | - selected projects - | - matching tags - | - shard 1 + shard 2 -``` - -The same values can be supplied by another workflow with YAML block scalars: - -```yaml -with: - projects: | - api - chromium - labels: | - @smoke - @browser - label-match: any -``` - -## SUT lifecycle in CI - -Each shard runs on its own runner and performs the full lifecycle: - -1. Validate the Compose file and base URL. -2. Build and start the SUT with `docker compose up --detach --build --wait`. -3. Run Playwright with `BASE_URL` set to the ready service. -4. Capture startup, service, and teardown logs. -5. Run `docker compose down --volumes --remove-orphans`, even after failure. - -A startup or cleanup failure is an infrastructure error. A Playwright failure remains the primary failure when log collection or cleanup also fails. - -## Test results and gate +| Artifact | Contents | +| --- | --- | +| `e2e-results` | Final JSON result, CTRF report, Playwright HTML report, test attachments, SUT logs, and teardown logs | +| `e2e-runner-image` | Compressed Docker archive for the exact runner used by the run | +| `e2e-report-image` | Compressed Docker archive serving the HTML report on port 8080 | -Every shard uploads its status, Compose logs, blob report, CTRF result, and Playwright test results. The merged test-results artifact is named `e2e-results` and contains: +Download and load a report image: -```text -e2e-results/ -+-- playwright-report/ -+-- ctrf-report.json -+-- shard-status/ -+-- result.json +```bash +docker load --input e2e-report-image/playwright-e2e-report.tar.zst +docker run --rm -p 8080:8080 "$REPORT_IMAGE_NAME" ``` -The same immutable runner image merges the shard blobs; pull-request dependencies are never installed directly on the GitHub host. `E2E Gate` fails for runner transport or identity problems, a failed or cancelled shard, unhealthy SUT lifecycle, missing test results, report merge failure, report-image publication failure, test-results artifact publication failure, or a non-passing result. Publishing reports cannot turn a failed run green. - -## Load the runner and report images +Open `http://127.0.0.1:8080`. Traces and other Playwright attachments are retained inside the report when a test creates them. -A healthy run publishes two Docker archives in addition to the merged test-results artifact. If report packaging or publication fails, the report outputs are empty and the final result is an infrastructure error. - -| Artifact | Archive inside it | Purpose | -| --- | --- | --- | -| `e2e-runner-image` | `playwright-e2e-runner.tar.zst` | Exact Playwright runner used by every shard | -| `e2e-report-image` | `playwright-e2e-report.tar.zst` | Merged HTML report served on port 8080 | +The images are workflow artifacts, not registry publications. Artifact retention applies and local image names are not remotely pullable. -The workflow exposes these values: +## Run this example locally -| Output | Meaning | -| --- | --- | -| `runner-image-name` | E2E-content-hash-tagged runner image name | -| `runner-image-id` | Content-addressed local runner image ID | -| `runner-image-artifact` | Runner archive artifact name | -| `runner-cache-hit` | Whether the content-addressed runner archive was reused | -| `runner-content-hash` | SHA-256 identity of the E2E runner inputs | -| `profile` | Effective execution profile | -| `report-image-name` | Run-tagged report image name; empty when unavailable | -| `report-image-id` | Content-addressed local report image ID; empty when unavailable | -| `report-image-artifact` | Report archive artifact name; empty when unavailable | - -After downloading the runner artifact, load it and run the exact test environment again. Replace the image variable with the corresponding workflow output: +Prerequisites: Docker, Node.js 22, Corepack, and pnpm 10.26.2. ```bash -docker load --input e2e-runner-image/playwright-e2e-runner.tar.zst -mkdir -p e2e-output -docker run --rm --init --ipc=host \ - --read-only \ - --tmpfs /tmp:rw,nosuid,nodev,size=512m \ - --cap-drop=ALL \ - --security-opt=no-new-privileges:true \ - --pids-limit=1024 \ - --user "$(id -u):$(id -g)" \ - --add-host=host.docker.internal:host-gateway \ - -e BASE_URL=http://host.docker.internal:4173 \ - -e HOME=/tmp \ - -v "$PWD/e2e-output:/test-output" \ - "$RUNNER_IMAGE_ID" +corepack enable +corepack prepare pnpm@10.26.2 --activate +pnpm --dir e2e install --frozen-lockfile +pnpm --dir e2e install:browsers +docker compose -f compose.e2e.yml up --detach --build --wait +BASE_URL=http://127.0.0.1:4173 pnpm --dir e2e test +docker compose -f compose.e2e.yml down --volumes --remove-orphans ``` -Append normal Playwright arguments, such as `--project=chromium --grep=@smoke`, after the image reference. - -Load and serve the merged report in the same way: +Always run the final cleanup command, including after a failed test. Run `pnpm --dir e2e test:static` for the TypeScript check. -```bash -docker load --input e2e-report-image/playwright-e2e-report.tar.zst -docker run --rm -p 8080:8080 "$REPORT_IMAGE_NAME" -``` +## Platform support -Open `http://127.0.0.1:8080`. The report image contains the complete merged HTML directory. Retained Playwright traces and other report attachments remain available from the report when the test produced them. +- GitHub.com hosted or self-hosted Linux runners with Docker and Docker Compose +- GitHub Actions runner 2.336.0 or newer, required for repository-relative `$/` action references +- One checked-out application repository per job -These images are currently GitHub artifacts, not registry publications. Download and `docker load` the archives before use; artifact retention still applies, and the local image names are not remotely pullable. +No inherited secrets are required by the action. The SUT may use repository or environment secrets supplied by its own workflow. -For pull requests from the same repository, the workflow creates or updates one sticky E2E result comment with the final result, selected profile, aggregate totals, active project and label filters, image identities, failure reason, and workflow-run link. Grant `pull-requests: write` in the caller workflow, or set `comment-on-pr: false` when comments are not wanted. Comment publication is non-blocking and does not change the gate result. +## License -The framework targets GitHub.com and uses `$/` references so the reusable workflow and its composite actions come from the same commit. That syntax is not available on GitHub Enterprise Server. +[Apache-2.0](LICENSE) diff --git a/action.yml b/action.yml new file mode 100644 index 0000000..73deb73 --- /dev/null +++ b/action.yml @@ -0,0 +1,246 @@ +name: Playwright E2E +author: Codotech +description: Run a Dockerized system under test with a portable Playwright runner and publish the test results. + +branding: + icon: check-circle + color: green + +inputs: + profile: + description: Named execution profile from e2e/ci.yml. + required: false + default: pull-request + projects: + description: Optional newline-delimited Playwright project override. + required: false + default: "" + labels: + description: Optional newline-delimited Playwright tag override. + required: false + default: "" + label-match: + description: Optional label matching override, either all or any. + required: false + default: "" + +outputs: + result: + description: "Final result: passed, failed, or infrastructure-error." + value: ${{ steps.finalize.outputs.result }} + primary-failure: + description: Primary failure message, empty when the run passed. + value: ${{ steps.finalize.outputs.primary-failure }} + total: + description: Total number of tests. + value: ${{ steps.publish.outputs.total }} + passed: + description: Number of passing tests. + value: ${{ steps.publish.outputs.passed }} + failed: + description: Number of failing tests. + value: ${{ steps.publish.outputs.failed }} + skipped: + description: Number of skipped tests. + value: ${{ steps.publish.outputs.skipped }} + profile: + description: Effective execution profile. + value: ${{ steps.plan.outputs.profile }} + projects: + description: Effective newline-delimited Playwright projects. + value: ${{ steps.plan.outputs.projects }} + labels: + description: Effective newline-delimited Playwright tags. + value: ${{ steps.plan.outputs.labels }} + label-match: + description: Effective label matching mode. + value: ${{ steps.plan.outputs.label-match }} + runner-cache-hit: + description: Whether a validated runner archive was reused. + value: ${{ steps.reuse.outcome == 'success' }} + runner-content-hash: + description: SHA-256 identity of the runner inputs. + value: ${{ steps.plan.outputs.runner-content-hash }} + runner-image-name: + description: Content-hash-tagged runner image name. + value: ${{ steps.reuse.outputs.image-name || steps.build.outputs.image-name }} + runner-image-id: + description: Content-addressed runner image ID. + value: ${{ steps.reuse.outputs.image-id || steps.build.outputs.image-id }} + runner-image-artifact: + description: Artifact containing the portable runner image. + value: ${{ steps.upload-runner.outcome == 'success' && 'e2e-runner-image' || '' }} + results-directory: + description: Absolute directory containing merged test results. + value: ${{ steps.merge.outputs.results-directory }} + results-artifact: + description: Artifact containing merged test results. + value: ${{ steps.publish.outputs.artifact-name }} + report-directory: + description: Absolute directory containing the Playwright HTML report. + value: ${{ steps.merge.outputs.report-directory }} + report-image-name: + description: Run-tagged report image name. + value: ${{ steps.package-report.outputs.image-name }} + report-image-id: + description: Content-addressed report image ID. + value: ${{ steps.package-report.outputs.image-id }} + report-image-artifact: + description: Artifact containing the portable report image. + value: ${{ steps.package-report.outputs.artifact-name }} + +runs: + using: composite + steps: + - id: plan + name: Validate E2E configuration + uses: $/.github/actions/plan-e2e + with: + profile: ${{ inputs.profile }} + projects: ${{ inputs.projects }} + labels: ${{ inputs.labels }} + label-match: ${{ inputs.label-match }} + + - name: Require single-job execution + if: steps.plan.outputs.shard-count != '1' + shell: bash + run: | + echo "execution.shards must be 1 when using the root action" >&2 + exit 1 + + - id: runner-cache + name: Restore E2E runner archive + continue-on-error: true + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: .e2e-runner + key: e2e-runner-v1-${{ runner.os }}-${{ runner.arch }}-${{ steps.plan.outputs.runner-content-hash }} + + - id: reuse + name: Validate cached E2E runner image + if: steps.runner-cache.outcome == 'success' && steps.runner-cache.outputs.cache-hit == 'true' + continue-on-error: true + uses: $/.github/actions/validate-e2e-runner + with: + archive-path: .e2e-runner/playwright-e2e-runner.tar.zst + image-name: playwright-e2e-runner:e2e-${{ steps.plan.outputs.runner-content-hash }} + content-hash: ${{ steps.plan.outputs.runner-content-hash }} + + - id: build + name: Build E2E runner image + if: >- + always() && !cancelled() && steps.runner-cache.outcome != 'skipped' && + steps.reuse.outcome != 'success' + uses: $/.github/actions/build-e2e-runner + with: + working-directory: ${{ steps.plan.outputs.working-directory }} + dockerfile: ${{ steps.plan.outputs.dockerfile }} + image-name: playwright-e2e-runner:e2e-${{ steps.plan.outputs.runner-content-hash }} + archive-path: .e2e-runner/playwright-e2e-runner.tar.zst + content-hash: ${{ steps.plan.outputs.runner-content-hash }} + + - id: upload-runner + name: Upload portable E2E runner image + if: always() && (steps.reuse.outcome == 'success' || steps.build.outcome == 'success') + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: e2e-runner-image + path: ${{ steps.reuse.outputs.archive-path || steps.build.outputs.archive-path }} + if-no-files-found: error + compression-level: 0 + retention-days: ${{ steps.plan.outputs.artifact-retention-days }} + + - id: load-runner + name: Load E2E runner image + if: always() && (steps.reuse.outcome == 'success' || steps.build.outcome == 'success') + continue-on-error: true + shell: bash + run: docker load --input "${{ steps.reuse.outputs.archive-path || steps.build.outputs.archive-path }}" + + - id: verify-runner + name: Verify immutable E2E runner image + if: always() && steps.load-runner.outcome == 'success' + continue-on-error: true + shell: bash + env: + E2E_RUNNER_IMAGE_NAME: ${{ steps.reuse.outputs.image-name || steps.build.outputs.image-name }} + E2E_RUNNER_IMAGE_ID: ${{ steps.reuse.outputs.image-id || steps.build.outputs.image-id }} + run: | + set -euo pipefail + actual_id="$(docker image inspect --format '{{.Id}}' "$E2E_RUNNER_IMAGE_NAME")" + if [[ "$actual_id" != "$E2E_RUNNER_IMAGE_ID" ]]; then + echo "Loaded runner image ID $actual_id does not match $E2E_RUNNER_IMAGE_ID" >&2 + exit 1 + fi + + - id: run + name: Run Playwright E2E + if: always() && steps.plan.outcome == 'success' + continue-on-error: true + uses: $/.github/actions/run-e2e + with: + working-directory: ${{ steps.plan.outputs.working-directory }} + playwright-config: ${{ steps.plan.outputs.playwright-config }} + compose-file: ${{ steps.plan.outputs.compose-file }} + base-url: ${{ steps.plan.outputs.base-url }} + runner-image-name: ${{ steps.reuse.outputs.image-name || steps.build.outputs.image-name }} + runner-image-id: ${{ steps.reuse.outputs.image-id || steps.build.outputs.image-id }} + runner-download-outcome: ${{ (steps.reuse.outcome == 'success' || steps.build.outcome == 'success') && 'success' || 'failure' }} + runner-load-outcome: ${{ steps.load-runner.outcome }} + projects: ${{ steps.plan.outputs.projects }} + labels: ${{ steps.plan.outputs.labels }} + label-match: ${{ steps.plan.outputs.label-match }} + shard-index: "1" + shard-total: "1" + results-directory: test-results + + - id: merge + name: Merge E2E results + if: always() && steps.plan.outcome == 'success' + continue-on-error: true + uses: $/.github/actions/merge-e2e-results + with: + download-directory: ${{ steps.plan.outputs.working-directory }}/test-results + results-directory: e2e-results + expected-shards: "1" + runner-image-id: ${{ steps.reuse.outputs.image-id || steps.build.outputs.image-id }} + + - id: package-report + name: Package E2E report image + if: always() && steps.merge.outcome == 'success' + continue-on-error: true + uses: $/.github/actions/package-e2e-report + with: + report-directory: e2e-results/playwright-report + image-name: playwright-e2e-report:${{ github.run_id }}-${{ github.run_attempt }} + archive-directory: e2e-report-image + artifact-name: e2e-report-image + retention-days: ${{ steps.plan.outputs.artifact-retention-days }} + + - id: publish + name: Publish E2E results + if: always() && steps.plan.outcome == 'success' + continue-on-error: true + uses: $/.github/actions/publish-e2e-results + with: + results-directory: e2e-results + expected-shards: "1" + artifact-name: e2e-results + retention-days: ${{ steps.plan.outputs.artifact-retention-days }} + runner-download-outcome: ${{ (steps.reuse.outcome == 'success' || steps.build.outcome == 'success') && 'success' || 'failure' }} + runner-load-outcome: ${{ steps.load-runner.outcome }} + runner-verification-outcome: ${{ steps.verify-runner.outcome }} + shard-download-outcome: ${{ steps.run.outputs.shard-directory != '' && 'success' || 'failure' }} + merge-outcome: ${{ steps.merge.outcome }} + report-publication-outcome: ${{ steps.package-report.outcome }} + + - id: finalize + name: Enforce final E2E result + if: always() + uses: $/.github/actions/finalize-e2e-result + with: + result-file: ${{ steps.publish.outputs.result-file || 'e2e-results/result.json' }} + shard-job-result: ${{ steps.run.outcome }} + merge-job-result: ${{ (steps.merge.outcome == 'success' && steps.publish.outcome == 'success') && 'success' || 'failure' }} + publication-result: ${{ steps.upload-runner.outcome }} diff --git a/e2e/ci.yml b/e2e/ci.yml index 3fba989..b9ade55 100644 --- a/e2e/ci.yml +++ b/e2e/ci.yml @@ -11,7 +11,7 @@ sut: baseUrl: http://127.0.0.1:4173 execution: - shards: 2 + shards: 1 artifactRetentionDays: 10 profiles: From d6547a91e259d285caa23a626fd207ac6747ef24 Mon Sep 17 00:00:00 2001 From: omermorad Date: Mon, 28 Sep 2026 16:33:08 +0300 Subject: [PATCH 10/11] fix: pin the checkout action to a published commit --- .github/workflows/e2e.yml | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index b08967d..3b53870 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -47,7 +47,7 @@ jobs: timeout-minutes: 45 steps: - name: Check out repository - uses: actions/checkout@de0fac2e4500dabe0009e912b2c3a1b1f917ef8b # v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - id: e2e name: Run E2E diff --git a/README.md b/README.md index 8410cba..83a880e 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ permissions: pull-requests: write steps: - - uses: actions/checkout@de0fac2e4500dabe0009e912b2c3a1b1f917ef8b + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 - id: e2e uses: codotech/playwright-e2e@<40-character-commit-sha> From 7a9a37c5ead24434a15d7c4682ce488e8a4a9eb4 Mon Sep 17 00:00:00 2001 From: omermorad Date: Mon, 28 Sep 2026 16:47:18 +0300 Subject: [PATCH 11/11] chore: use the current checkout runtime --- .github/workflows/e2e.yml | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 3b53870..496d6f3 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -47,7 +47,7 @@ jobs: timeout-minutes: 45 steps: - name: Check out repository - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - id: e2e name: Run E2E diff --git a/README.md b/README.md index 83a880e..d154c7f 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ permissions: pull-requests: write steps: - - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - id: e2e uses: codotech/playwright-e2e@<40-character-commit-sha>