diff --git a/.github/actions/build-e2e-runner/action.yml b/.github/actions/build-e2e-runner/action.yml new file mode 100644 index 0000000..c094ec2 --- /dev/null +++ b/.github/actions/build-e2e-runner/action.yml @@ -0,0 +1,82 @@ +name: Build E2E Runner +description: Build and archive a content-addressed Playwright runner image. + +inputs: + working-directory: + description: Repository-relative Docker build context containing the E2E runner. + required: false + default: e2e + dockerfile: + description: Dockerfile path relative to working-directory. + required: false + default: Dockerfile + image-name: + description: Complete local Docker image reference, including an immutable tag. + required: true + archive-path: + description: Repository-relative output path for the zstd-compressed Docker image archive. + required: false + default: .e2e-runner/playwright-e2e-runner.tar.zst + content-hash: + description: SHA-256 of the E2E source tree used as the cache identity. + required: true + +outputs: + image-name: + description: Local Docker image reference saved in the archive. + value: ${{ steps.build.outputs.image-name }} + image-id: + description: Content-addressed Docker image ID. + value: ${{ steps.build.outputs.image-id }} + archive-path: + description: Absolute path to the portable .tar.zst Docker image archive. + value: ${{ steps.build.outputs.archive-path }} + archive-sha256: + description: SHA-256 digest of the compressed Docker archive. + value: ${{ steps.build.outputs.archive-sha256 }} + +runs: + using: composite + steps: + - id: build + name: Build and save runner image + shell: bash + env: + E2E_WORKING_DIRECTORY: ${{ inputs.working-directory }} + E2E_DOCKERFILE: ${{ inputs.dockerfile }} + E2E_RUNNER_IMAGE: ${{ inputs.image-name }} + E2E_ARCHIVE_PATH: ${{ inputs.archive-path }} + E2E_CONTENT_HASH: ${{ inputs.content-hash }} + run: | + set -euo pipefail + + eval "$(node "$GITHUB_ACTION_PATH/prepare-build.mjs" \ + "$GITHUB_WORKSPACE" \ + "$E2E_WORKING_DIRECTORY" \ + "$E2E_DOCKERFILE" \ + "$E2E_RUNNER_IMAGE" \ + "$E2E_ARCHIVE_PATH")" + + docker build \ + --file "$E2E_ABSOLUTE_DOCKERFILE" \ + --tag "$E2E_RUNNER_IMAGE" \ + "$E2E_BUILD_CONTEXT" + + image_id="$(docker image inspect --format '{{.Id}}' "$E2E_RUNNER_IMAGE")" + if [[ ! "$image_id" =~ ^sha256:[a-f0-9]{64}$ ]]; then + echo "Unexpected Docker image ID: $image_id" >&2 + exit 1 + fi + + command -v zstd > /dev/null + docker save "$E2E_RUNNER_IMAGE" | \ + zstd -T0 -3 --force -o "$E2E_ABSOLUTE_ARCHIVE_PATH" + test -s "$E2E_ABSOLUTE_ARCHIVE_PATH" + + node "$GITHUB_ACTION_PATH/runner-archive.mjs" \ + "$GITHUB_WORKSPACE" \ + "$E2E_ARCHIVE_PATH" \ + "$E2E_RUNNER_IMAGE" \ + "$E2E_CONTENT_HASH" \ + "$GITHUB_OUTPUT" \ + "$image_id" diff --git a/.github/actions/build-e2e-runner/prepare-build.mjs b/.github/actions/build-e2e-runner/prepare-build.mjs new file mode 100644 index 0000000..a091e13 --- /dev/null +++ b/.github/actions/build-e2e-runner/prepare-build.mjs @@ -0,0 +1,99 @@ +import { + existsSync, + lstatSync, + mkdirSync, + realpathSync, + rmSync, +} from "node:fs"; +import { basename, dirname, isAbsolute, relative, resolve, sep } from "node:path"; + +const [ + workspace, + requestedWorkingDirectory, + requestedDockerfile, + imageName, + requestedArchivePath, +] = process.argv.slice(2); + +const assertRelative = (value, label) => { + if (!value || isAbsolute(value)) { + throw new Error(`${label} must be a non-empty repository-relative path`); + } + const relativePath = relative("/contract-root", resolve("/contract-root", value)); + if (relativePath === "" || relativePath === ".." || relativePath.startsWith(`..${sep}`)) { + throw new Error(`${label} must resolve below its parent directory`); + } +}; + +assertRelative(requestedWorkingDirectory, "working-directory"); +assertRelative(requestedDockerfile, "dockerfile"); +assertRelative(requestedArchivePath, "archive-path"); +if (!imageName || /[\s\0]/.test(imageName)) { + throw new Error("image-name must be a non-empty Docker reference without whitespace"); +} + +const workspacePath = realpathSync(workspace); +const buildContext = resolve(workspacePath, requestedWorkingDirectory); +if (!existsSync(buildContext) || !lstatSync(buildContext).isDirectory()) { + throw new Error(`E2E runner build context does not exist: ${requestedWorkingDirectory}`); +} +const resolvedBuildContext = realpathSync(buildContext); +const contextRelativePath = relative(workspacePath, resolvedBuildContext); +if (contextRelativePath.startsWith("..") || isAbsolute(contextRelativePath)) { + throw new Error("working-directory must not escape GITHUB_WORKSPACE through a symlink"); +} + +const dockerfile = resolve(resolvedBuildContext, requestedDockerfile); +if (!existsSync(dockerfile) || !lstatSync(dockerfile).isFile()) { + throw new Error(`E2E runner Dockerfile does not exist: ${requestedDockerfile}`); +} +const resolvedDockerfile = realpathSync(dockerfile); +const dockerfileRelativePath = relative(resolvedBuildContext, resolvedDockerfile); +if (dockerfileRelativePath.startsWith("..") || isAbsolute(dockerfileRelativePath)) { + throw new Error("dockerfile must not escape working-directory through a symlink"); +} + +const archivePath = resolve(workspacePath, requestedArchivePath); +if (basename(archivePath) !== "playwright-e2e-runner.tar.zst") { + throw new Error("archive-path must end with playwright-e2e-runner.tar.zst"); +} +const archiveRelativePath = relative(workspacePath, archivePath); +if (archiveRelativePath.startsWith("..") || isAbsolute(archiveRelativePath)) { + throw new Error("archive-path must remain inside GITHUB_WORKSPACE"); +} +let existingArchiveAncestor = archivePath; +while (!existsSync(existingArchiveAncestor)) { + const parent = dirname(existingArchiveAncestor); + if (parent === existingArchiveAncestor) { + throw new Error("archive-path has no existing parent directory"); + } + existingArchiveAncestor = parent; +} +const resolvedArchiveAncestor = realpathSync(existingArchiveAncestor); +const archiveAncestorRelativePath = relative( + workspacePath, + resolvedArchiveAncestor, +); +if ( + archiveAncestorRelativePath.startsWith("..") || + isAbsolute(archiveAncestorRelativePath) +) { + throw new Error("archive-path must not traverse a symlink outside GITHUB_WORKSPACE"); +} +mkdirSync(dirname(archivePath), { recursive: true }); +for (const outputPath of [archivePath, resolve(dirname(archivePath), "metadata.json")]) { + if (existsSync(outputPath) && lstatSync(outputPath).isDirectory()) { + throw new Error(`runner output path must not be a directory: ${outputPath}`); + } + rmSync(outputPath, { force: true }); +} + +const quoteForShell = (value) => `'${value.replaceAll("'", `'"'"'`)}'`; +for (const [name, value] of Object.entries({ + E2E_BUILD_CONTEXT: resolvedBuildContext, + E2E_ABSOLUTE_DOCKERFILE: resolvedDockerfile, + E2E_RUNNER_IMAGE: imageName, + E2E_ABSOLUTE_ARCHIVE_PATH: archivePath, +})) { + process.stdout.write(`${name}=${quoteForShell(value)}\n`); +} diff --git a/.github/actions/build-e2e-runner/runner-archive.mjs b/.github/actions/build-e2e-runner/runner-archive.mjs new file mode 100644 index 0000000..c63334e --- /dev/null +++ b/.github/actions/build-e2e-runner/runner-archive.mjs @@ -0,0 +1,106 @@ +import { + appendFileSync, + createReadStream, + existsSync, + lstatSync, + realpathSync, + statSync, + writeFileSync, +} from "node:fs"; +import { createHash } from "node:crypto"; +import { basename, dirname, isAbsolute, relative, resolve, sep } from "node:path"; + +const [ + workspace, + requestedArchivePath, + expectedImageName, + expectedContentHash, + outputFile, + imageId, +] = process.argv.slice(2); + +const fail = (message) => { + throw new Error(message); +}; + +if (!/^[a-f0-9]{64}$/.test(expectedContentHash)) { + fail("content-hash must be a lowercase SHA-256 digest"); +} +if (!expectedImageName || /[\s\0]/.test(expectedImageName)) { + fail("image-name must be a non-empty Docker reference without whitespace"); +} +if (!requestedArchivePath || isAbsolute(requestedArchivePath)) { + fail("archive-path must be repository-relative"); +} + +const workspacePath = realpathSync(workspace); +const archivePath = resolve(workspacePath, requestedArchivePath); +if (basename(archivePath) !== "playwright-e2e-runner.tar.zst") { + fail("archive-path must end with playwright-e2e-runner.tar.zst"); +} +const relativeArchive = relative(workspacePath, archivePath); +if ( + relativeArchive === "" || + relativeArchive === ".." || + relativeArchive.startsWith(`..${sep}`) || + isAbsolute(relativeArchive) +) { + fail("archive-path must remain inside GITHUB_WORKSPACE"); +} +if (!existsSync(archivePath) || !lstatSync(archivePath).isFile()) { + fail(`runner archive does not exist: ${requestedArchivePath}`); +} +if (lstatSync(archivePath).isSymbolicLink()) { + fail("runner archive must not be a symbolic link"); +} + +const digestFile = async (path) => { + const hash = createHash("sha256"); + const stream = createReadStream(path); + for await (const chunk of stream) hash.update(chunk); + return hash.digest("hex"); +}; + +const archiveDigest = await digestFile(archivePath); +const archiveSize = statSync(archivePath).size; +const archiveDirectory = realpathSync(dirname(archivePath)); +const relativeArchiveDirectory = relative(workspacePath, archiveDirectory); +if ( + relativeArchiveDirectory === ".." || + relativeArchiveDirectory.startsWith(`..${sep}`) || + isAbsolute(relativeArchiveDirectory) +) { + fail("runner archive directory must remain inside GITHUB_WORKSPACE"); +} +const metadataPath = resolve(archiveDirectory, "metadata.json"); +if (existsSync(metadataPath) && lstatSync(metadataPath).isSymbolicLink()) { + fail("runner metadata must not be a symbolic link"); +} + +if (!/^sha256:[a-f0-9]{64}$/.test(imageId ?? "")) { + fail("image-id must be a content-addressed Docker image ID"); +} + +const metadata = { + schemaVersion: 1, + contentHash: expectedContentHash, + imageName: expectedImageName, + imageId, + archive: { + file: "playwright-e2e-runner.tar.zst", + sha256: archiveDigest, + size: archiveSize, + }, +}; +writeFileSync(metadataPath, `${JSON.stringify(metadata, null, 2)}\n`, { + mode: 0o600, +}); + +for (const [name, value] of [ + ["image-name", metadata.imageName], + ["image-id", metadata.imageId], + ["archive-path", archivePath], + ["archive-sha256", archiveDigest], +]) { + appendFileSync(outputFile, `${name}=${value}\n`); +} diff --git a/.github/actions/finalize-e2e-verdict/action.yml b/.github/actions/finalize-e2e-result/action.yml similarity index 66% rename from .github/actions/finalize-e2e-verdict/action.yml rename to .github/actions/finalize-e2e-result/action.yml index 304c73e..f1f3113 100644 --- a/.github/actions/finalize-e2e-verdict/action.yml +++ b/.github/actions/finalize-e2e-result/action.yml @@ -1,9 +1,9 @@ -name: Finalize E2E Verdict -description: Enforce the evidence verdict and all upstream GitHub Actions job results. +name: Finalize E2E Result +description: Enforce the merged E2E result and all upstream GitHub Actions job results. inputs: - verdict-file: - description: Path to the downloaded verdict.json, absolute or relative to GITHUB_WORKSPACE. + result-file: + description: Path to the downloaded result.json, absolute or relative to GITHUB_WORKSPACE. required: true shard-job-result: description: GitHub Actions result of the shard matrix job. @@ -17,21 +17,21 @@ inputs: default: success outputs: - verdict: - description: Final enforced verdict. - value: ${{ steps.finalize.outputs.verdict }} + result: + description: Final enforced result. + value: ${{ steps.finalize.outputs.result }} primary-failure: - description: Primary failure message, empty when the verdict passed. + description: Primary failure message, empty when the result passed. value: ${{ steps.finalize.outputs.primary-failure }} runs: using: composite steps: - - name: Enforce final E2E verdict + - name: Enforce final E2E result id: finalize shell: bash env: - E2E_VERDICT_FILE: ${{ inputs.verdict-file }} + E2E_RESULT_FILE: ${{ inputs.result-file }} E2E_SHARD_JOB_RESULT: ${{ inputs.shard-job-result }} E2E_MERGE_JOB_RESULT: ${{ inputs.merge-job-result }} E2E_PUBLICATION_RESULT: ${{ inputs.publication-result }} @@ -39,7 +39,7 @@ runs: set -euo pipefail node "$GITHUB_ACTION_PATH/finalize.mjs" \ "$GITHUB_WORKSPACE" \ - "$E2E_VERDICT_FILE" \ + "$E2E_RESULT_FILE" \ "$E2E_SHARD_JOB_RESULT" \ "$E2E_MERGE_JOB_RESULT" \ "$E2E_PUBLICATION_RESULT" \ diff --git a/.github/actions/finalize-e2e-verdict/finalize.mjs b/.github/actions/finalize-e2e-result/finalize.mjs similarity index 68% rename from .github/actions/finalize-e2e-verdict/finalize.mjs rename to .github/actions/finalize-e2e-result/finalize.mjs index e216454..294dfb1 100644 --- a/.github/actions/finalize-e2e-verdict/finalize.mjs +++ b/.github/actions/finalize-e2e-result/finalize.mjs @@ -8,7 +8,7 @@ import { isAbsolute, relative, resolve } from "node:path"; const [ workspace, - requestedVerdictFile, + requestedResultFile, shardJobResult, mergeJobResult, publicationResult, @@ -17,31 +17,31 @@ const [ ] = process.argv.slice(2); const workspacePath = realpathSync(workspace); -const verdictFile = isAbsolute(requestedVerdictFile) - ? requestedVerdictFile - : resolve(workspacePath, requestedVerdictFile); -const verdictRelativePath = relative(workspacePath, verdictFile); +const resultFile = isAbsolute(requestedResultFile) + ? requestedResultFile + : resolve(workspacePath, requestedResultFile); +const resultRelativePath = relative(workspacePath, resultFile); const orchestrationFailures = []; if ( - verdictRelativePath === ".." || - verdictRelativePath.startsWith( + resultRelativePath === ".." || + resultRelativePath.startsWith( `..${process.platform === "win32" ? "\\" : "/"}`, ) || - isAbsolute(verdictRelativePath) + isAbsolute(resultRelativePath) ) { - orchestrationFailures.push("verdict-file must be inside GITHUB_WORKSPACE"); + orchestrationFailures.push("result-file must be inside GITHUB_WORKSPACE"); } -let verdict = null; -if (orchestrationFailures.length === 0 && existsSync(verdictFile)) { +let result = null; +if (orchestrationFailures.length === 0 && existsSync(resultFile)) { try { - verdict = JSON.parse(readFileSync(verdictFile, "utf8")); + result = JSON.parse(readFileSync(resultFile, "utf8")); } catch (error) { - orchestrationFailures.push(`Cannot parse verdict.json: ${error.message}`); + orchestrationFailures.push(`Cannot parse result.json: ${error.message}`); } } else if (orchestrationFailures.length === 0) { - orchestrationFailures.push(`Missing verdict.json at ${verdictRelativePath}`); + orchestrationFailures.push(`Missing result.json at ${resultRelativePath}`); } const upstreamResults = [ @@ -55,8 +55,8 @@ for (const [label, result] of upstreamResults) { } } -let primaryFailure = verdict?.primaryFailure ?? null; -const secondaryFailures = [...(verdict?.secondaryFailures ?? [])]; +let primaryFailure = result?.primaryFailure ?? null; +const secondaryFailures = [...(result?.secondaryFailures ?? [])]; for (const message of orchestrationFailures) { const failure = { kind: "infrastructure", message }; if (!primaryFailure) primaryFailure = failure; @@ -64,7 +64,7 @@ for (const message of orchestrationFailures) { secondaryFailures.push(failure); } -const recordedResult = verdict?.result ?? "infrastructure-error"; +const recordedResult = result?.result ?? "infrastructure-error"; const finalResult = recordedResult === "failed" ? "failed" @@ -78,7 +78,7 @@ appendFileSync( [ "## Final E2E Gate", "", - `**Verdict:** ${finalResult}`, + `**Result:** ${finalResult}`, "", `- Shard job: ${shardJobResult}`, `- Merge job: ${mergeJobResult}`, @@ -100,11 +100,11 @@ const escapeOutput = (value) => .replaceAll("%", "%25") .replaceAll("\r", "%0D") .replaceAll("\n", "%0A"); -appendFileSync(outputFile, `verdict=${finalResult}\n`); +appendFileSync(outputFile, `result=${finalResult}\n`); appendFileSync(outputFile, `primary-failure=${escapeOutput(primaryMessage)}\n`); if (finalResult !== "passed") { - console.error(`Final E2E verdict: ${finalResult}`); + console.error(`Final E2E result: ${finalResult}`); if (primaryMessage) console.error(`Primary failure: ${primaryMessage}`); for (const failure of secondaryFailures) console.error(`Additional failure: ${failure.message}`); diff --git a/.github/actions/merge-e2e-evidence/action.yml b/.github/actions/merge-e2e-evidence/action.yml deleted file mode 100644 index aaafd61..0000000 --- a/.github/actions/merge-e2e-evidence/action.yml +++ /dev/null @@ -1,145 +0,0 @@ -name: Merge E2E Evidence -description: Validate all shard evidence, merge reports, publish evidence, and fail closed. - -inputs: - working-directory: - description: Repository-relative directory containing the installed Playwright package. - required: false - default: e2e - download-directory: - description: Repository-relative directory containing downloaded shard artifacts. - required: false - default: downloaded-shards - evidence-directory: - description: Repository-relative directory where merged evidence is assembled. - required: false - default: e2e-evidence - expected-shards: - description: Number of shard status files and report sets required. - required: true - artifact-name: - description: Name of the durable merged evidence artifact. - required: false - default: e2e-evidence - retention-days: - description: Number of days GitHub retains the merged evidence artifact. - required: false - default: "10" - -outputs: - verdict: - description: "Authoritative verdict: passed, failed, or infrastructure-error." - value: ${{ steps.evaluate.outputs.verdict }} - artifact-name: - description: Merged evidence artifact name. - value: ${{ inputs.artifact-name }} - total: - description: Aggregated test count. - value: ${{ steps.evaluate.outputs.total }} - passed: - description: Aggregated passed test count. - value: ${{ steps.evaluate.outputs.passed }} - failed: - description: Aggregated failed test count. - value: ${{ steps.evaluate.outputs.failed }} - skipped: - description: Aggregated skipped test count. - value: ${{ steps.evaluate.outputs.skipped }} - verdict-file: - description: Absolute path to verdict.json. - value: ${{ steps.evaluate.outputs.verdict-file }} - -runs: - using: composite - steps: - - name: Stage and validate shard evidence - id: stage - shell: bash - env: - E2E_WORKING_DIRECTORY: ${{ inputs.working-directory }} - E2E_DOWNLOAD_DIRECTORY: ${{ inputs.download-directory }} - E2E_EVIDENCE_DIRECTORY: ${{ inputs.evidence-directory }} - E2E_EXPECTED_SHARDS: ${{ inputs.expected-shards }} - run: | - set -euo pipefail - node "$GITHUB_ACTION_PATH/stage-evidence.mjs" \ - "$GITHUB_WORKSPACE" \ - "$E2E_WORKING_DIRECTORY" \ - "$E2E_DOWNLOAD_DIRECTORY" \ - "$E2E_EVIDENCE_DIRECTORY" \ - "$E2E_EXPECTED_SHARDS" \ - "$GITHUB_OUTPUT" - - - name: Merge Playwright blob reports - id: merge-html - if: always() - continue-on-error: true - shell: bash - env: - E2E_CAN_MERGE: ${{ steps.stage.outputs.can-merge }} - E2E_WORKING_DIRECTORY: ${{ steps.stage.outputs.working-directory }} - E2E_BLOB_DIRECTORY: ${{ steps.stage.outputs.blob-directory }} - PLAYWRIGHT_HTML_OUTPUT_DIR: ${{ steps.stage.outputs.html-directory }} - PLAYWRIGHT_HTML_OPEN: never - run: | - set +e - exit_code=0 - - if [[ "$E2E_CAN_MERGE" != "true" ]]; then - echo "Blob report merge cannot run because shard evidence is incomplete." - exit_code=42 - else - ( - cd "$E2E_WORKING_DIRECTORY" - pnpm exec playwright merge-reports --reporter html "$E2E_BLOB_DIRECTORY" - ) - exit_code=$? - fi - - echo "exit-code=$exit_code" >> "$GITHUB_OUTPUT" - exit "$exit_code" - - - name: Assemble deterministic evidence - id: assemble - if: always() - shell: bash - env: - E2E_EVIDENCE_DIRECTORY: ${{ steps.stage.outputs.evidence-directory }} - E2E_EXPECTED_SHARDS: ${{ inputs.expected-shards }} - E2E_HTML_MERGE_EXIT_CODE: ${{ steps.merge-html.outputs.exit-code }} - run: | - set -euo pipefail - node "$GITHUB_ACTION_PATH/assemble-evidence.mjs" \ - "$E2E_EVIDENCE_DIRECTORY" \ - "$E2E_EXPECTED_SHARDS" \ - "${E2E_HTML_MERGE_EXIT_CODE:-99}" - - - name: Upload merged E2E evidence - id: publish - if: always() - continue-on-error: true - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: ${{ inputs.artifact-name }} - path: ${{ steps.stage.outputs.evidence-directory }}/ - if-no-files-found: error - retention-days: ${{ inputs.retention-days }} - - - name: Publish summary and enforce verdict - id: evaluate - if: always() - shell: bash - env: - E2E_EVIDENCE_DIRECTORY: ${{ steps.stage.outputs.evidence-directory }} - E2E_EXPECTED_SHARDS: ${{ inputs.expected-shards }} - E2E_ARTIFACT_NAME: ${{ inputs.artifact-name }} - E2E_PUBLICATION_OUTCOME: ${{ steps.publish.outcome }} - run: | - set -euo pipefail - node "$GITHUB_ACTION_PATH/evaluate-verdict.mjs" \ - "$E2E_EVIDENCE_DIRECTORY" \ - "$E2E_EXPECTED_SHARDS" \ - "$E2E_ARTIFACT_NAME" \ - "${E2E_PUBLICATION_OUTCOME:-failure}" \ - "$GITHUB_STEP_SUMMARY" \ - "$GITHUB_OUTPUT" diff --git a/.github/actions/merge-e2e-evidence/evaluate-verdict.mjs b/.github/actions/merge-e2e-evidence/evaluate-verdict.mjs deleted file mode 100644 index ddce94e..0000000 --- a/.github/actions/merge-e2e-evidence/evaluate-verdict.mjs +++ /dev/null @@ -1,103 +0,0 @@ -import { - appendFileSync, - existsSync, - readFileSync, - writeFileSync, -} from "node:fs"; -import { resolve } from "node:path"; - -const [ - evidenceDirectory, - requestedExpectedShards, - artifactName, - publicationOutcome, - summaryFile, - outputFile, -] = process.argv.slice(2); -const verdictFile = resolve(evidenceDirectory, "verdict.json"); - -let verdict; -if (existsSync(verdictFile)) { - try { - verdict = JSON.parse(readFileSync(verdictFile, "utf8")); - } catch (error) { - verdict = null; - } -} - -if (!verdict) { - verdict = { - schemaVersion: 1, - result: "infrastructure-error", - primaryFailure: { - kind: "infrastructure", - message: "Evidence assembly did not produce a readable verdict.json", - }, - secondaryFailures: [], - expectedShards: Number(requestedExpectedShards), - observedShards: [], - summary: { - total: 0, - passed: 0, - failed: 0, - skipped: 0, - pending: 0, - other: 0, - }, - shards: [], - reports: {}, - generatedAt: new Date().toISOString(), - }; -} - -let enforcedResult = verdict.result; -let enforcedPrimaryFailure = verdict.primaryFailure ?? null; -if (publicationOutcome !== "success") { - const publicationFailure = { - kind: "infrastructure", - message: `Evidence artifact publication finished with ${publicationOutcome || "unknown"}`, - }; - if (!enforcedPrimaryFailure) enforcedPrimaryFailure = publicationFailure; - if (enforcedResult === "passed") enforcedResult = "infrastructure-error"; -} - -const summary = verdict.summary ?? {}; -appendFileSync( - summaryFile, - [ - "## E2E Gate", - "", - `**Verdict:** ${enforcedResult}`, - "", - "| Result | Count |", - "| --- | ---: |", - `| Total | ${summary.total ?? 0} |`, - `| Passed | ${summary.passed ?? 0} |`, - `| Failed | ${summary.failed ?? 0} |`, - `| Skipped | ${summary.skipped ?? 0} |`, - "", - `**Shards:** ${(verdict.observedShards ?? []).length}/${verdict.expectedShards ?? 0}`, - "", - `**Evidence artifact:** \`${artifactName}\``, - ...(enforcedPrimaryFailure - ? ["", `**Primary failure:** ${enforcedPrimaryFailure.message}`] - : []), - "", - ].join("\n"), -); - -const appendOutput = (name, value) => { - appendFileSync(outputFile, `${name}=${value}\n`); -}; -appendOutput("verdict", enforcedResult); -appendOutput("total", Number(summary.total ?? 0)); -appendOutput("passed", Number(summary.passed ?? 0)); -appendOutput("failed", Number(summary.failed ?? 0)); -appendOutput("skipped", Number(summary.skipped ?? 0)); -appendOutput("verdict-file", verdictFile); - -if (enforcedResult !== "passed" || publicationOutcome !== "success") { - console.error(`E2E evidence is not healthy: ${enforcedResult}`); - if (enforcedPrimaryFailure) console.error(enforcedPrimaryFailure.message); - process.exit(1); -} diff --git a/.github/actions/merge-e2e-results/action.yml b/.github/actions/merge-e2e-results/action.yml new file mode 100644 index 0000000..b47902e --- /dev/null +++ b/.github/actions/merge-e2e-results/action.yml @@ -0,0 +1,123 @@ +name: Merge E2E Results +description: Validate shard results and merge reports inside the immutable E2E runner image. + +inputs: + download-directory: + description: Repository-relative directory containing downloaded shard artifacts. + required: false + default: downloaded-shards + results-directory: + description: Repository-relative directory where merged results are assembled. + required: false + default: e2e-results + expected-shards: + description: Number of shard status files and report sets required. + required: true + runner-image-id: + description: Content-addressed Docker image ID used to merge Playwright reports. + required: true + +outputs: + results-directory: + description: Absolute directory containing merged E2E results. + value: ${{ steps.stage.outputs.results-directory }} + report-directory: + description: Absolute directory containing the merged Playwright HTML report. + value: ${{ steps.stage.outputs.html-directory }} + result: + description: "Final result: passed, failed, or infrastructure-error." + value: ${{ steps.assemble.outputs.result }} + total: + description: Aggregated test count. + value: ${{ steps.assemble.outputs.total }} + passed: + description: Aggregated passed test count. + value: ${{ steps.assemble.outputs.passed }} + failed: + description: Aggregated failed test count. + value: ${{ steps.assemble.outputs.failed }} + skipped: + description: Aggregated skipped test count. + value: ${{ steps.assemble.outputs.skipped }} + result-file: + description: Absolute path to result.json. + value: ${{ steps.assemble.outputs.result-file }} + +runs: + using: composite + steps: + - name: Stage and validate shard results + id: stage + shell: bash + env: + E2E_DOWNLOAD_DIRECTORY: ${{ inputs.download-directory }} + E2E_RESULTS_DIRECTORY: ${{ inputs.results-directory }} + E2E_EXPECTED_SHARDS: ${{ inputs.expected-shards }} + run: | + set -euo pipefail + node "$GITHUB_ACTION_PATH/stage-results.mjs" \ + "$GITHUB_WORKSPACE" \ + "$E2E_DOWNLOAD_DIRECTORY" \ + "$E2E_RESULTS_DIRECTORY" \ + "$E2E_EXPECTED_SHARDS" \ + "$GITHUB_OUTPUT" + + - name: Merge Playwright blob reports + id: merge-html + continue-on-error: true + shell: bash + env: + E2E_CAN_MERGE: ${{ steps.stage.outputs.can-merge }} + E2E_BLOB_DIRECTORY: ${{ steps.stage.outputs.blob-directory }} + PLAYWRIGHT_HTML_OUTPUT_DIR: ${{ steps.stage.outputs.html-directory }} + E2E_RUNNER_IMAGE_ID: ${{ inputs.runner-image-id }} + run: | + set +e + exit_code=0 + + if [[ "$E2E_CAN_MERGE" != "true" ]]; then + echo "Blob report merge cannot run because shard results are incomplete." + exit_code=42 + elif [[ ! "$E2E_RUNNER_IMAGE_ID" =~ ^sha256:[a-f0-9]{64}$ ]]; then + echo "Blob report merge cannot run because the runner image ID is invalid." + exit_code=42 + else + docker run --rm \ + --init \ + --read-only \ + --network none \ + --cap-drop ALL \ + --security-opt no-new-privileges:true \ + --pids-limit 256 \ + --tmpfs /tmp:rw,nosuid,nodev,noexec,size=256m \ + --user "$(id -u):$(id -g)" \ + --env CI=true \ + --env HOME=/tmp \ + --env PLAYWRIGHT_HTML_OUTPUT_DIR=/merge-output \ + --env PLAYWRIGHT_HTML_OPEN=never \ + --volume "$E2E_BLOB_DIRECTORY:/merge-input/blob-report" \ + --volume "$PLAYWRIGHT_HTML_OUTPUT_DIR:/merge-output" \ + --entrypoint /workspace/e2e/node_modules/.bin/playwright \ + "$E2E_RUNNER_IMAGE_ID" \ + merge-reports --reporter html /merge-input/blob-report + exit_code=$? + fi + + echo "exit-code=$exit_code" >> "$GITHUB_OUTPUT" + exit "$exit_code" + + - name: Assemble deterministic results + id: assemble + if: always() && steps.stage.outcome == 'success' + shell: bash + env: + E2E_RESULTS_DIRECTORY: ${{ steps.stage.outputs.results-directory }} + E2E_EXPECTED_SHARDS: ${{ inputs.expected-shards }} + E2E_HTML_MERGE_EXIT_CODE: ${{ steps.merge-html.outputs.exit-code }} + run: | + set -euo pipefail + node "$GITHUB_ACTION_PATH/assemble-results.mjs" \ + "$E2E_RESULTS_DIRECTORY" \ + "$E2E_EXPECTED_SHARDS" \ + "${E2E_HTML_MERGE_EXIT_CODE:-99}" \ + "$GITHUB_OUTPUT" diff --git a/.github/actions/merge-e2e-evidence/assemble-evidence.mjs b/.github/actions/merge-e2e-results/assemble-results.mjs similarity index 72% rename from .github/actions/merge-e2e-evidence/assemble-evidence.mjs rename to .github/actions/merge-e2e-results/assemble-results.mjs index 9e227cb..d0d2c86 100644 --- a/.github/actions/merge-e2e-evidence/assemble-evidence.mjs +++ b/.github/actions/merge-e2e-results/assemble-results.mjs @@ -7,18 +7,22 @@ import { } from "node:fs"; import { resolve } from "node:path"; -const [evidenceDirectory, requestedExpectedShards, requestedHtmlMergeExitCode] = - process.argv.slice(2); +const [ + resultsDirectory, + requestedExpectedShards, + requestedHtmlMergeExitCode, + outputFile, +] = process.argv.slice(2); const expectedShards = Number(requestedExpectedShards); const htmlMergeExitCode = Number(requestedHtmlMergeExitCode); -const stagingDirectory = resolve(evidenceDirectory, ".merge-input"); +const stagingDirectory = resolve(resultsDirectory, ".merge-input"); const scanFile = resolve(stagingDirectory, "scan.json"); const infrastructureFailures = []; let scan = { expectedShards, observedShards: [], statuses: [], errors: [] }; if (!existsSync(scanFile)) { infrastructureFailures.push( - "Shard evidence staging did not produce scan.json", + "Shard results staging did not produce scan.json", ); } else { try { @@ -26,7 +30,7 @@ if (!existsSync(scanFile)) { infrastructureFailures.push(...(scan.errors ?? [])); } catch (error) { infrastructureFailures.push( - `Cannot parse shard evidence scan: ${error.message}`, + `Cannot parse shard results scan: ${error.message}`, ); } } @@ -93,6 +97,9 @@ for (const fileName of ctrfFiles) { if (summary.tests === 0 && combinedTests.length > 0) { summary.tests = combinedTests.length; } +if (summary.tests === 0) { + infrastructureFailures.push("No Playwright tests were executed"); +} if (starts.length > 0) summary.start = Math.min(...starts); if (stops.length > 0) summary.stop = Math.max(...stops); @@ -111,21 +118,41 @@ const ctrfReport = { }, }; writeFileSync( - resolve(evidenceDirectory, "ctrf-report.json"), + resolve(resultsDirectory, "ctrf-report.json"), `${JSON.stringify(ctrfReport, null, 2)}\n`, ); const failedShards = (scan.statuses ?? []).filter( (status) => status.result === "failed", ); +const infrastructureErrorShards = (scan.statuses ?? []).filter( + (status) => status.result === "infrastructure-error", +); const incompleteShards = (scan.statuses ?? []).filter( - (status) => !["passed", "failed"].includes(status.result), + (status) => + !["passed", "failed", "infrastructure-error"].includes(status.result), +); +const shardInfrastructureFailures = (scan.statuses ?? []).flatMap((status) => + [status.primaryFailure, ...(status.secondaryFailures ?? [])] + .filter((failure) => failure?.kind === "infrastructure") + .map( + (failure) => + `Shard ${status.shard.index} ${failure.phase ?? "infrastructure"}: ${failure.message}`, + ), ); +if (infrastructureErrorShards.length > 0 && shardInfrastructureFailures.length === 0) { + shardInfrastructureFailures.push( + ...infrastructureErrorShards.map( + (status) => `Shard ${status.shard.index} reported an infrastructure error`, + ), + ); +} for (const status of incompleteShards) { - infrastructureFailures.push( + shardInfrastructureFailures.push( `Shard ${status.shard.index} did not complete; last state is ${status.result ?? "unknown"}`, ); } +infrastructureFailures.unshift(...shardInfrastructureFailures); const hasTestFailure = failedShards.length > 0 || summary.failed > 0; let primaryFailure = null; @@ -153,8 +180,8 @@ if (hasTestFailure) { ); } -const verdict = { - schemaVersion: 1, +const result = { + schemaVersion: 2, result: hasTestFailure ? "failed" : infrastructureFailures.length > 0 @@ -187,8 +214,21 @@ const verdict = { generatedAt: new Date().toISOString(), }; writeFileSync( - resolve(evidenceDirectory, "verdict.json"), - `${JSON.stringify(verdict, null, 2)}\n`, + resolve(resultsDirectory, "result.json"), + `${JSON.stringify(result, null, 2)}\n`, ); +if (outputFile) { + for (const [name, value] of [ + ["result", result.result], + ["total", result.summary.total], + ["passed", result.summary.passed], + ["failed", result.summary.failed], + ["skipped", result.summary.skipped], + ["result-file", resolve(resultsDirectory, "result.json")], + ]) { + writeFileSync(outputFile, `${name}=${value}\n`, { flag: "a" }); + } +} + rmSync(stagingDirectory, { recursive: true, force: true }); diff --git a/.github/actions/merge-e2e-evidence/stage-evidence.mjs b/.github/actions/merge-e2e-results/stage-results.mjs similarity index 84% rename from .github/actions/merge-e2e-evidence/stage-evidence.mjs rename to .github/actions/merge-e2e-results/stage-results.mjs index 8bdbd2a..925b577 100644 --- a/.github/actions/merge-e2e-evidence/stage-evidence.mjs +++ b/.github/actions/merge-e2e-results/stage-results.mjs @@ -13,9 +13,8 @@ import { basename, isAbsolute, relative, resolve, sep } from "node:path"; const [ workspace, - requestedWorkingDirectory, requestedDownloadDirectory, - requestedEvidenceDirectory, + requestedResultsDirectory, requestedExpectedShards, outputFile, ] = process.argv.slice(2); @@ -47,40 +46,36 @@ const assertRelative = (value, label) => { }; for (const [value, label] of [ - [requestedWorkingDirectory, "working-directory"], [requestedDownloadDirectory, "download-directory"], - [requestedEvidenceDirectory, "evidence-directory"], + [requestedResultsDirectory, "results-directory"], ]) { assertRelative(value, label); } const workspacePath = realpathSync(workspace); -const workingDirectory = resolve(workspacePath, requestedWorkingDirectory); const downloadDirectory = resolve(workspacePath, requestedDownloadDirectory); -const evidenceDirectory = resolve(workspacePath, requestedEvidenceDirectory); +const resultsDirectory = resolve(workspacePath, requestedResultsDirectory); if ( - existsSync(evidenceDirectory) && - lstatSync(evidenceDirectory).isSymbolicLink() + existsSync(resultsDirectory) && + lstatSync(resultsDirectory).isSymbolicLink() ) { - throw new Error("evidence-directory must not be a symbolic link"); + throw new Error("results-directory must not be a symbolic link"); } -rmSync(evidenceDirectory, { recursive: true, force: true }); +rmSync(resultsDirectory, { recursive: true, force: true }); -const stagingDirectory = resolve(evidenceDirectory, ".merge-input"); +const stagingDirectory = resolve(resultsDirectory, ".merge-input"); const blobDirectory = resolve(stagingDirectory, "blob-report"); const ctrfDirectory = resolve(stagingDirectory, "ctrf"); -const statusDirectory = resolve(evidenceDirectory, "shard-status"); -const htmlDirectory = resolve(evidenceDirectory, "playwright-report"); +const statusDirectory = resolve(resultsDirectory, "shard-status"); +const htmlDirectory = resolve(resultsDirectory, "playwright-report"); mkdirSync(blobDirectory, { recursive: true }); mkdirSync(ctrfDirectory, { recursive: true }); mkdirSync(statusDirectory, { recursive: true }); +mkdirSync(htmlDirectory, { recursive: true }); const errors = []; -for (const [path, label] of [ - [workingDirectory, "working-directory"], - [downloadDirectory, "download-directory"], -]) { +for (const [path, label] of [[downloadDirectory, "download-directory"]]) { if (!existsSync(path) || !lstatSync(path).isDirectory()) { errors.push(`${label} does not exist or is not a directory: ${path}`); continue; @@ -205,8 +200,7 @@ const appendOutput = (name, value) => { writeFileSync(outputFile, `${name}=${escaped}\n`, { flag: "a" }); }; -appendOutput("working-directory", workingDirectory); -appendOutput("evidence-directory", evidenceDirectory); +appendOutput("results-directory", resultsDirectory); appendOutput("blob-directory", blobDirectory); appendOutput("html-directory", htmlDirectory); appendOutput("can-merge", errors.length === 0 ? "true" : "false"); diff --git a/.github/actions/package-e2e-report/Dockerfile b/.github/actions/package-e2e-report/Dockerfile new file mode 100644 index 0000000..11f5199 --- /dev/null +++ b/.github/actions/package-e2e-report/Dockerfile @@ -0,0 +1,20 @@ +FROM node:22.22.1-alpine3.23@sha256:8094c002d08262dba12645a3b4a15cd6cd627d30bc782f53229a2ec13ee22a00 + +ENV NODE_ENV=production \ + REPORT_HOST=0.0.0.0 \ + REPORT_PORT=8080 \ + REPORT_ROOT=/app/report + +WORKDIR /app + +COPY --chown=node:node server.mjs ./server.mjs +COPY --chown=node:node report ./report + +USER node + +EXPOSE 8080 + +HEALTHCHECK --interval=5s --timeout=2s --start-period=2s --retries=5 \ + CMD ["node", "-e", "fetch('http://127.0.0.1:8080/').then(response => { if (!response.ok) process.exit(1) }).catch(() => process.exit(1))"] + +CMD ["node", "server.mjs"] diff --git a/.github/actions/package-e2e-report/action.yml b/.github/actions/package-e2e-report/action.yml new file mode 100644 index 0000000..780d6d2 --- /dev/null +++ b/.github/actions/package-e2e-report/action.yml @@ -0,0 +1,91 @@ +name: Package E2E Report Image +description: Build and upload a portable Docker image containing a merged Playwright HTML report. + +inputs: + report-directory: + description: Directory containing the merged Playwright HTML report and its index.html. + required: true + image-name: + description: Immutable-by-run Docker image name and tag supplied by the caller. + required: true + archive-directory: + description: Repository-relative directory where the Docker image archive is written. + required: false + default: e2e-report-image + artifact-name: + description: GitHub artifact containing the Docker image archive. + required: false + default: e2e-report-image + retention-days: + description: Number of days GitHub retains the image artifact. + required: false + default: "10" + +outputs: + image-name: + description: Docker image name and tag built by the action. + value: ${{ steps.package.outputs.image-name }} + image-id: + description: Immutable local Docker image ID. + value: ${{ steps.package.outputs.image-id }} + artifact-name: + description: GitHub artifact containing the Docker archive. + value: ${{ inputs.artifact-name }} + archive-path: + description: Absolute path to playwright-e2e-report.tar.zst on the runner. + value: ${{ steps.package.outputs.archive-path }} + +runs: + using: composite + steps: + - id: package + name: Build and archive report image + shell: bash + env: + E2E_REPORT_DIRECTORY: ${{ inputs.report-directory }} + E2E_REPORT_IMAGE_NAME: ${{ inputs.image-name }} + E2E_REPORT_ARCHIVE_DIRECTORY: ${{ inputs.archive-directory }} + run: | + set -euo pipefail + + eval "$(node "$GITHUB_ACTION_PATH/prepare-context.mjs" \ + "$GITHUB_WORKSPACE" \ + "$GITHUB_ACTION_PATH" \ + "$E2E_REPORT_DIRECTORY" \ + "$E2E_REPORT_IMAGE_NAME" \ + "$E2E_REPORT_ARCHIVE_DIRECTORY")" + + cleanup() { + rm -rf -- "$E2E_REPORT_BUILD_CONTEXT" + } + trap cleanup EXIT + + docker build \ + --file "$E2E_REPORT_BUILD_CONTEXT/Dockerfile" \ + --tag "$E2E_REPORT_IMAGE_NAME" \ + "$E2E_REPORT_BUILD_CONTEXT" + + image_id="$(docker image inspect --format '{{.Id}}' "$E2E_REPORT_IMAGE_NAME")" + if [[ ! "$image_id" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo "Invalid Docker image ID: $image_id" >&2 + exit 1 + fi + + docker save "$E2E_REPORT_IMAGE_NAME" | + zstd -T0 -3 --quiet -o "$E2E_REPORT_ARCHIVE_PATH" + test -s "$E2E_REPORT_ARCHIVE_PATH" + + { + echo "image-name=$E2E_REPORT_IMAGE_NAME" + echo "image-id=$image_id" + echo "archive-path=$E2E_REPORT_ARCHIVE_PATH" + } >> "$GITHUB_OUTPUT" + + - name: Upload report image archive + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: ${{ inputs.artifact-name }} + path: ${{ steps.package.outputs.archive-path }} + if-no-files-found: error + retention-days: ${{ inputs.retention-days }} + compression-level: 0 diff --git a/.github/actions/package-e2e-report/prepare-context.mjs b/.github/actions/package-e2e-report/prepare-context.mjs new file mode 100644 index 0000000..6cb8b1c --- /dev/null +++ b/.github/actions/package-e2e-report/prepare-context.mjs @@ -0,0 +1,163 @@ +import { + copyFileSync, + cpSync, + existsSync, + lstatSync, + mkdirSync, + readdirSync, + realpathSync, + rmSync, +} from "node:fs"; +import { basename, isAbsolute, relative, resolve, sep } from "node:path"; + +const [ + workspace, + actionPath, + requestedReportDirectory, + imageName, + requestedArchiveDirectory, +] = process.argv.slice(2); + +const fail = (message) => { + throw new Error(message); +}; + +if ( + !workspace || + !actionPath || + !requestedReportDirectory || + !imageName || + !requestedArchiveDirectory +) { + fail( + "prepare-context requires workspace, action path, report directory, " + + "image name, and archive directory", + ); +} + +const workspacePath = realpathSync(workspace); +const isInsideWorkspace = (path) => { + const relativePath = relative(workspacePath, path); + return ( + relativePath !== "" && + relativePath !== ".." && + !relativePath.startsWith(`..${sep}`) && + !isAbsolute(relativePath) + ); +}; + +const resolveWorkspaceInput = (value, label, mustExist) => { + let candidate = isAbsolute(value) + ? resolve(value) + : resolve(workspacePath, value); + if (mustExist && !existsSync(candidate)) { + fail(`${label} does not exist: ${value}`); + } + if (mustExist) { + candidate = realpathSync(candidate); + } else { + let existingAncestor = candidate; + while (!existsSync(existingAncestor)) { + const parent = resolve(existingAncestor, ".."); + if (parent === existingAncestor) { + fail(`${label} has no existing parent directory`); + } + existingAncestor = parent; + } + const realAncestor = realpathSync(existingAncestor); + candidate = resolve(realAncestor, relative(existingAncestor, candidate)); + } + if (!isInsideWorkspace(candidate)) { + fail(`${label} must resolve below GITHUB_WORKSPACE`); + } + return candidate; +}; + +const tagSeparator = imageName.lastIndexOf(":"); +const lastSlash = imageName.lastIndexOf("/"); +const repository = imageName.slice(0, tagSeparator); +const tag = imageName.slice(tagSeparator + 1); +if ( + imageName.length > 255 || + tagSeparator <= lastSlash || + !/^[a-z0-9][a-z0-9._/:-]*$/.test(repository) || + !/^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$/.test(tag) +) { + fail( + `image-name must be a tagged Docker image reference; received ${imageName}`, + ); +} + +const reportDirectory = resolveWorkspaceInput( + requestedReportDirectory, + "report-directory", + true, +); +if (!lstatSync(reportDirectory).isDirectory()) { + fail("report-directory must be a directory"); +} +if (!existsSync(resolve(reportDirectory, "index.html"))) { + fail("report-directory must contain index.html"); +} + +const assertPortableTree = (directory) => { + for (const entry of readdirSync(directory, { withFileTypes: true })) { + const entryPath = resolve(directory, entry.name); + if (entry.isSymbolicLink()) { + fail(`report-directory must not contain symbolic links: ${entry.name}`); + } + if (entry.isDirectory()) { + assertPortableTree(entryPath); + } else if (!entry.isFile()) { + fail(`report-directory contains an unsupported file type: ${entry.name}`); + } + } +}; +assertPortableTree(reportDirectory); + +const archiveDirectory = resolveWorkspaceInput( + requestedArchiveDirectory, + "archive-directory", + false, +); +const isSameOrInside = (parent, child) => { + const relativePath = relative(parent, child); + return ( + relativePath === "" || + (!relativePath.startsWith(`..${sep}`) && !isAbsolute(relativePath)) + ); +}; +if ( + isSameOrInside(reportDirectory, archiveDirectory) || + isSameOrInside(archiveDirectory, reportDirectory) +) { + fail("archive-directory and report-directory must not overlap"); +} +mkdirSync(archiveDirectory, { recursive: true }); + +const buildContext = resolve(archiveDirectory, ".build-context"); +rmSync(buildContext, { recursive: true, force: true }); +mkdirSync(buildContext); +copyFileSync( + resolve(actionPath, "Dockerfile"), + resolve(buildContext, "Dockerfile"), +); +copyFileSync( + resolve(actionPath, "server.mjs"), + resolve(buildContext, "server.mjs"), +); +cpSync(reportDirectory, resolve(buildContext, "report"), { recursive: true }); + +const archivePath = resolve(archiveDirectory, "playwright-e2e-report.tar.zst"); +rmSync(archivePath, { force: true }); +const quoteForShell = (value) => `'${value.replaceAll("'", `'"'"'`)}'`; +for (const [name, value] of Object.entries({ + E2E_REPORT_BUILD_CONTEXT: buildContext, + E2E_REPORT_ARCHIVE_PATH: archivePath, +})) { + process.stdout.write(`${name}=${quoteForShell(value)}\n`); +} + +process.stderr.write( + `Packaging ${basename(reportDirectory)} as ${imageName} into ${archivePath}\n`, +); diff --git a/.github/actions/package-e2e-report/server.mjs b/.github/actions/package-e2e-report/server.mjs new file mode 100644 index 0000000..52bf08a --- /dev/null +++ b/.github/actions/package-e2e-report/server.mjs @@ -0,0 +1,196 @@ +import { createReadStream, existsSync, realpathSync, statSync } from "node:fs"; +import { createServer } from "node:http"; +import { extname, isAbsolute, relative, resolve, sep } from "node:path"; + +const host = process.env.REPORT_HOST ?? "0.0.0.0"; +const port = Number.parseInt(process.env.REPORT_PORT ?? "8080", 10); +const reportRoot = realpathSync(process.env.REPORT_ROOT ?? "/app/report"); +const indexFile = resolve(reportRoot, "index.html"); + +if (!Number.isInteger(port) || port < 1 || port > 65_535) { + throw new Error( + `REPORT_PORT must be an integer from 1 through 65535; received ${port}`, + ); +} +if (!existsSync(indexFile) || !statSync(indexFile).isFile()) { + throw new Error(`Playwright report index does not exist: ${indexFile}`); +} + +const contentTypes = new Map([ + [".css", "text/css; charset=utf-8"], + [".gif", "image/gif"], + [".gz", "application/gzip"], + [".html", "text/html; charset=utf-8"], + [".ico", "image/x-icon"], + [".jpeg", "image/jpeg"], + [".jpg", "image/jpeg"], + [".js", "text/javascript; charset=utf-8"], + [".json", "application/json; charset=utf-8"], + [".map", "application/json; charset=utf-8"], + [".mjs", "text/javascript; charset=utf-8"], + [".mp4", "video/mp4"], + [".png", "image/png"], + [".svg", "image/svg+xml"], + [".ttf", "font/ttf"], + [".txt", "text/plain; charset=utf-8"], + [".webm", "video/webm"], + [".wasm", "application/wasm"], + [".webmanifest", "application/manifest+json"], + [".webp", "image/webp"], + [".woff", "font/woff"], + [".woff2", "font/woff2"], + [".xml", "application/xml; charset=utf-8"], + [".zip", "application/zip"], +]); + +const sendText = (response, statusCode, message, method = "GET") => { + const body = `${message}\n`; + response.writeHead(statusCode, { + "content-length": Buffer.byteLength(body), + "content-type": "text/plain; charset=utf-8", + "x-content-type-options": "nosniff", + }); + if (method === "HEAD") response.end(); + else response.end(body); +}; + +const isInsideReport = (path) => { + const relativePath = relative(reportRoot, path); + return ( + relativePath !== ".." && + !relativePath.startsWith(`..${sep}`) && + !isAbsolute(relativePath) + ); +}; + +const findFile = (pathname, acceptsHtml) => { + let decodedPath; + try { + decodedPath = decodeURIComponent(pathname); + } catch { + return { error: 400 }; + } + if (decodedPath.includes("\0")) return { error: 400 }; + + const candidate = resolve(reportRoot, `.${decodedPath}`); + if (!isInsideReport(candidate)) return { error: 403 }; + + let selected = candidate; + if (existsSync(selected) && statSync(selected).isDirectory()) { + selected = resolve(selected, "index.html"); + } + + if (!existsSync(selected) || !statSync(selected).isFile()) { + const hasExtension = extname(decodedPath) !== ""; + if (acceptsHtml && !hasExtension) return { path: indexFile }; + return { error: 404 }; + } + + const realPath = realpathSync(selected); + if (!isInsideReport(realPath)) return { error: 403 }; + return { path: realPath }; +}; + +const parseRange = (header, size) => { + if (!header) return null; + const match = /^bytes=(\d*)-(\d*)$/.exec(header); + if (!match || (match[1] === "" && match[2] === "")) return { error: true }; + + let start; + let end; + if (match[1] === "") { + const suffixLength = Number(match[2]); + if (!Number.isSafeInteger(suffixLength) || suffixLength < 1) { + return { error: true }; + } + start = Math.max(0, size - suffixLength); + end = size - 1; + } else { + start = Number(match[1]); + end = match[2] === "" ? size - 1 : Number(match[2]); + } + + if ( + !Number.isSafeInteger(start) || + !Number.isSafeInteger(end) || + start < 0 || + end < start || + start >= size + ) { + return { error: true }; + } + return { start, end: Math.min(end, size - 1) }; +}; + +const server = createServer((request, response) => { + const method = request.method ?? "GET"; + if (method !== "GET" && method !== "HEAD") { + response.setHeader("allow", "GET, HEAD"); + sendText(response, 405, "Method not allowed", method); + return; + } + + const url = new URL(request.url ?? "/", "http://localhost"); + const acceptsHtml = (request.headers.accept ?? "").includes("text/html"); + const selection = findFile(url.pathname, acceptsHtml); + if (selection.error) { + sendText( + response, + selection.error, + selection.error === 404 ? "Not found" : "Invalid path", + method, + ); + return; + } + + const filePath = selection.path; + const size = statSync(filePath).size; + const range = parseRange(request.headers.range, size); + if (range?.error) { + response.writeHead(416, { + "content-range": `bytes */${size}`, + "content-length": "0", + }); + response.end(); + return; + } + + const start = range?.start ?? 0; + const end = range?.end ?? Math.max(0, size - 1); + const statusCode = range ? 206 : 200; + const headers = { + "accept-ranges": "bytes", + "cache-control": + extname(filePath) === ".html" ? "no-cache" : "public, max-age=3600", + "content-length": String(size === 0 ? 0 : end - start + 1), + "content-type": + contentTypes.get(extname(filePath).toLowerCase()) ?? + "application/octet-stream", + "x-content-type-options": "nosniff", + }; + if (range) headers["content-range"] = `bytes ${start}-${end}/${size}`; + response.writeHead(statusCode, headers); + if (method === "HEAD" || size === 0) { + response.end(); + return; + } + + const stream = createReadStream(filePath, { start, end }); + stream.on("error", () => response.destroy()); + stream.pipe(response); +}); + +server.listen(port, host, () => { + console.log(`Playwright report available at http://${host}:${port}`); +}); + +const shutdown = () => { + server.close((error) => { + if (error) { + console.error(error); + process.exit(1); + } + }); +}; +process.once("SIGINT", shutdown); +process.once("SIGTERM", shutdown); diff --git a/.github/actions/plan-e2e/action.yml b/.github/actions/plan-e2e/action.yml new file mode 100644 index 0000000..e8d3e97 --- /dev/null +++ b/.github/actions/plan-e2e/action.yml @@ -0,0 +1,81 @@ +name: Plan E2E +description: Validate the E2E execution manifest and produce one deterministic CI plan. + +inputs: + profile: + description: Named execution profile from the manifest. + required: true + projects: + description: Optional newline-delimited project override. + required: false + default: "" + labels: + description: Optional newline-delimited label override. + required: false + default: "" + label-match: + description: Optional label matching override, either all or any. + required: false + default: "" + +outputs: + working-directory: + description: Repository-relative E2E package and runner context. + value: ${{ steps.plan.outputs.working-directory }} + dockerfile: + description: Dockerfile path relative to working-directory. + value: ${{ steps.plan.outputs.dockerfile }} + playwright-config: + description: Playwright config path relative to working-directory. + value: ${{ steps.plan.outputs.playwright-config }} + compose-file: + description: Docker Compose file relative to the repository root. + value: ${{ steps.plan.outputs.compose-file }} + base-url: + description: Ready SUT URL used by Playwright. + value: ${{ steps.plan.outputs.base-url }} + projects: + description: Effective newline-delimited Playwright projects. + value: ${{ steps.plan.outputs.projects }} + labels: + description: Effective newline-delimited Playwright labels. + value: ${{ steps.plan.outputs.labels }} + label-match: + description: Effective label matching mode. + value: ${{ steps.plan.outputs.label-match }} + shard-count: + description: Number of parallel Playwright shards. + value: ${{ steps.plan.outputs.shard-count }} + matrix: + description: GitHub Actions shard matrix JSON. + value: ${{ steps.plan.outputs.matrix }} + artifact-retention-days: + description: Results and image retention period. + value: ${{ steps.plan.outputs.artifact-retention-days }} + runner-content-hash: + description: SHA-256 of committed and non-ignored E2E runner inputs. + value: ${{ steps.plan.outputs.runner-content-hash }} + profile: + description: Effective execution profile name. + value: ${{ steps.plan.outputs.profile }} + +runs: + using: composite + steps: + - id: plan + name: Validate manifest and create plan + shell: bash + env: + E2E_PROFILE: ${{ inputs.profile }} + E2E_PROJECTS_OVERRIDE: ${{ inputs.projects }} + E2E_LABELS_OVERRIDE: ${{ inputs.labels }} + E2E_LABEL_MATCH_OVERRIDE: ${{ inputs.label-match }} + run: | + set -euo pipefail + ruby "$GITHUB_ACTION_PATH/plan.rb" \ + "$GITHUB_WORKSPACE" \ + "$E2E_PROFILE" \ + "$E2E_PROJECTS_OVERRIDE" \ + "$E2E_LABELS_OVERRIDE" \ + "$E2E_LABEL_MATCH_OVERRIDE" \ + "$GITHUB_OUTPUT" diff --git a/.github/actions/plan-e2e/plan.rb b/.github/actions/plan-e2e/plan.rb new file mode 100644 index 0000000..d10b4e3 --- /dev/null +++ b/.github/actions/plan-e2e/plan.rb @@ -0,0 +1,230 @@ +require "digest" +require "json" +require "open3" +require "pathname" +require "securerandom" +require "uri" +require "yaml" + +workspace_argument, profile_name, projects_override, + labels_override, label_match_override, output_file = ARGV + +def fail_plan(message) + raise ArgumentError, message +end + +def require_mapping(value, label) + fail_plan("#{label} must be a mapping") unless value.is_a?(Hash) + value +end + +def reject_unknown_keys(mapping, allowed, label) + unknown = mapping.keys.map(&:to_s) - allowed + return if unknown.empty? + + fail_plan("#{label} contains unknown keys: #{unknown.sort.join(', ')}") +end + +def require_relative_path(value, label) + fail_plan("#{label} must be a non-empty string") unless value.is_a?(String) && !value.empty? + + path = Pathname.new(value) + fail_plan("#{label} must be repository-relative") if path.absolute? + + clean = path.cleanpath.to_s + fail_plan("#{label} must remain below its parent directory") if clean == "." || clean == ".." || clean.start_with?("../") + clean +end + +def require_string_list(value, label, allow_empty: true) + fail_plan("#{label} must be a list") unless value.is_a?(Array) + fail_plan("#{label} must not be empty") if !allow_empty && value.empty? + + normalized = value.map do |entry| + fail_plan("#{label} entries must be non-empty strings") unless entry.is_a?(String) && !entry.strip.empty? + fail_plan("#{label} entries must not contain newlines") if entry.match?(/[\r\n]/) + fail_plan("#{label} entries must not contain null bytes") if entry.include?("\0") + entry.strip + end + fail_plan("#{label} entries must be unique") unless normalized.uniq.length == normalized.length + normalized +end + +def parse_override(value) + value.to_s.lines.map(&:strip).reject(&:empty?).uniq +end + +def normalize_profile(name, value) + profile = require_mapping(value, "profiles.#{name}") + reject_unknown_keys(profile, %w[projects labels labelMatch], "profiles.#{name}") + projects = require_string_list(profile["projects"], "profiles.#{name}.projects", allow_empty: false) + labels = require_string_list(profile["labels"], "profiles.#{name}.labels") + labels.each do |label| + fail_plan("labels must begin with @ and contain a name: #{label}") unless label.start_with?("@") && label.length > 1 + end + label_match = profile["labelMatch"] + fail_plan("profiles.#{name}.labelMatch must be all or any") unless %w[all any].include?(label_match) + { "projects" => projects, "labels" => labels, "labelMatch" => label_match } +end + +def append_output(file, name, value) + string = value.to_s + if string.include?("\n") + begin + delimiter = "E2E_#{SecureRandom.hex(16)}" + end while string.lines.map(&:chomp).include?(delimiter) + File.open(file, "a") do |output| + output.puts("#{name}<<#{delimiter}") + output.puts(string) + output.puts(delimiter) + end + else + File.open(file, "a") { |output| output.puts("#{name}=#{string}") } + end +end + +workspace = Pathname.new(workspace_argument).realpath +config_relative = "e2e/ci.yml" +config_path = workspace.join(config_relative) +fail_plan("e2e/ci.yml does not exist") unless config_path.file? +fail_plan("e2e/ci.yml must not be a symbolic link") if config_path.symlink? +fail_plan("e2e/ci.yml is too large") if config_path.size > 65_536 + +resolved_config = config_path.realpath +relative_config = resolved_config.relative_path_from(workspace).to_s +fail_plan("e2e/ci.yml must remain inside GITHUB_WORKSPACE") if relative_config == ".." || relative_config.start_with?("../") + +manifest = YAML.safe_load( + resolved_config.read, + permitted_classes: [], + permitted_symbols: [], + aliases: false, +) +manifest = require_mapping(manifest, "manifest") +reject_unknown_keys(manifest, %w[version runner playwright sut execution profiles], "manifest") +fail_plan("manifest version must be 1") unless manifest["version"] == 1 + +runner = require_mapping(manifest["runner"], "runner") +reject_unknown_keys(runner, %w[dockerfile], "runner") +dockerfile = require_relative_path(runner["dockerfile"], "runner.dockerfile") + +playwright = require_mapping(manifest["playwright"], "playwright") +reject_unknown_keys(playwright, %w[config], "playwright") +playwright_config = require_relative_path(playwright["config"], "playwright.config") + +sut = require_mapping(manifest["sut"], "sut") +reject_unknown_keys(sut, %w[composeFile baseUrl], "sut") +compose_file = require_relative_path(sut["composeFile"], "sut.composeFile") +begin + base_url = URI(sut["baseUrl"].to_s) +rescue URI::InvalidURIError + fail_plan("sut.baseUrl must be an HTTP or HTTPS URL") +end +fail_plan("sut.baseUrl must be an HTTP or HTTPS URL") unless %w[http https].include?(base_url.scheme) && base_url.host +fail_plan("sut.baseUrl must not contain credentials or a fragment") if base_url.userinfo || base_url.fragment + +execution = require_mapping(manifest["execution"], "execution") +reject_unknown_keys(execution, %w[shards artifactRetentionDays], "execution") +shard_count = execution["shards"] +fail_plan("execution.shards must be an integer from 1 through 32") unless shard_count.is_a?(Integer) && shard_count.between?(1, 32) +retention_days = execution["artifactRetentionDays"] +fail_plan("execution.artifactRetentionDays must be an integer from 1 through 90") unless retention_days.is_a?(Integer) && retention_days.between?(1, 90) + +profiles = require_mapping(manifest["profiles"], "profiles") +fail_plan("profiles must contain at least one profile") if profiles.empty? +normalized_profiles = profiles.each_with_object({}) do |(name, value), result| + fail_plan("profile names must be non-empty strings") unless name.is_a?(String) && !name.empty? + result[name] = normalize_profile(name, value) +end +fail_plan("profile must be a non-empty string") unless profile_name.is_a?(String) && !profile_name.empty? +unless normalized_profiles.key?(profile_name) + fail_plan("unknown profile #{profile_name}; available profiles: #{normalized_profiles.keys.sort.join(', ')}") +end +profile = normalized_profiles.fetch(profile_name) +projects = profile.fetch("projects") +labels = profile.fetch("labels") +label_match = profile.fetch("labelMatch") + +override_projects = parse_override(projects_override) +override_labels = parse_override(labels_override) +projects = override_projects unless override_projects.empty? +labels = override_labels unless override_labels.empty? +unless label_match_override.to_s.empty? + fail_plan("label-match override must be all or any") unless %w[all any].include?(label_match_override) + label_match = label_match_override +end +labels.each do |label| + fail_plan("labels must begin with @ and contain a name: #{label}") unless label.start_with?("@") && label.length > 1 +end + +working_directory_path = resolved_config.dirname.realpath +working_directory = working_directory_path.relative_path_from(workspace).to_s +fail_plan("e2e/ci.yml must live below the repository root") if working_directory == "." || working_directory.start_with?("../") + +dockerfile_path = working_directory_path.join(dockerfile).cleanpath +playwright_config_path = working_directory_path.join(playwright_config).cleanpath +compose_path = workspace.join(compose_file).cleanpath +[ + [dockerfile_path, "runner.dockerfile"], + [playwright_config_path, "playwright.config"], + [compose_path, "sut.composeFile"], +].each do |path, label| + fail_plan("#{label} does not exist: #{path}") unless path.file? +end +fail_plan("runner.dockerfile must remain inside the E2E directory") unless dockerfile_path.realpath.to_s.start_with?("#{working_directory_path}/") +fail_plan("playwright.config must remain inside the E2E directory") unless playwright_config_path.realpath.to_s.start_with?("#{working_directory_path}/") +fail_plan("sut.composeFile must remain inside the repository") unless compose_path.realpath.to_s.start_with?("#{workspace}/") + +files_output, files_error, files_status = Open3.capture3( + "git", + "-C", + workspace.to_s, + "ls-files", + "-z", + "--cached", + "--others", + "--exclude-standard", + "--", + working_directory, +) +fail_plan("cannot enumerate E2E sources: #{files_error.strip}") unless files_status.success? +files = files_output.split("\0").reject(&:empty?).sort +files.reject! { |relative_file| relative_file == relative_config } +fail_plan("E2E directory contains no source files") if files.empty? + +digest = Digest::SHA256.new +digest.update("e2e-runner-content-v1\0") +digest.update("runner.dockerfile\0") +digest.update(dockerfile) +digest.update("\0") +files.each do |relative_file| + path = workspace.join(relative_file) + digest.update(relative_file) + digest.update("\0") + digest.update(path.lstat.mode.to_s(8)) + digest.update("\0") + if path.symlink? + digest.update("symlink\0") + digest.update(path.readlink.to_s) + elsif path.file? + digest.update("file\0") + digest.update(path.binread) + else + fail_plan("unsupported E2E source type: #{relative_file}") + end + digest.update("\0") +end + +append_output(output_file, "working-directory", working_directory) +append_output(output_file, "dockerfile", dockerfile) +append_output(output_file, "playwright-config", playwright_config) +append_output(output_file, "compose-file", compose_file) +append_output(output_file, "base-url", base_url.to_s) +append_output(output_file, "projects", projects.join("\n")) +append_output(output_file, "labels", labels.join("\n")) +append_output(output_file, "label-match", label_match) +append_output(output_file, "shard-count", shard_count) +append_output(output_file, "matrix", { shard: (1..shard_count).to_a }.to_json) +append_output(output_file, "artifact-retention-days", retention_days) +append_output(output_file, "runner-content-hash", digest.hexdigest) +append_output(output_file, "profile", profile_name) diff --git a/.github/actions/publish-e2e-results/action.yml b/.github/actions/publish-e2e-results/action.yml new file mode 100644 index 0000000..9e69ea9 --- /dev/null +++ b/.github/actions/publish-e2e-results/action.yml @@ -0,0 +1,116 @@ +name: Publish E2E Results +description: Finalize, publish, summarize, and enforce the final E2E result. + +inputs: + results-directory: + description: Repository-relative directory containing merged E2E results. + required: false + default: e2e-results + expected-shards: + description: Number of shard results required by the run. + required: true + artifact-name: + description: Name of the durable merged results artifact. + required: false + default: e2e-results + retention-days: + description: Number of days GitHub retains the merged results artifact. + required: false + default: "10" + runner-download-outcome: + description: Outcome of downloading the immutable runner archive. + required: true + runner-load-outcome: + description: Outcome of loading the immutable runner archive. + required: true + runner-verification-outcome: + description: Outcome of verifying the loaded runner image ID. + required: true + shard-download-outcome: + description: Outcome of downloading shard results. + required: true + merge-outcome: + description: Outcome of deterministic results assembly. + required: true + report-publication-outcome: + description: Outcome of building and publishing the report image archive. + required: true + +outputs: + result: + description: "Final result: passed, failed, or infrastructure-error." + value: ${{ steps.evaluate.outputs.result }} + artifact-name: + description: Merged results artifact name. + value: ${{ inputs.artifact-name }} + total: + description: Aggregated test count. + value: ${{ steps.evaluate.outputs.total }} + passed: + description: Aggregated passed test count. + value: ${{ steps.evaluate.outputs.passed }} + failed: + description: Aggregated failed test count. + value: ${{ steps.evaluate.outputs.failed }} + skipped: + description: Aggregated skipped test count. + value: ${{ steps.evaluate.outputs.skipped }} + result-file: + description: Absolute path to the finalized result.json. + value: ${{ steps.prepare.outputs.result-file }} + +runs: + using: composite + steps: + - id: prepare + name: Finalize final result + shell: bash + env: + E2E_RESULTS_DIRECTORY: ${{ inputs.results-directory }} + E2E_EXPECTED_SHARDS: ${{ inputs.expected-shards }} + E2E_RUNNER_DOWNLOAD_OUTCOME: ${{ inputs.runner-download-outcome }} + E2E_RUNNER_LOAD_OUTCOME: ${{ inputs.runner-load-outcome }} + E2E_RUNNER_VERIFICATION_OUTCOME: ${{ inputs.runner-verification-outcome }} + E2E_SHARD_DOWNLOAD_OUTCOME: ${{ inputs.shard-download-outcome }} + E2E_MERGE_OUTCOME: ${{ inputs.merge-outcome }} + E2E_REPORT_PUBLICATION_OUTCOME: ${{ inputs.report-publication-outcome }} + run: | + set -euo pipefail + node "$GITHUB_ACTION_PATH/prepare-publication.mjs" \ + "$GITHUB_WORKSPACE" \ + "$E2E_RESULTS_DIRECTORY" \ + "$E2E_EXPECTED_SHARDS" \ + "$E2E_RUNNER_DOWNLOAD_OUTCOME" \ + "$E2E_RUNNER_LOAD_OUTCOME" \ + "$E2E_RUNNER_VERIFICATION_OUTCOME" \ + "$E2E_SHARD_DOWNLOAD_OUTCOME" \ + "$E2E_MERGE_OUTCOME" \ + "$E2E_REPORT_PUBLICATION_OUTCOME" \ + "$GITHUB_OUTPUT" + + - id: publish + name: Upload final E2E results + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: ${{ inputs.artifact-name }} + path: ${{ steps.prepare.outputs.results-directory }}/ + if-no-files-found: error + retention-days: ${{ inputs.retention-days }} + + - id: evaluate + name: Publish summary and enforce result + if: always() && steps.prepare.outcome == 'success' + shell: bash + env: + E2E_RESULT_FILE: ${{ steps.prepare.outputs.result-file }} + E2E_ARTIFACT_NAME: ${{ inputs.artifact-name }} + E2E_PUBLICATION_OUTCOME: ${{ steps.publish.outcome }} + run: | + set -euo pipefail + node "$GITHUB_ACTION_PATH/evaluate-publication.mjs" \ + "$E2E_RESULT_FILE" \ + "$E2E_ARTIFACT_NAME" \ + "${E2E_PUBLICATION_OUTCOME:-failure}" \ + "$GITHUB_STEP_SUMMARY" \ + "$GITHUB_OUTPUT" diff --git a/.github/actions/publish-e2e-results/evaluate-publication.mjs b/.github/actions/publish-e2e-results/evaluate-publication.mjs new file mode 100644 index 0000000..429b42d --- /dev/null +++ b/.github/actions/publish-e2e-results/evaluate-publication.mjs @@ -0,0 +1,63 @@ +import { appendFileSync, readFileSync } from "node:fs"; + +const [resultFile, artifactName, publicationOutcome, summaryFile, outputFile] = + process.argv.slice(2); +const result = JSON.parse(readFileSync(resultFile, "utf8")); +const summary = result.summary ?? {}; +const enforcedResult = + result.result === "passed" && publicationOutcome !== "success" + ? "infrastructure-error" + : result.result; +const publicationFailure = + publicationOutcome === "success" + ? null + : `Results artifact publication finished with ${publicationOutcome || "unknown"}`; +const primaryFailure = result.primaryFailure?.message ?? publicationFailure; + +appendFileSync( + summaryFile, + [ + "## E2E Gate", + "", + `**Result:** ${enforcedResult}`, + "", + "| Result | Count |", + "| --- | ---: |", + `| Total | ${summary.total ?? 0} |`, + `| Passed | ${summary.passed ?? 0} |`, + `| Failed | ${summary.failed ?? 0} |`, + `| Skipped | ${summary.skipped ?? 0} |`, + "", + `**Shards:** ${(result.observedShards ?? []).length}/${result.expectedShards ?? 0}`, + "", + `**Results artifact:** \`${artifactName}\``, + ...(primaryFailure ? ["", `**Primary failure:** ${primaryFailure}`] : []), + ...(publicationFailure && publicationFailure !== primaryFailure + ? ["", `**Publication failure:** ${publicationFailure}`] + : []), + "", + ].join("\n"), +); + +const appendOutput = (name, value) => { + const escaped = String(value) + .replaceAll("%", "%25") + .replaceAll("\r", "%0D") + .replaceAll("\n", "%0A"); + appendFileSync(outputFile, `${name}=${escaped}\n`); +}; +appendOutput("result", enforcedResult); +appendOutput("total", Number(summary.total ?? 0)); +appendOutput("passed", Number(summary.passed ?? 0)); +appendOutput("failed", Number(summary.failed ?? 0)); +appendOutput("skipped", Number(summary.skipped ?? 0)); +appendOutput("primary-failure", primaryFailure ?? ""); + +if (enforcedResult !== "passed" || publicationOutcome !== "success") { + console.error(`E2E results did not complete successfully: ${enforcedResult}`); + if (primaryFailure) console.error(primaryFailure); + if (publicationFailure && publicationFailure !== primaryFailure) { + console.error(publicationFailure); + } + process.exit(1); +} diff --git a/.github/actions/publish-e2e-results/prepare-publication.mjs b/.github/actions/publish-e2e-results/prepare-publication.mjs new file mode 100644 index 0000000..c8cd5c7 --- /dev/null +++ b/.github/actions/publish-e2e-results/prepare-publication.mjs @@ -0,0 +1,137 @@ +import { + existsSync, + lstatSync, + mkdirSync, + readFileSync, + realpathSync, + writeFileSync, +} from "node:fs"; +import { isAbsolute, relative, resolve, sep } from "node:path"; + +const [ + workspace, + requestedResultsDirectory, + requestedExpectedShards, + runnerDownloadOutcome, + runnerLoadOutcome, + runnerVerificationOutcome, + shardDownloadOutcome, + mergeOutcome, + reportPublicationOutcome, + outputFile, +] = process.argv.slice(2); + +const expectedShards = Number(requestedExpectedShards); +if ( + !Number.isSafeInteger(expectedShards) || + expectedShards < 1 || + expectedShards > 100 +) { + throw new Error("expected-shards must be an integer between 1 and 100"); +} +if (!requestedResultsDirectory || isAbsolute(requestedResultsDirectory)) { + throw new Error("results-directory must be a non-empty repository-relative path"); +} + +const workspacePath = realpathSync(workspace); +const resultsDirectory = resolve(workspacePath, requestedResultsDirectory); +const relativeResultsDirectory = relative(workspacePath, resultsDirectory); +if ( + relativeResultsDirectory === "" || + relativeResultsDirectory === ".." || + relativeResultsDirectory.startsWith(`..${sep}`) || + isAbsolute(relativeResultsDirectory) +) { + throw new Error("results-directory must resolve below GITHUB_WORKSPACE"); +} +if ( + existsSync(resultsDirectory) && + lstatSync(resultsDirectory).isSymbolicLink() +) { + throw new Error("results-directory must not be a symbolic link"); +} +mkdirSync(resultsDirectory, { recursive: true }); + +const resultFile = resolve(resultsDirectory, "result.json"); +let result = null; +if (existsSync(resultFile)) { + try { + result = JSON.parse(readFileSync(resultFile, "utf8")); + } catch { + result = null; + } +} +if (!result || typeof result !== "object") { + result = { + schemaVersion: 2, + result: "infrastructure-error", + primaryFailure: { + kind: "infrastructure", + phase: "results-assembly", + message: "Results assembly did not produce a readable result.json", + }, + secondaryFailures: [], + expectedShards, + observedShards: [], + summary: { + total: 0, + passed: 0, + failed: 0, + skipped: 0, + pending: 0, + other: 0, + }, + shards: [], + reports: {}, + generatedAt: new Date().toISOString(), + }; +} + +const checks = [ + ["runner-archive-download", runnerDownloadOutcome], + ["runner-image-load", runnerLoadOutcome], + ["runner-image-verification", runnerVerificationOutcome], + ["shard-results-download", shardDownloadOutcome], + ["results-merge", mergeOutcome], + ["report-image-publication", reportPublicationOutcome], +]; +const newFailures = checks + .filter(([, outcome]) => outcome !== "success") + .map(([phase, outcome]) => ({ + kind: "infrastructure", + phase, + message: `${phase} finished with ${outcome || "unknown"}`, + })); + +result.secondaryFailures = Array.isArray(result.secondaryFailures) + ? result.secondaryFailures + : []; +if (newFailures.length > 0) { + if (!result.primaryFailure) { + result.primaryFailure = newFailures.shift(); + } + result.secondaryFailures.push(...newFailures); + if (result.result === "passed" || !result.result) { + result.result = "infrastructure-error"; + } +} +result.expectedShards = expectedShards; +result.artifacts = { + ...(result.artifacts ?? {}), + reportImage: + reportPublicationOutcome === "success" + ? { published: true, name: "e2e-report-image" } + : { published: false, name: null }, +}; +result.finalizedAt = new Date().toISOString(); +writeFileSync(resultFile, `${JSON.stringify(result, null, 2)}\n`); + +const appendOutput = (name, value) => { + const escaped = String(value) + .replaceAll("%", "%25") + .replaceAll("\r", "%0D") + .replaceAll("\n", "%0A"); + writeFileSync(outputFile, `${name}=${escaped}\n`, { flag: "a" }); +}; +appendOutput("results-directory", resultsDirectory); +appendOutput("result-file", resultFile); diff --git a/.github/actions/run-e2e/action.yml b/.github/actions/run-e2e/action.yml index 776c5b2..c83ecbe 100644 --- a/.github/actions/run-e2e/action.yml +++ b/.github/actions/run-e2e/action.yml @@ -1,5 +1,5 @@ name: Run E2E Shard -description: Run one Playwright shard and always leave machine-readable shard evidence. +description: Run one Playwright shard and always leave machine-readable shard results. inputs: working-directory: @@ -10,6 +10,38 @@ inputs: description: Path to the Playwright configuration, relative to working-directory. required: false default: playwright.config.ts + compose-file: + description: Docker Compose file for the SUT, relative to the repository root. + required: false + default: compose.e2e.yml + base-url: + description: Base URL exposed by the Dockerized SUT. + required: false + default: http://127.0.0.1:4173 + runner-image-name: + description: Exact local Docker image reference loaded from the runner artifact. + required: true + runner-image-id: + description: Expected content-addressed Docker image ID. + required: true + runner-download-outcome: + description: Outcome of the workflow step that downloaded the runner image artifact. + required: true + runner-load-outcome: + description: Outcome of the workflow step that loaded the runner image archive. + required: true + projects: + description: Newline-delimited Playwright project names; empty selects all projects. + required: false + default: "" + labels: + description: Newline-delimited Playwright tags beginning with @; empty selects all tags. + required: false + default: "" + label-match: + description: Label matching mode, either all or any. + required: false + default: all shard-index: description: One-based shard index. required: true @@ -23,27 +55,36 @@ inputs: outputs: result: - description: Shard result, either passed or failed. - value: ${{ steps.execute.outputs.result }} + description: "Shard result: passed, failed, or infrastructure-error." + value: ${{ steps.finalize.outputs.result }} exit-code: - description: Playwright process exit code. - value: ${{ steps.execute.outputs.exit-code }} + description: Effective shard exit code after SUT, Playwright, results, and cleanup checks. + value: ${{ steps.finalize.outputs.exit-code }} shard-directory: - description: Absolute path containing this shard's evidence. - value: ${{ steps.execute.outputs.shard-directory }} + description: Absolute path containing this shard's results. + value: ${{ steps.initialize.outputs.shard-directory }} status-file: description: Absolute path to this shard's status JSON. - value: ${{ steps.execute.outputs.status-file }} + value: ${{ steps.initialize.outputs.status-file }} runs: using: composite steps: - - name: Run Playwright shard - id: execute + - name: Initialize shard results + id: initialize shell: bash env: E2E_WORKING_DIRECTORY: ${{ inputs.working-directory }} E2E_PLAYWRIGHT_CONFIG: ${{ inputs.playwright-config }} + E2E_COMPOSE_FILE: ${{ inputs.compose-file }} + E2E_BASE_URL: ${{ inputs.base-url }} + E2E_RUNNER_IMAGE_NAME: ${{ inputs.runner-image-name }} + E2E_RUNNER_IMAGE_ID: ${{ inputs.runner-image-id }} + E2E_RUNNER_DOWNLOAD_OUTCOME: ${{ inputs.runner-download-outcome }} + E2E_RUNNER_LOAD_OUTCOME: ${{ inputs.runner-load-outcome }} + E2E_PROJECTS: ${{ inputs.projects }} + E2E_LABELS: ${{ inputs.labels }} + E2E_LABEL_MATCH: ${{ inputs.label-match }} E2E_SHARD_INDEX: ${{ inputs.shard-index }} E2E_SHARD_TOTAL: ${{ inputs.shard-total }} E2E_RESULTS_DIRECTORY: ${{ inputs.results-directory }} @@ -54,40 +95,224 @@ runs: "$GITHUB_WORKSPACE" \ "$E2E_WORKING_DIRECTORY" \ "$E2E_PLAYWRIGHT_CONFIG" \ + "$E2E_COMPOSE_FILE" \ + "$E2E_BASE_URL" \ + "$E2E_RUNNER_IMAGE_NAME" \ + "$E2E_RUNNER_IMAGE_ID" \ + "$E2E_RUNNER_DOWNLOAD_OUTCOME" \ + "$E2E_RUNNER_LOAD_OUTCOME" \ + "$E2E_PROJECTS" \ + "$E2E_LABELS" \ + "$E2E_LABEL_MATCH" \ "$E2E_SHARD_INDEX" \ "$E2E_SHARD_TOTAL" \ "$E2E_RESULTS_DIRECTORY")" - set +e - ( - cd "$E2E_ABSOLUTE_WORKING_DIRECTORY" - CI=true \ - E2E_RESULTS_DIR="$E2E_SHARD_DIRECTORY/test-results" \ - PLAYWRIGHT_BLOB_OUTPUT_DIR="$E2E_SHARD_DIRECTORY/blob-report" \ - PLAYWRIGHT_OUTPUT_DIR="$E2E_SHARD_DIRECTORY/test-results" \ - CTRF_OUTPUT_FILE="$E2E_SHARD_DIRECTORY/ctrf/ctrf-report.json" \ - pnpm exec playwright test \ - --config "$E2E_ABSOLUTE_CONFIG" \ - --shard="$E2E_SHARD_INDEX/$E2E_SHARD_TOTAL" - ) - playwright_exit_code=$? - set -e + { + echo "working-directory=$E2E_ABSOLUTE_WORKING_DIRECTORY" + echo "config=$E2E_CONTAINER_CONFIG" + echo "compose-file=$E2E_ABSOLUTE_COMPOSE_FILE" + echo "base-url=$E2E_BASE_URL" + echo "compose-project=$E2E_COMPOSE_PROJECT" + echo "runner-image-name=$E2E_RUNNER_IMAGE_NAME" + echo "runner-image-id=$E2E_RUNNER_IMAGE_ID" + echo "selection-file=$E2E_SELECTION_FILE" + echo "shard-directory=$E2E_SHARD_DIRECTORY" + echo "status-file=$E2E_STATUS_FILE" + } >> "$GITHUB_OUTPUT" - node "$GITHUB_ACTION_PATH/complete-shard.mjs" \ - "$E2E_STATUS_FILE" \ - "$playwright_exit_code" + - name: Verify loaded E2E runner image + id: runner + if: >- + always() && steps.initialize.outcome == 'success' && + inputs.runner-download-outcome == 'success' && + inputs.runner-load-outcome == 'success' + continue-on-error: true + shell: bash + env: + E2E_RUNNER_IMAGE_NAME: ${{ steps.initialize.outputs.runner-image-name }} + E2E_RUNNER_IMAGE_ID: ${{ steps.initialize.outputs.runner-image-id }} + E2E_SHARD_DIRECTORY: ${{ steps.initialize.outputs.shard-directory }} + run: | + set +e + runner_log="$E2E_SHARD_DIRECTORY/runner-image.log" + actual_image_id="$(docker image inspect \ + --format '{{.Id}}' "$E2E_RUNNER_IMAGE_NAME" 2> "$runner_log")" + runner_exit_code=$? - if [[ "$playwright_exit_code" -eq 0 ]]; then - result=passed - else - result=failed + if [[ "$runner_exit_code" -eq 0 && "$actual_image_id" != "$E2E_RUNNER_IMAGE_ID" ]]; then + echo "Runner image ID mismatch: expected $E2E_RUNNER_IMAGE_ID, got $actual_image_id" \ + >> "$runner_log" + runner_exit_code=42 fi + cat "$runner_log" { - echo "result=$result" - echo "exit-code=$playwright_exit_code" - echo "shard-directory=$E2E_SHARD_DIRECTORY" - echo "status-file=$E2E_STATUS_FILE" + echo "exit-code=$runner_exit_code" + echo "actual-image-id=$actual_image_id" } >> "$GITHUB_OUTPUT" + exit "$runner_exit_code" + + - name: Start Dockerized SUT + id: start + if: >- + always() && steps.initialize.outcome == 'success' && + steps.runner.outputs.exit-code == '0' + continue-on-error: true + shell: bash + env: + E2E_COMPOSE_FILE: ${{ steps.initialize.outputs.compose-file }} + E2E_COMPOSE_PROJECT: ${{ steps.initialize.outputs.compose-project }} + E2E_SHARD_DIRECTORY: ${{ steps.initialize.outputs.shard-directory }} + run: | + set +e + set -uo pipefail + startup_log="$E2E_SHARD_DIRECTORY/docker-compose-startup.log" + docker compose \ + --file "$E2E_COMPOSE_FILE" \ + --project-name "$E2E_COMPOSE_PROJECT" \ + up --detach --build --wait > "$startup_log" 2>&1 + startup_exit_code=$? + cat "$startup_log" + echo "exit-code=$startup_exit_code" >> "$GITHUB_OUTPUT" + exit "$startup_exit_code" + - name: Run Playwright shard + id: playwright + if: >- + steps.initialize.outcome == 'success' && + steps.runner.outputs.exit-code == '0' && + steps.start.outputs.exit-code == '0' + continue-on-error: true + shell: bash + env: + E2E_CONFIG: ${{ steps.initialize.outputs.config }} + E2E_BASE_URL: ${{ steps.initialize.outputs.base-url }} + E2E_RUNNER_IMAGE_ID: ${{ steps.runner.outputs.actual-image-id }} + E2E_SHARD_DIRECTORY: ${{ steps.initialize.outputs.shard-directory }} + E2E_SELECTION_FILE: ${{ steps.initialize.outputs.selection-file }} + E2E_SHARD_INDEX: ${{ inputs.shard-index }} + E2E_SHARD_TOTAL: ${{ inputs.shard-total }} + run: | + set +e + selection_args_file="$E2E_SHARD_DIRECTORY/.playwright-selection-args" + node "$GITHUB_ACTION_PATH/playwright-selection.mjs" \ + emit-args "$E2E_SELECTION_FILE" > "$selection_args_file" + selection_exit_code=$? + if [[ "$selection_exit_code" -ne 0 ]]; then + echo "exit-code=$selection_exit_code" >> "$GITHUB_OUTPUT" + echo "failure-kind=selection" >> "$GITHUB_OUTPUT" + exit "$selection_exit_code" + fi + + playwright_selection_args=() + while IFS= read -r -d '' argument; do + playwright_selection_args+=("$argument") + done < "$selection_args_file" + ( + docker run --rm \ + --init \ + --network=host \ + --ipc=host \ + --read-only \ + --tmpfs /tmp:rw,nosuid,nodev,size=512m \ + --cap-drop=ALL \ + --security-opt=no-new-privileges:true \ + --pids-limit=1024 \ + --user "$(id -u):$(id -g)" \ + --env BASE_URL="$E2E_BASE_URL" \ + --env CI=true \ + --env HOME=/tmp \ + --env E2E_RESULTS_DIR=/test-output/test-results \ + --env PLAYWRIGHT_BLOB_OUTPUT_DIR=/test-output/blob-report \ + --env PLAYWRIGHT_OUTPUT_DIR=/test-output/test-results \ + --env CTRF_OUTPUT_FILE=/test-output/ctrf/ctrf-report.json \ + --volume "$E2E_SHARD_DIRECTORY:/test-output" \ + "$E2E_RUNNER_IMAGE_ID" \ + --config "$E2E_CONFIG" \ + --shard="$E2E_SHARD_INDEX/$E2E_SHARD_TOTAL" \ + "${playwright_selection_args[@]}" + ) + playwright_exit_code=$? + echo "exit-code=$playwright_exit_code" >> "$GITHUB_OUTPUT" + if [[ "$playwright_exit_code" =~ ^12[567]$ ]] || \ + [[ "$playwright_exit_code" -ge 128 ]]; then + echo "failure-kind=runner-container" >> "$GITHUB_OUTPUT" + else + echo "failure-kind=test" >> "$GITHUB_OUTPUT" + fi exit "$playwright_exit_code" + + - name: Capture Dockerized SUT logs + id: logs + if: always() && steps.initialize.outcome == 'success' + continue-on-error: true + shell: bash + env: + E2E_COMPOSE_FILE: ${{ steps.initialize.outputs.compose-file }} + E2E_COMPOSE_PROJECT: ${{ steps.initialize.outputs.compose-project }} + E2E_SHARD_DIRECTORY: ${{ steps.initialize.outputs.shard-directory }} + run: | + set +e + service_log="$E2E_SHARD_DIRECTORY/docker-compose.log" + docker compose \ + --file "$E2E_COMPOSE_FILE" \ + --project-name "$E2E_COMPOSE_PROJECT" \ + logs --no-color --timestamps > "$service_log" 2>&1 + log_exit_code=$? + cat "$service_log" + echo "exit-code=$log_exit_code" >> "$GITHUB_OUTPUT" + exit "$log_exit_code" + + - name: Tear down Dockerized SUT + id: teardown + if: always() && steps.initialize.outcome == 'success' + continue-on-error: true + shell: bash + env: + E2E_COMPOSE_FILE: ${{ steps.initialize.outputs.compose-file }} + E2E_COMPOSE_PROJECT: ${{ steps.initialize.outputs.compose-project }} + E2E_SHARD_DIRECTORY: ${{ steps.initialize.outputs.shard-directory }} + run: | + set +e + teardown_log="$E2E_SHARD_DIRECTORY/docker-compose-teardown.log" + docker compose \ + --file "$E2E_COMPOSE_FILE" \ + --project-name "$E2E_COMPOSE_PROJECT" \ + down --volumes --remove-orphans > "$teardown_log" 2>&1 + teardown_exit_code=$? + cat "$teardown_log" + echo "exit-code=$teardown_exit_code" >> "$GITHUB_OUTPUT" + exit "$teardown_exit_code" + + - name: Finalize shard results + id: finalize + if: always() && steps.initialize.outcome == 'success' + shell: bash + env: + E2E_STATUS_FILE: ${{ steps.initialize.outputs.status-file }} + E2E_RUNNER_DOWNLOAD_OUTCOME: ${{ inputs.runner-download-outcome }} + E2E_RUNNER_LOAD_OUTCOME: ${{ inputs.runner-load-outcome }} + E2E_RUNNER_EXIT_CODE: ${{ steps.runner.outputs.exit-code }} + E2E_ACTUAL_RUNNER_IMAGE_ID: ${{ steps.runner.outputs.actual-image-id }} + E2E_STARTUP_EXIT_CODE: ${{ steps.start.outputs.exit-code }} + E2E_PLAYWRIGHT_EXIT_CODE: ${{ steps.playwright.outputs.exit-code }} + E2E_PLAYWRIGHT_FAILURE_KIND: ${{ steps.playwright.outputs.failure-kind }} + E2E_LOG_EXIT_CODE: ${{ steps.logs.outputs.exit-code }} + E2E_TEARDOWN_EXIT_CODE: ${{ steps.teardown.outputs.exit-code }} + run: | + set +e + node "$GITHUB_ACTION_PATH/complete-shard.mjs" \ + "$E2E_STATUS_FILE" \ + "$E2E_RUNNER_DOWNLOAD_OUTCOME" \ + "$E2E_RUNNER_LOAD_OUTCOME" \ + "$E2E_RUNNER_EXIT_CODE" \ + "$E2E_ACTUAL_RUNNER_IMAGE_ID" \ + "$E2E_STARTUP_EXIT_CODE" \ + "$E2E_PLAYWRIGHT_EXIT_CODE" \ + "$E2E_PLAYWRIGHT_FAILURE_KIND" \ + "$E2E_LOG_EXIT_CODE" \ + "$E2E_TEARDOWN_EXIT_CODE" \ + "$GITHUB_OUTPUT" + completion_exit_code=$? + exit "$completion_exit_code" diff --git a/.github/actions/run-e2e/complete-shard.mjs b/.github/actions/run-e2e/complete-shard.mjs index 384415a..77534b8 100644 --- a/.github/actions/run-e2e/complete-shard.mjs +++ b/.github/actions/run-e2e/complete-shard.mjs @@ -1,19 +1,221 @@ -import { readFileSync, writeFileSync } from "node:fs"; +import { appendFileSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname } from "node:path"; -const [statusFile, requestedExitCode] = process.argv.slice(2); -const exitCode = Number(requestedExitCode); +const [ + statusFile, + runnerDownloadOutcome, + runnerLoadOutcome, + requestedRunnerExitCode, + actualRunnerImageId, + requestedStartupExitCode, + requestedPlaywrightExitCode, + requestedPlaywrightFailureKind, + requestedLogExitCode, + requestedTeardownExitCode, + outputFile, +] = process.argv.slice(2); +const parseExitCode = (requestedValue, label) => { + if (requestedValue === "") return null; + const value = Number(requestedValue); + if (!Number.isSafeInteger(value) || value < 0) { + throw new Error(`Invalid ${label} exit code: ${requestedValue}`); + } + return value; +}; -if (!Number.isSafeInteger(exitCode) || exitCode < 0) { - throw new Error(`Invalid Playwright exit code: ${requestedExitCode}`); -} +const startupExitCode = parseExitCode( + requestedStartupExitCode, + "startup", +); +const runnerExitCode = parseExitCode( + requestedRunnerExitCode, + "runner image verification", +); +const playwrightExitCode = parseExitCode( + requestedPlaywrightExitCode, + "Playwright", +); +const logExitCode = parseExitCode(requestedLogExitCode, "log capture"); +const teardownExitCode = parseExitCode( + requestedTeardownExitCode, + "teardown", +); const status = JSON.parse(readFileSync(statusFile, "utf8")); const finishedAt = new Date(); const startedAt = new Date(status.startedAt); -status.result = exitCode === 0 ? "passed" : "failed"; -status.exitCode = exitCode; +const failures = []; +const runnerTransportSucceeded = + runnerDownloadOutcome === "success" && runnerLoadOutcome === "success"; +if (runnerDownloadOutcome !== "success") { + failures.push({ + kind: "infrastructure", + phase: "runner-image-download", + message: `E2E runner image download outcome was ${runnerDownloadOutcome}`, + }); +} +if (runnerLoadOutcome !== "success") { + failures.push({ + kind: "infrastructure", + phase: "runner-image-load", + message: `E2E runner image load outcome was ${runnerLoadOutcome}`, + }); +} +if (runnerTransportSucceeded && runnerExitCode === null) { + failures.push({ + kind: "infrastructure", + phase: "runner-image", + message: "E2E runner image verification did not report an exit code", + }); +} else if (runnerTransportSucceeded && runnerExitCode !== 0) { + failures.push({ + kind: "infrastructure", + phase: "runner-image", + message: `E2E runner image verification exited with ${runnerExitCode}`, + }); +} +if ( + runnerTransportSucceeded && + runnerExitCode === 0 && + startupExitCode === null +) { + failures.push({ + kind: "infrastructure", + phase: "sut-startup", + message: "Dockerized SUT startup did not report an exit code", + }); +} else if ( + runnerTransportSucceeded && + runnerExitCode === 0 && + startupExitCode !== 0 +) { + failures.push({ + kind: "infrastructure", + phase: "sut-startup", + message: `Dockerized SUT startup exited with ${startupExitCode}`, + }); +} +if (runnerExitCode === 0 && startupExitCode === 0 && playwrightExitCode === null) { + failures.push({ + kind: "infrastructure", + phase: "playwright-execution", + message: "Playwright execution did not report an exit code", + }); +} else if ( + playwrightExitCode !== null && + playwrightExitCode > 0 && + requestedPlaywrightFailureKind !== "test" +) { + failures.push({ + kind: "infrastructure", + phase: + requestedPlaywrightFailureKind === "selection" + ? "playwright-selection" + : "runner-container", + message: + requestedPlaywrightFailureKind === "selection" + ? `Playwright selection preparation exited with ${playwrightExitCode}` + : `E2E runner container exited with ${playwrightExitCode}`, + }); +} else if (playwrightExitCode !== null && playwrightExitCode > 0) { + failures.push({ + kind: "test", + phase: "playwright", + message: `Playwright exited with ${playwrightExitCode}`, + }); +} +if (logExitCode === null) { + failures.push({ + kind: "infrastructure", + phase: "results", + message: "Docker Compose log capture did not report an exit code", + }); +} else if (logExitCode !== 0) { + failures.push({ + kind: "infrastructure", + phase: "results", + message: `Docker Compose log capture exited with ${logExitCode}`, + }); +} +if (teardownExitCode === null) { + failures.push({ + kind: "infrastructure", + phase: "cleanup", + message: "Docker Compose teardown did not report an exit code", + }); +} else if (teardownExitCode !== 0) { + failures.push({ + kind: "infrastructure", + phase: "cleanup", + message: `Docker Compose teardown exited with ${teardownExitCode}`, + }); +} + +const testFailure = failures.find((failure) => failure.kind === "test"); +const infrastructureFailure = failures.find( + (failure) => failure.kind === "infrastructure", +); +status.primaryFailure = testFailure ?? infrastructureFailure ?? null; +status.secondaryFailures = failures.filter( + (failure) => failure !== status.primaryFailure, +); +status.result = testFailure + ? "failed" + : infrastructureFailure + ? "infrastructure-error" + : "passed"; + +const exitCodeForFailure = (failure) => { + switch (failure?.phase) { + case "playwright": + case "playwright-selection": + case "runner-container": + return playwrightExitCode ?? 1; + case "runner-image": + return runnerExitCode ?? 1; + case "sut-startup": + return startupExitCode ?? 1; + case "results": + return logExitCode ?? 1; + case "cleanup": + return teardownExitCode ?? 1; + default: + return failure ? 1 : 0; + } +}; +const effectiveExitCode = exitCodeForFailure(status.primaryFailure); +status.exitCode = effectiveExitCode; +status.lifecycle.runner.verificationExitCode = runnerExitCode; +status.lifecycle.runner.actualImageId = actualRunnerImageId || null; +status.lifecycle.sut.startupExitCode = startupExitCode; +status.lifecycle.playwright = { + exitCode: playwrightExitCode, + started: playwrightExitCode !== null, +}; +status.lifecycle.results.logCaptureExitCode = logExitCode; +status.lifecycle.cleanup.teardownExitCode = teardownExitCode; status.finishedAt = finishedAt.toISOString(); status.durationMs = Math.max(0, finishedAt.getTime() - startedAt.getTime()); writeFileSync(statusFile, `${JSON.stringify(status, null, 2)}\n`); + +for (const [name, value] of [ + ["result", status.result], + ["exit-code", effectiveExitCode], + ["shard-directory", dirname(statusFile)], + ["status-file", statusFile], +]) { + appendFileSync(outputFile, `${name}=${value}\n`); +} + +if (status.primaryFailure) { + console.error(`${status.primaryFailure.kind}: ${status.primaryFailure.message}`); +} +for (const failure of status.secondaryFailures) { + console.error(`additional ${failure.kind}: ${failure.message}`); +} + +if (status.result !== "passed") { + process.exit(effectiveExitCode || 1); +} diff --git a/.github/actions/run-e2e/playwright-selection.mjs b/.github/actions/run-e2e/playwright-selection.mjs new file mode 100644 index 0000000..ab1d4fa --- /dev/null +++ b/.github/actions/run-e2e/playwright-selection.mjs @@ -0,0 +1,57 @@ +import { readFileSync, writeFileSync } from "node:fs"; + +const parseLines = (value) => + value + .split(/\r?\n/) + .map((entry) => entry.trim()) + .filter(Boolean); + +const unique = (values) => [...new Set(values)]; + +const escapeRegex = (value) => + value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + +export const createSelection = (rawProjects, rawLabels, labelMatch) => { + if (!new Set(["all", "any"]).has(labelMatch)) { + throw new Error(`label-match must be all or any; received ${labelMatch}`); + } + + const projects = unique(parseLines(rawProjects)); + const labels = unique(parseLines(rawLabels)); + for (const label of labels) { + if (!label.startsWith("@") || label.length === 1) { + throw new Error(`Every label must begin with @ and contain a name; received ${label}`); + } + } + + const escapedLabels = labels.map(escapeRegex); + const grep = + escapedLabels.length === 0 + ? null + : labelMatch === "all" + ? escapedLabels.map((label) => `(?=.*${label})`).join("") + : `(?:${escapedLabels.join("|")})`; + const args = projects.map((project) => `--project=${project}`); + if (grep) args.push(`--grep=${grep}`); + + return { projects, labels, labelMatch, grep, args }; +}; + +const [command, ...args] = process.argv.slice(2); +if (command === "emit-args") { + const [selectionFile] = args; + const selection = JSON.parse(readFileSync(selectionFile, "utf8")); + for (const argument of selection.args ?? []) { + if (typeof argument !== "string" || argument.includes("\0")) { + throw new Error("Selection file contains an invalid Playwright argument"); + } + process.stdout.write(argument); + process.stdout.write("\0"); + } +} else if (command === "write") { + const [rawProjects, rawLabels, labelMatch, selectionFile] = args; + const selection = createSelection(rawProjects, rawLabels, labelMatch); + writeFileSync(selectionFile, `${JSON.stringify(selection, null, 2)}\n`); +} else if (process.argv[1]?.endsWith("playwright-selection.mjs")) { + throw new Error(`Unknown playwright-selection command: ${command ?? ""}`); +} diff --git a/.github/actions/run-e2e/prepare-shard.mjs b/.github/actions/run-e2e/prepare-shard.mjs index 5aeeb3a..2813ccc 100644 --- a/.github/actions/run-e2e/prepare-shard.mjs +++ b/.github/actions/run-e2e/prepare-shard.mjs @@ -1,5 +1,4 @@ import { - appendFileSync, existsSync, mkdirSync, realpathSync, @@ -7,11 +6,21 @@ import { writeFileSync, } from "node:fs"; import { isAbsolute, relative, resolve, sep } from "node:path"; +import { createSelection } from "./playwright-selection.mjs"; const [ workspace, requestedWorkingDirectory, requestedConfig, + requestedComposeFile, + requestedBaseUrl, + requestedRunnerImageName, + requestedRunnerImageId, + requestedRunnerDownloadOutcome, + requestedRunnerLoadOutcome, + requestedProjects, + requestedLabels, + requestedLabelMatch, requestedShardIndex, requestedShardTotal, requestedResultsDirectory, @@ -39,8 +48,46 @@ const assertRelative = (value, label) => { assertRelative(requestedWorkingDirectory, "working-directory"); assertRelative(requestedConfig, "playwright-config"); +assertRelative(requestedComposeFile, "compose-file"); assertRelative(requestedResultsDirectory, "results-directory"); +let baseUrl; +try { + baseUrl = new URL(requestedBaseUrl); +} catch { + fail(`base-url must be a valid URL: ${requestedBaseUrl}`); +} +if (!["http:", "https:"].includes(baseUrl.protocol)) { + fail("base-url must use HTTP or HTTPS"); +} +if (!requestedRunnerImageName || /[\s\0]/.test(requestedRunnerImageName)) { + fail("runner-image-name must be a non-empty Docker reference without whitespace"); +} +if (!/^sha256:[a-f0-9]{64}$/.test(requestedRunnerImageId)) { + fail(`runner-image-id must be a sha256 image ID; received ${requestedRunnerImageId}`); +} +const allowedStepOutcomes = new Set([ + "success", + "failure", + "cancelled", + "skipped", +]); +for (const [label, outcome] of [ + ["runner-download-outcome", requestedRunnerDownloadOutcome], + ["runner-load-outcome", requestedRunnerLoadOutcome], +]) { + if (!allowedStepOutcomes.has(outcome)) { + fail( + `${label} must be success, failure, cancelled, or skipped; received ${outcome}`, + ); + } +} +const selection = createSelection( + requestedProjects, + requestedLabels, + requestedLabelMatch, +); + const shardIndex = Number(requestedShardIndex); const shardTotal = Number(requestedShardTotal); if (!Number.isSafeInteger(shardIndex) || !Number.isSafeInteger(shardTotal)) { @@ -73,6 +120,16 @@ if (!existsSync(configPath)) { fail(`Playwright configuration does not exist: ${requestedConfig}`); } +const composeFile = resolve(workspacePath, requestedComposeFile); +if (!existsSync(composeFile)) { + fail(`Docker Compose file does not exist: ${requestedComposeFile}`); +} +const resolvedComposeFile = realpathSync(composeFile); +const relativeComposeFile = relative(workspacePath, resolvedComposeFile); +if (relativeComposeFile.startsWith("..") || isAbsolute(relativeComposeFile)) { + fail("compose-file must not escape GITHUB_WORKSPACE"); +} + const resultsRoot = resolve( resolvedWorkingDirectory, requestedResultsDirectory, @@ -87,16 +144,45 @@ rmSync(shardDirectory, { recursive: true, force: true }); mkdirSync(resolve(shardDirectory, "blob-report"), { recursive: true }); mkdirSync(resolve(shardDirectory, "ctrf"), { recursive: true }); mkdirSync(resolve(shardDirectory, "test-results"), { recursive: true }); +const selectionFile = resolve(shardDirectory, "playwright-selection.json"); +writeFileSync(selectionFile, `${JSON.stringify(selection, null, 2)}\n`); const statusFile = resolve(shardDirectory, "shard-status.json"); const status = { - schemaVersion: 1, + schemaVersion: 2, shard: { index: shardIndex, total: shardTotal }, result: "running", exitCode: null, startedAt: new Date().toISOString(), finishedAt: null, durationMs: null, + primaryFailure: null, + secondaryFailures: [], + selection: { + projects: selection.projects, + labels: selection.labels, + labelMatch: selection.labelMatch, + grep: selection.grep, + }, + lifecycle: { + runner: { + imageName: requestedRunnerImageName, + expectedImageId: requestedRunnerImageId, + actualImageId: null, + downloadOutcome: requestedRunnerDownloadOutcome, + loadOutcome: requestedRunnerLoadOutcome, + verificationExitCode: null, + }, + sut: { + composeFile: requestedComposeFile, + baseUrl: baseUrl.toString(), + projectName: null, + startupExitCode: null, + }, + playwright: { exitCode: null, started: false }, + results: { logCaptureExitCode: null }, + cleanup: { teardownExitCode: null }, + }, github: { repository: process.env.GITHUB_REPOSITORY ?? null, sha: process.env.GITHUB_SHA ?? null, @@ -107,10 +193,29 @@ const status = { }; writeFileSync(statusFile, `${JSON.stringify(status, null, 2)}\n`); +const composeProject = [ + "e2e", + process.env.GITHUB_RUN_ID ?? "local", + process.env.GITHUB_RUN_ATTEMPT ?? "1", + shardIndex, +] + .join("-") + .toLowerCase() + .replaceAll(/[^a-z0-9_-]/g, "-"); +status.lifecycle.sut.projectName = composeProject; +writeFileSync(statusFile, `${JSON.stringify(status, null, 2)}\n`); + const quoteForShell = (value) => `'${value.replaceAll("'", `'"'"'`)}'`; const variables = { E2E_ABSOLUTE_WORKING_DIRECTORY: resolvedWorkingDirectory, E2E_ABSOLUTE_CONFIG: configPath, + E2E_CONTAINER_CONFIG: requestedConfig, + E2E_ABSOLUTE_COMPOSE_FILE: resolvedComposeFile, + E2E_BASE_URL: baseUrl.toString(), + E2E_COMPOSE_PROJECT: composeProject, + E2E_RUNNER_IMAGE_NAME: requestedRunnerImageName, + E2E_RUNNER_IMAGE_ID: requestedRunnerImageId, + E2E_SELECTION_FILE: selectionFile, E2E_SHARD_DIRECTORY: shardDirectory, E2E_STATUS_FILE: statusFile, }; diff --git a/.github/actions/validate-e2e-runner/action.yml b/.github/actions/validate-e2e-runner/action.yml new file mode 100644 index 0000000..cc178f9 --- /dev/null +++ b/.github/actions/validate-e2e-runner/action.yml @@ -0,0 +1,47 @@ +name: Validate E2E Runner Archive +description: Validate a cached runner archive and expose its immutable identity. + +inputs: + archive-path: + description: Repository-relative cached Docker archive path. + required: false + default: .e2e-runner/playwright-e2e-runner.tar.zst + image-name: + description: Expected content-addressed local image reference. + required: true + content-hash: + description: Expected SHA-256 identity of the E2E runner inputs. + required: true + +outputs: + image-name: + description: Local Docker image reference stored in the archive. + value: ${{ steps.validate.outputs.image-name }} + image-id: + description: Content-addressed Docker image ID stored in metadata. + value: ${{ steps.validate.outputs.image-id }} + archive-path: + description: Absolute path to the validated archive. + value: ${{ steps.validate.outputs.archive-path }} + archive-sha256: + description: Verified SHA-256 digest of the compressed archive. + value: ${{ steps.validate.outputs.archive-sha256 }} + +runs: + using: composite + steps: + - id: validate + name: Validate cached runner archive + shell: bash + env: + E2E_ARCHIVE_PATH: ${{ inputs.archive-path }} + E2E_RUNNER_IMAGE: ${{ inputs.image-name }} + E2E_CONTENT_HASH: ${{ inputs.content-hash }} + run: | + set -euo pipefail + node "$GITHUB_ACTION_PATH/validate-archive.mjs" \ + "$GITHUB_WORKSPACE" \ + "$E2E_ARCHIVE_PATH" \ + "$E2E_RUNNER_IMAGE" \ + "$E2E_CONTENT_HASH" \ + "$GITHUB_OUTPUT" diff --git a/.github/actions/validate-e2e-runner/validate-archive.mjs b/.github/actions/validate-e2e-runner/validate-archive.mjs new file mode 100644 index 0000000..da7ead8 --- /dev/null +++ b/.github/actions/validate-e2e-runner/validate-archive.mjs @@ -0,0 +1,104 @@ +import { + appendFileSync, + createReadStream, + existsSync, + lstatSync, + readFileSync, + realpathSync, + statSync, +} from "node:fs"; +import { createHash } from "node:crypto"; +import { basename, dirname, isAbsolute, relative, resolve, sep } from "node:path"; + +const [ + workspace, + requestedArchivePath, + expectedImageName, + expectedContentHash, + outputFile, +] = process.argv.slice(2); + +const fail = (message) => { + throw new Error(message); +}; + +if (!/^[a-f0-9]{64}$/.test(expectedContentHash)) { + fail("content-hash must be a lowercase SHA-256 digest"); +} +if (!expectedImageName || /[\s\0]/.test(expectedImageName)) { + fail("image-name must be a non-empty Docker reference without whitespace"); +} +if (!requestedArchivePath || isAbsolute(requestedArchivePath)) { + fail("archive-path must be repository-relative"); +} + +const workspacePath = realpathSync(workspace); +const archivePath = resolve(workspacePath, requestedArchivePath); +if (basename(archivePath) !== "playwright-e2e-runner.tar.zst") { + fail("archive-path must end with playwright-e2e-runner.tar.zst"); +} +const relativeArchive = relative(workspacePath, archivePath); +if ( + relativeArchive === "" || + relativeArchive === ".." || + relativeArchive.startsWith(`..${sep}`) || + isAbsolute(relativeArchive) +) { + fail("archive-path must remain inside GITHUB_WORKSPACE"); +} +if (!existsSync(archivePath) || !lstatSync(archivePath).isFile()) { + fail(`runner archive does not exist: ${requestedArchivePath}`); +} +if (lstatSync(archivePath).isSymbolicLink()) { + fail("runner archive must not be a symbolic link"); +} + +const archiveDirectory = realpathSync(dirname(archivePath)); +const relativeArchiveDirectory = relative(workspacePath, archiveDirectory); +if ( + relativeArchiveDirectory === ".." || + relativeArchiveDirectory.startsWith(`..${sep}`) || + isAbsolute(relativeArchiveDirectory) +) { + fail("runner archive directory must remain inside GITHUB_WORKSPACE"); +} +const metadataPath = resolve(archiveDirectory, "metadata.json"); +if (!existsSync(metadataPath) || !lstatSync(metadataPath).isFile()) { + fail("cached runner metadata.json is missing"); +} +if (lstatSync(metadataPath).isSymbolicLink()) { + fail("cached runner metadata must not be a symbolic link"); +} + +const digestFile = async (path) => { + const hash = createHash("sha256"); + const stream = createReadStream(path); + for await (const chunk of stream) hash.update(chunk); + return hash.digest("hex"); +}; + +let metadata; +try { + metadata = JSON.parse(readFileSync(metadataPath, "utf8")); +} catch (error) { + fail(`cached runner metadata is invalid JSON: ${error.message}`); +} + +const archiveDigest = await digestFile(archivePath); +const archiveSize = statSync(archivePath).size; +if (metadata.schemaVersion !== 1) fail("unsupported runner metadata schema"); +if (metadata.contentHash !== expectedContentHash) fail("cached runner content hash does not match"); +if (metadata.imageName !== expectedImageName) fail("cached runner image name does not match"); +if (!/^sha256:[a-f0-9]{64}$/.test(metadata.imageId ?? "")) fail("cached runner image ID is invalid"); +if (metadata.archive?.file !== "playwright-e2e-runner.tar.zst") fail("cached runner archive name is invalid"); +if (metadata.archive?.sha256 !== archiveDigest) fail("cached runner archive digest does not match"); +if (metadata.archive?.size !== archiveSize) fail("cached runner archive size does not match"); + +for (const [name, value] of [ + ["image-name", metadata.imageName], + ["image-id", metadata.imageId], + ["archive-path", archivePath], + ["archive-sha256", archiveDigest], +]) { + appendFileSync(outputFile, `${name}=${value}\n`); +} diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 685453f..496d6f3 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -6,9 +6,35 @@ on: branches: - main workflow_dispatch: + inputs: + profile: + description: Execution profile from e2e/ci.yml. + required: true + default: regression + type: string + projects: + description: Optional project override, one per line. + required: false + default: "" + type: string + labels: + description: Optional tag override, one per line. + required: false + default: "" + type: string + label-match: + description: Optional tag matching override. + required: false + default: profile + type: choice + options: + - profile + - all + - any permissions: contents: read + pull-requests: write concurrency: group: e2e-${{ github.event.pull_request.number || github.ref }} @@ -16,8 +42,105 @@ concurrency: jobs: e2e: - name: E2E pipeline - uses: $/.github/workflows/reusable-e2e.yml - with: - working-directory: e2e - shard-count: 2 + name: E2E Gate + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - id: e2e + name: Run E2E + continue-on-error: true + uses: $/ + with: + profile: >- + ${{ github.event_name == 'pull_request' && 'pull-request' || + github.event_name == 'push' && 'main' || inputs.profile }} + projects: ${{ github.event_name == 'workflow_dispatch' && inputs.projects || '' }} + labels: ${{ github.event_name == 'workflow_dispatch' && inputs.labels || '' }} + label-match: >- + ${{ github.event_name == 'workflow_dispatch' && inputs.label-match != 'profile' && + inputs.label-match || '' }} + + - name: Update pull request + if: always() && github.event_name == 'pull_request' + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + E2E_RESULT: ${{ steps.e2e.outputs.result }} + E2E_PRIMARY_FAILURE: ${{ steps.e2e.outputs.primary-failure }} + E2E_TOTAL: ${{ steps.e2e.outputs.total }} + E2E_PASSED: ${{ steps.e2e.outputs.passed }} + E2E_FAILED: ${{ steps.e2e.outputs.failed }} + E2E_SKIPPED: ${{ steps.e2e.outputs.skipped }} + E2E_PROFILE: ${{ steps.e2e.outputs.profile }} + E2E_PROJECTS: ${{ steps.e2e.outputs.projects }} + E2E_LABELS: ${{ steps.e2e.outputs.labels }} + E2E_LABEL_MATCH: ${{ steps.e2e.outputs.label-match }} + E2E_RUNNER_CACHE_HIT: ${{ steps.e2e.outputs.runner-cache-hit }} + E2E_RUNNER_IMAGE: ${{ steps.e2e.outputs.runner-image-name }} + E2E_RESULTS_ARTIFACT: ${{ steps.e2e.outputs.results-artifact }} + E2E_REPORT_IMAGE: ${{ steps.e2e.outputs.report-image-name }} + E2E_REPORT_ARTIFACT: ${{ steps.e2e.outputs.report-image-artifact }} + with: + script: | + const marker = ''; + const value = (name, fallback = 'n/a') => process.env[name] || fallback; + const list = (name) => value(name).split('\n').filter(Boolean).join(', '); + const result = value('E2E_RESULT', 'infrastructure-error'); + const icon = result === 'passed' ? '✅' : result === 'failed' ? '❌' : '⚠️'; + const cache = value('E2E_RUNNER_CACHE_HIT', 'false') === 'true' ? 'reused' : 'built'; + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const rows = [ + ['Gate', `${icon} ${result}`], + ['Tests', `${value('E2E_PASSED', '0')} passed · ${value('E2E_FAILED', '0')} failed · ${value('E2E_SKIPPED', '0')} skipped · ${value('E2E_TOTAL', '0')} total`], + ['Profile', value('E2E_PROFILE')], + ['Projects', list('E2E_PROJECTS')], + ['Tags', `${list('E2E_LABELS')} (${value('E2E_LABEL_MATCH')})`], + ['Runner', `${value('E2E_RUNNER_IMAGE')} (${cache})`], + ['Results', value('E2E_RESULTS_ARTIFACT')], + ['Report image', `${value('E2E_REPORT_IMAGE')} · ${value('E2E_REPORT_ARTIFACT')}`], + ['Run', `[Open workflow run](${runUrl})`], + ]; + const failure = process.env.E2E_PRIMARY_FAILURE + ? `\n> ${process.env.E2E_PRIMARY_FAILURE}\n` + : ''; + const body = [ + marker, + '## E2E test report', + '', + '| | |', + '|---|---|', + ...rows.map(([name, detail]) => `| ${name} | ${detail} |`), + failure, + ].join('\n'); + const comments = await github.paginate(github.rest.issues.listComments, { + ...context.repo, + issue_number: context.issue.number, + per_page: 100, + }); + const previous = comments.find((comment) => + comment.user?.type === 'Bot' && comment.body?.includes(marker)); + if (previous) { + await github.rest.issues.updateComment({ + ...context.repo, + comment_id: previous.id, + body, + }); + } else { + await github.rest.issues.createComment({ + ...context.repo, + issue_number: context.issue.number, + body, + }); + } + + - name: Preserve E2E result + if: always() + shell: bash + env: + E2E_STEP_OUTCOME: ${{ steps.e2e.outcome }} + run: | + if [[ "$E2E_STEP_OUTCOME" != "success" ]]; then + exit 1 + fi diff --git a/.github/workflows/reusable-e2e.yml b/.github/workflows/reusable-e2e.yml deleted file mode 100644 index 1f0a9e4..0000000 --- a/.github/workflows/reusable-e2e.yml +++ /dev/null @@ -1,216 +0,0 @@ -name: Reusable E2E - -on: - workflow_call: - inputs: - working-directory: - description: Directory containing the Playwright package. - required: false - default: e2e - type: string - node-version: - description: Node.js version used by the E2E package. - required: false - default: "22" - type: string - pnpm-version: - description: pnpm version used to install the E2E package. - required: false - default: 10.26.2 - type: string - playwright-config: - description: Playwright configuration path, relative to the working directory. - required: false - default: playwright.config.ts - type: string - shard-count: - description: Number of Playwright shards to run in parallel (1-32). - required: false - default: 2 - type: number - artifact-retention-days: - description: Number of days to retain shard and merged evidence artifacts. - required: false - default: 10 - type: number - outputs: - verdict: - description: Authoritative E2E verdict. - value: ${{ jobs.e2e_gate.outputs.verdict }} - artifact-name: - description: Name of the merged evidence artifact. - value: ${{ jobs.merge.outputs.artifact-name }} - total: - description: Total number of tests in the merged report. - value: ${{ jobs.merge.outputs.total }} - passed: - description: Number of passing tests in the merged report. - value: ${{ jobs.merge.outputs.passed }} - failed: - description: Number of failing tests in the merged report. - value: ${{ jobs.merge.outputs.failed }} - skipped: - description: Number of skipped tests in the merged report. - value: ${{ jobs.merge.outputs.skipped }} - -permissions: - contents: read - -jobs: - plan: - name: Plan E2E shards - runs-on: ubuntu-latest - timeout-minutes: 2 - outputs: - matrix: ${{ steps.matrix.outputs.matrix }} - steps: - - id: matrix - name: Validate and create shard matrix - env: - SHARD_COUNT: ${{ inputs.shard-count }} - shell: bash - run: | - matrix="$(node -e ' - const count = Number(process.env.SHARD_COUNT); - if (!Number.isInteger(count) || count < 1 || count > 32) { - console.error("shard-count must be an integer from 1 through 32"); - process.exit(1); - } - process.stdout.write(JSON.stringify({ shard: Array.from({ length: count }, (_, index) => index + 1) })); - ')" - echo "matrix=$matrix" >> "$GITHUB_OUTPUT" - - shards: - name: E2E shard ${{ matrix.shard }}/${{ inputs.shard-count }} - needs: plan - runs-on: ubuntu-latest - timeout-minutes: 20 - strategy: - fail-fast: false - matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} - steps: - - name: Check out caller repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - - name: Prepare E2E environment - uses: $/.github/actions/prepare-e2e - with: - working-directory: ${{ inputs.working-directory }} - node-version: ${{ inputs.node-version }} - pnpm-version: ${{ inputs.pnpm-version }} - playwright-browser: chromium - - - id: run - name: Run Playwright shard - continue-on-error: true - uses: $/.github/actions/run-e2e - with: - working-directory: ${{ inputs.working-directory }} - playwright-config: ${{ inputs.playwright-config }} - shard-index: ${{ matrix.shard }} - shard-total: ${{ inputs.shard-count }} - results-directory: test-results - - - name: Upload shard evidence - if: always() && steps.run.outcome != 'skipped' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: e2e-shard-${{ matrix.shard }} - path: ${{ steps.run.outputs.shard-directory }} - if-no-files-found: error - retention-days: ${{ inputs.artifact-retention-days }} - - - name: Preserve shard verdict - if: always() && steps.run.outcome != 'success' - shell: bash - run: | - echo "::error::E2E shard ${{ matrix.shard }} did not complete successfully" - exit 1 - - merge: - name: Merge E2E evidence - needs: - - plan - - shards - if: always() - runs-on: ubuntu-latest - timeout-minutes: 10 - outputs: - verdict: ${{ steps.merge.outputs.verdict }} - artifact-name: ${{ steps.merge.outputs.artifact-name }} - total: ${{ steps.merge.outputs.total }} - passed: ${{ steps.merge.outputs.passed }} - failed: ${{ steps.merge.outputs.failed }} - skipped: ${{ steps.merge.outputs.skipped }} - steps: - - name: Check out caller repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - - name: Prepare report tooling - uses: $/.github/actions/prepare-e2e - with: - working-directory: ${{ inputs.working-directory }} - node-version: ${{ inputs.node-version }} - pnpm-version: ${{ inputs.pnpm-version }} - playwright-browser: chromium - install-browser: "false" - - - id: download - name: Download shard evidence - if: always() - continue-on-error: true - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - pattern: e2e-shard-* - path: downloaded-shards - merge-multiple: false - - - id: merge - name: Merge and publish evidence - if: always() - continue-on-error: true - uses: $/.github/actions/merge-e2e-evidence - with: - working-directory: ${{ inputs.working-directory }} - download-directory: downloaded-shards - evidence-directory: e2e-evidence - expected-shards: ${{ inputs.shard-count }} - artifact-name: e2e-evidence - retention-days: ${{ inputs.artifact-retention-days }} - - - name: Preserve merge verdict - if: always() && (steps.download.outcome != 'success' || steps.merge.outcome != 'success') - shell: bash - run: | - echo "::error::E2E evidence could not be completed successfully" - exit 1 - - e2e_gate: - name: E2E Gate - needs: - - plan - - shards - - merge - if: always() - runs-on: ubuntu-latest - timeout-minutes: 5 - outputs: - verdict: ${{ steps.finalize.outputs.verdict }} - primary-failure: ${{ steps.finalize.outputs.primary-failure }} - steps: - - id: download - name: Download authoritative evidence - continue-on-error: true - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - name: e2e-evidence - path: final-evidence - - - id: finalize - name: Enforce authoritative verdict - uses: $/.github/actions/finalize-e2e-verdict - with: - verdict-file: final-evidence/verdict.json - shard-job-result: ${{ needs.shards.result }} - merge-job-result: ${{ needs.merge.result }} - publication-result: ${{ steps.download.outcome }} diff --git a/README.md b/README.md index 77b6557..d154c7f 100644 --- a/README.md +++ b/README.md @@ -1,109 +1,206 @@ -# Playwright E2E CI Framework +# Playwright E2E -A small, reusable GitHub Actions pipeline for running Playwright tests with reliable evidence and one fail-closed gate. +A portable GitHub Action and lightweight framework for running Playwright against a Dockerized system under test (SUT). -The repository includes a real example: Playwright starts a Node.js echo server, waits for its health endpoint, and tests it through API and Chromium clients. +It keeps the application repository in control of its tests and services while providing a consistent CI engine: deterministic runner images, smart reuse, SUT lifecycle management, test filtering, Playwright reports, traces, portable artifacts, and a fail-closed result. ```text pull request / main / manual | - v - plan two shards - / \ - v v - shard 1 shard 2 - \ / - v v - merge evidence + validate e2e/ci.yml + | + hash E2E runner inputs + | + +------+------+ + | | + validated cache cache miss + | build runner + +------+------+ + | + start Dockerized SUT + | + Playwright projects + tags + | + results + traces + | + HTML report + report image | - v E2E Gate ``` -## Run the example locally +## Start from the template -Prerequisites: Node.js 22 and Corepack. +The quickest path is [codotech/playwright-e2e-starter](https://github.com/codotech/playwright-e2e-starter). Create a repository from that template, replace its example SUT and tests, then adapt the profiles in `e2e/ci.yml`. -```bash -corepack enable -corepack prepare pnpm@10.26.2 --activate -pnpm --dir e2e install --frozen-lockfile -pnpm --dir e2e install:browsers -pnpm --dir e2e test -``` +The template owns repository policy: triggers, permissions, concurrency, checkout, the sticky pull-request comment, and preservation of the action result. This repository owns the portable execution engine. -The Playwright configuration owns the server lifecycle, so a separate server process is not required. +## Use the action -Use the CI reporter set locally when you need to inspect the exact CI artifacts: +Check out the caller repository first, then invoke the root action. Pin it to a full commit SHA: -```bash -pnpm --dir e2e test:ci +```yaml +permissions: + contents: read + pull-requests: write + +steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + + - id: e2e + uses: codotech/playwright-e2e@<40-character-commit-sha> + with: + profile: pull-request ``` -Run `pnpm --dir e2e test:static` to type-check the configuration and suite without starting the server. +There is intentionally no release tag yet. Adopt an immutable commit only after reviewing it; the starter contains a working pinned example. + +### Inputs + +| Input | Default | Purpose | +| --- | --- | --- | +| `profile` | `pull-request` | Named execution profile from `e2e/ci.yml` | +| `projects` | empty | Optional Playwright project override, one per line | +| `labels` | empty | Optional Playwright tag override, one per line | +| `label-match` | empty | Optional `all` or `any` tag matching override | + +Projects and tags are independent filters. Projects select configured Playwright variants. Tags select tests through Playwright's `--grep`. The action runs the intersection. + +### Outputs + +| Output | Meaning | +| --- | --- | +| `result` | `passed`, `failed`, or `infrastructure-error` | +| `primary-failure` | Main failure message, empty on success | +| `total`, `passed`, `failed`, `skipped` | Aggregate test counts | +| `profile`, `projects`, `labels`, `label-match` | Effective selection | +| `runner-cache-hit` | Whether a validated runner archive was reused | +| `runner-content-hash` | SHA-256 identity of runner inputs | +| `runner-image-name`, `runner-image-id`, `runner-image-artifact` | Exact portable runner image | +| `results-directory`, `results-artifact` | Merged machine-readable and Playwright results | +| `report-directory` | Generated Playwright HTML report | +| `report-image-name`, `report-image-id`, `report-image-artifact` | Portable image serving the HTML report | + +The action fails at the end when tests fail or infrastructure is incomplete. A caller that needs to publish a pull-request comment first can set `continue-on-error: true`, publish the comment from the outputs, and add a final step that fails when the action step outcome is not `success`. The starter demonstrates this pattern. + +## Repository contract + +The caller keeps these files: -## Use the reusable workflow +```text +. ++-- compose.e2e.yml ++-- e2e/ +| +-- ci.yml +| +-- Dockerfile +| +-- runner-entrypoint.mjs +| +-- package.json +| +-- pnpm-lock.yaml +| +-- playwright.config.ts +| +-- tests/ ++-- sut/ # example only; use your real services +``` -Consumer repositories keep their Playwright package in a root-level `e2e/` directory. Call the workflow from a job and pin it to the full commit SHA of a released version: +`e2e/ci.yml` is the CI contract: ```yaml -name: E2E +version: 1 -on: - pull_request: +runner: + dockerfile: Dockerfile -permissions: - contents: read +playwright: + config: playwright.config.ts -jobs: - e2e: - uses: codotech/playwright-e2e-ci-framework/.github/workflows/reusable-e2e.yml@<40-character-release-commit-sha> - with: - working-directory: e2e - shard-count: 2 +sut: + composeFile: compose.e2e.yml + baseUrl: http://127.0.0.1:4173 + +execution: + shards: 1 + artifactRetentionDays: 10 + +profiles: + pull-request: + projects: [api, chromium] + labels: ["@smoke"] + labelMatch: any + + main: + projects: [api, chromium] + labels: [] + labelMatch: all ``` -Replace `<40-character-release-commit-sha>` with the immutable SHA published for the version you adopt. The workflow needs no inherited secrets and does not write pull-request comments. +The root action is deliberately a single GitHub job. Playwright can still use multiple workers inside the runner container. Keep `execution.shards: 1`; repository-level job matrices can be added later by the caller without changing the portable engine. -The framework targets GitHub.com. It uses GitHub's `$/` self-repository references so the reusable workflow and its composite actions always come from the same commit; that syntax is not available on GitHub Enterprise Server. +`playwright.config.ts` defines projects, browser/device settings, matching, dependencies, reporters, and runtime behavior. `e2e/ci.yml` selects what CI runs. Unknown keys, invalid paths, unsafe values, nonexistent projects, or malformed tags fail before the SUT starts. -The supported inputs are: +## What rebuilds the runner -| Input | Default | Purpose | -| ------------------------- | ---------------------- | ---------------------------------- | -| `working-directory` | `e2e` | Playwright package location | -| `node-version` | `22` | Node.js runtime | -| `pnpm-version` | `10.26.2` | pnpm runtime | -| `playwright-config` | `playwright.config.ts` | Config path inside the package | -| `shard-count` | `2` | Parallel shards, from 1 through 32 | -| `artifact-retention-days` | `10` | Evidence retention period | +The runner identity covers every Git-tracked or non-ignored untracked entry below `e2e/`, except `e2e/ci.yml`. Paths, file types, Unix modes, file bytes, and symlink targets all contribute. -The workflow exposes `verdict`, `artifact-name`, `total`, `passed`, `failed`, and `skipped` outputs. +A new runner is built when tests, fixtures, dependencies, the lockfile, Playwright configuration, runner entrypoint, Dockerfile, or Docker-ignore rules change. Profiles, filters, retention, and service-only changes reuse the same runner bytes while still running the suite against the newly built SUT. -## Read the evidence +On an exact cache hit, the action validates the archive metadata, image reference, content hash, byte size, SHA-256, and image ID before using it. A missing, evicted, or invalid cache causes a clean rebuild. -Every shard uploads its raw status, blob report, CTRF result, and Playwright test results. The merge job verifies that all expected shards are present and publishes one `e2e-evidence` artifact: +## Test lifecycle -```text -e2e-evidence/ -+-- playwright-report/ -+-- ctrf-report.json -+-- shard-status/ -+-- verdict.json +The action performs one fail-closed sequence: + +1. Validate configuration and resolve the requested profile. +2. Reuse or build the content-addressed Playwright runner. +3. Build and start the SUT with Docker Compose and wait for health checks. +4. Run the selected Playwright projects and tags. +5. Capture results, traces, screenshots, SUT logs, and teardown logs. +6. Tear down containers, volumes, and networks even after failure. +7. Build the HTML report and portable report image. +8. Upload artifacts and enforce the final result. + +A Playwright failure remains the primary failure even if later log collection or cleanup also fails. Missing results, runner identity problems, unhealthy services, or publication failures produce `infrastructure-error`; they never turn a run green. + +## Artifacts + +| Artifact | Contents | +| --- | --- | +| `e2e-results` | Final JSON result, CTRF report, Playwright HTML report, test attachments, SUT logs, and teardown logs | +| `e2e-runner-image` | Compressed Docker archive for the exact runner used by the run | +| `e2e-report-image` | Compressed Docker archive serving the HTML report on port 8080 | + +Download and load a report image: + +```bash +docker load --input e2e-report-image/playwright-e2e-report.tar.zst +docker run --rm -p 8080:8080 "$REPORT_IMAGE_NAME" +``` + +Open `http://127.0.0.1:8080`. Traces and other Playwright attachments are retained inside the report when a test creates them. + +The images are workflow artifacts, not registry publications. Artifact retention applies and local image names are not remotely pullable. + +## Run this example locally + +Prerequisites: Docker, Node.js 22, Corepack, and pnpm 10.26.2. + +```bash +corepack enable +corepack prepare pnpm@10.26.2 --activate +pnpm --dir e2e install --frozen-lockfile +pnpm --dir e2e install:browsers +docker compose -f compose.e2e.yml up --detach --build --wait +BASE_URL=http://127.0.0.1:4173 pnpm --dir e2e test +docker compose -f compose.e2e.yml down --volumes --remove-orphans ``` -The HTML report is for investigation. `verdict.json` is the machine-readable result. The Actions job summary shows the merged totals without requiring repository write permissions. +Always run the final cleanup command, including after a failed test. Run `pnpm --dir e2e test:static` for the TypeScript check. -## Fail-closed behavior +## Platform support -`E2E Gate` fails when any of these conditions occurs: +- GitHub.com hosted or self-hosted Linux runners with Docker and Docker Compose +- GitHub Actions runner 2.336.0 or newer, required for repository-relative `$/` action references +- One checked-out application repository per job -- A Playwright shard fails or is cancelled. -- A shard status or report is missing or duplicated. -- Reports cannot be merged. -- The evidence artifact cannot be published or downloaded. -- The authoritative verdict is absent or does not say `passed`. +No inherited secrets are required by the action. The SUT may use repository or environment secrets supplied by its own workflow. -Evidence publication runs after test failure, but publishing a report never turns a failed test run green. +## License -To verify these guarantees on a branch, intentionally break one echo assertion, interrupt the example server startup, or temporarily remove one shard upload. Each experiment must end with a red `E2E Gate` and retained evidence; revert the controlled change after verification. +[Apache-2.0](LICENSE) diff --git a/action.yml b/action.yml new file mode 100644 index 0000000..73deb73 --- /dev/null +++ b/action.yml @@ -0,0 +1,246 @@ +name: Playwright E2E +author: Codotech +description: Run a Dockerized system under test with a portable Playwright runner and publish the test results. + +branding: + icon: check-circle + color: green + +inputs: + profile: + description: Named execution profile from e2e/ci.yml. + required: false + default: pull-request + projects: + description: Optional newline-delimited Playwright project override. + required: false + default: "" + labels: + description: Optional newline-delimited Playwright tag override. + required: false + default: "" + label-match: + description: Optional label matching override, either all or any. + required: false + default: "" + +outputs: + result: + description: "Final result: passed, failed, or infrastructure-error." + value: ${{ steps.finalize.outputs.result }} + primary-failure: + description: Primary failure message, empty when the run passed. + value: ${{ steps.finalize.outputs.primary-failure }} + total: + description: Total number of tests. + value: ${{ steps.publish.outputs.total }} + passed: + description: Number of passing tests. + value: ${{ steps.publish.outputs.passed }} + failed: + description: Number of failing tests. + value: ${{ steps.publish.outputs.failed }} + skipped: + description: Number of skipped tests. + value: ${{ steps.publish.outputs.skipped }} + profile: + description: Effective execution profile. + value: ${{ steps.plan.outputs.profile }} + projects: + description: Effective newline-delimited Playwright projects. + value: ${{ steps.plan.outputs.projects }} + labels: + description: Effective newline-delimited Playwright tags. + value: ${{ steps.plan.outputs.labels }} + label-match: + description: Effective label matching mode. + value: ${{ steps.plan.outputs.label-match }} + runner-cache-hit: + description: Whether a validated runner archive was reused. + value: ${{ steps.reuse.outcome == 'success' }} + runner-content-hash: + description: SHA-256 identity of the runner inputs. + value: ${{ steps.plan.outputs.runner-content-hash }} + runner-image-name: + description: Content-hash-tagged runner image name. + value: ${{ steps.reuse.outputs.image-name || steps.build.outputs.image-name }} + runner-image-id: + description: Content-addressed runner image ID. + value: ${{ steps.reuse.outputs.image-id || steps.build.outputs.image-id }} + runner-image-artifact: + description: Artifact containing the portable runner image. + value: ${{ steps.upload-runner.outcome == 'success' && 'e2e-runner-image' || '' }} + results-directory: + description: Absolute directory containing merged test results. + value: ${{ steps.merge.outputs.results-directory }} + results-artifact: + description: Artifact containing merged test results. + value: ${{ steps.publish.outputs.artifact-name }} + report-directory: + description: Absolute directory containing the Playwright HTML report. + value: ${{ steps.merge.outputs.report-directory }} + report-image-name: + description: Run-tagged report image name. + value: ${{ steps.package-report.outputs.image-name }} + report-image-id: + description: Content-addressed report image ID. + value: ${{ steps.package-report.outputs.image-id }} + report-image-artifact: + description: Artifact containing the portable report image. + value: ${{ steps.package-report.outputs.artifact-name }} + +runs: + using: composite + steps: + - id: plan + name: Validate E2E configuration + uses: $/.github/actions/plan-e2e + with: + profile: ${{ inputs.profile }} + projects: ${{ inputs.projects }} + labels: ${{ inputs.labels }} + label-match: ${{ inputs.label-match }} + + - name: Require single-job execution + if: steps.plan.outputs.shard-count != '1' + shell: bash + run: | + echo "execution.shards must be 1 when using the root action" >&2 + exit 1 + + - id: runner-cache + name: Restore E2E runner archive + continue-on-error: true + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: .e2e-runner + key: e2e-runner-v1-${{ runner.os }}-${{ runner.arch }}-${{ steps.plan.outputs.runner-content-hash }} + + - id: reuse + name: Validate cached E2E runner image + if: steps.runner-cache.outcome == 'success' && steps.runner-cache.outputs.cache-hit == 'true' + continue-on-error: true + uses: $/.github/actions/validate-e2e-runner + with: + archive-path: .e2e-runner/playwright-e2e-runner.tar.zst + image-name: playwright-e2e-runner:e2e-${{ steps.plan.outputs.runner-content-hash }} + content-hash: ${{ steps.plan.outputs.runner-content-hash }} + + - id: build + name: Build E2E runner image + if: >- + always() && !cancelled() && steps.runner-cache.outcome != 'skipped' && + steps.reuse.outcome != 'success' + uses: $/.github/actions/build-e2e-runner + with: + working-directory: ${{ steps.plan.outputs.working-directory }} + dockerfile: ${{ steps.plan.outputs.dockerfile }} + image-name: playwright-e2e-runner:e2e-${{ steps.plan.outputs.runner-content-hash }} + archive-path: .e2e-runner/playwright-e2e-runner.tar.zst + content-hash: ${{ steps.plan.outputs.runner-content-hash }} + + - id: upload-runner + name: Upload portable E2E runner image + if: always() && (steps.reuse.outcome == 'success' || steps.build.outcome == 'success') + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: e2e-runner-image + path: ${{ steps.reuse.outputs.archive-path || steps.build.outputs.archive-path }} + if-no-files-found: error + compression-level: 0 + retention-days: ${{ steps.plan.outputs.artifact-retention-days }} + + - id: load-runner + name: Load E2E runner image + if: always() && (steps.reuse.outcome == 'success' || steps.build.outcome == 'success') + continue-on-error: true + shell: bash + run: docker load --input "${{ steps.reuse.outputs.archive-path || steps.build.outputs.archive-path }}" + + - id: verify-runner + name: Verify immutable E2E runner image + if: always() && steps.load-runner.outcome == 'success' + continue-on-error: true + shell: bash + env: + E2E_RUNNER_IMAGE_NAME: ${{ steps.reuse.outputs.image-name || steps.build.outputs.image-name }} + E2E_RUNNER_IMAGE_ID: ${{ steps.reuse.outputs.image-id || steps.build.outputs.image-id }} + run: | + set -euo pipefail + actual_id="$(docker image inspect --format '{{.Id}}' "$E2E_RUNNER_IMAGE_NAME")" + if [[ "$actual_id" != "$E2E_RUNNER_IMAGE_ID" ]]; then + echo "Loaded runner image ID $actual_id does not match $E2E_RUNNER_IMAGE_ID" >&2 + exit 1 + fi + + - id: run + name: Run Playwright E2E + if: always() && steps.plan.outcome == 'success' + continue-on-error: true + uses: $/.github/actions/run-e2e + with: + working-directory: ${{ steps.plan.outputs.working-directory }} + playwright-config: ${{ steps.plan.outputs.playwright-config }} + compose-file: ${{ steps.plan.outputs.compose-file }} + base-url: ${{ steps.plan.outputs.base-url }} + runner-image-name: ${{ steps.reuse.outputs.image-name || steps.build.outputs.image-name }} + runner-image-id: ${{ steps.reuse.outputs.image-id || steps.build.outputs.image-id }} + runner-download-outcome: ${{ (steps.reuse.outcome == 'success' || steps.build.outcome == 'success') && 'success' || 'failure' }} + runner-load-outcome: ${{ steps.load-runner.outcome }} + projects: ${{ steps.plan.outputs.projects }} + labels: ${{ steps.plan.outputs.labels }} + label-match: ${{ steps.plan.outputs.label-match }} + shard-index: "1" + shard-total: "1" + results-directory: test-results + + - id: merge + name: Merge E2E results + if: always() && steps.plan.outcome == 'success' + continue-on-error: true + uses: $/.github/actions/merge-e2e-results + with: + download-directory: ${{ steps.plan.outputs.working-directory }}/test-results + results-directory: e2e-results + expected-shards: "1" + runner-image-id: ${{ steps.reuse.outputs.image-id || steps.build.outputs.image-id }} + + - id: package-report + name: Package E2E report image + if: always() && steps.merge.outcome == 'success' + continue-on-error: true + uses: $/.github/actions/package-e2e-report + with: + report-directory: e2e-results/playwright-report + image-name: playwright-e2e-report:${{ github.run_id }}-${{ github.run_attempt }} + archive-directory: e2e-report-image + artifact-name: e2e-report-image + retention-days: ${{ steps.plan.outputs.artifact-retention-days }} + + - id: publish + name: Publish E2E results + if: always() && steps.plan.outcome == 'success' + continue-on-error: true + uses: $/.github/actions/publish-e2e-results + with: + results-directory: e2e-results + expected-shards: "1" + artifact-name: e2e-results + retention-days: ${{ steps.plan.outputs.artifact-retention-days }} + runner-download-outcome: ${{ (steps.reuse.outcome == 'success' || steps.build.outcome == 'success') && 'success' || 'failure' }} + runner-load-outcome: ${{ steps.load-runner.outcome }} + runner-verification-outcome: ${{ steps.verify-runner.outcome }} + shard-download-outcome: ${{ steps.run.outputs.shard-directory != '' && 'success' || 'failure' }} + merge-outcome: ${{ steps.merge.outcome }} + report-publication-outcome: ${{ steps.package-report.outcome }} + + - id: finalize + name: Enforce final E2E result + if: always() + uses: $/.github/actions/finalize-e2e-result + with: + result-file: ${{ steps.publish.outputs.result-file || 'e2e-results/result.json' }} + shard-job-result: ${{ steps.run.outcome }} + merge-job-result: ${{ (steps.merge.outcome == 'success' && steps.publish.outcome == 'success') && 'success' || 'failure' }} + publication-result: ${{ steps.upload-runner.outcome }} diff --git a/compose.e2e.yml b/compose.e2e.yml new file mode 100644 index 0000000..cddb015 --- /dev/null +++ b/compose.e2e.yml @@ -0,0 +1,15 @@ +services: + sut: + build: + context: ./sut + dockerfile: Dockerfile + init: true + ports: + - "4173:4173" + read_only: true + tmpfs: + - /tmp + cap_drop: + - ALL + security_opt: + - no-new-privileges:true diff --git a/e2e/.dockerignore b/e2e/.dockerignore new file mode 100644 index 0000000..987a4b7 --- /dev/null +++ b/e2e/.dockerignore @@ -0,0 +1,82 @@ +# Dependency and package-manager state +node_modules/ +**/node_modules/ +.pnpm-store/ +**/.pnpm-store/ +.npm/ +**/.npm/ +.npmrc +**/.npmrc + +# Generated test results and reports +blob-report/ +**/blob-report/ +ctrf/ +**/ctrf/ +playwright-report/ +**/playwright-report/ +test-results/ +**/test-results/ +e2e-results/ +**/e2e-results/ +downloaded-shards/ +**/downloaded-shards/ +coverage/ +**/coverage/ + +# Build caches and portable image archives +.cache/ +**/.cache/ +.turbo/ +**/.turbo/ +.eslintcache +**/.eslintcache +*.tsbuildinfo +.e2e-runner/ +e2e-report-image/ +runner-image/ +*.tar +*.tar.gz +*.tar.zst +*.tgz +*.zst + +# Version-control and editor metadata +.git/ +**/.git/ +.gitignore +**/.gitignore +.gitattributes +**/.gitattributes +.hg/ +**/.hg/ +.svn/ +**/.svn/ +.idea/ +**/.idea/ +.vscode/ +**/.vscode/ +.DS_Store +**/.DS_Store + +# Local secrets and private keys +.env +**/.env +.env.* +**/.env.* +!.env.example +!**/.env.example +*.pem +*.key +*.p12 +*.pfx +*.jks +id_rsa +id_rsa.* +id_ed25519 +id_ed25519.* + +# CI and build instructions are not runtime inputs +ci.yml +Dockerfile +.dockerignore diff --git a/e2e/Dockerfile b/e2e/Dockerfile new file mode 100644 index 0000000..91e244b --- /dev/null +++ b/e2e/Dockerfile @@ -0,0 +1,21 @@ +FROM mcr.microsoft.com/playwright:v1.61.1-noble@sha256:5b8f294aff9041b7191c34a4bab3ac270157a28774d4b0660e9743297b697e48 + +ENV CI=true \ + E2E_RESULTS_DIR=/test-output/test-results \ + PLAYWRIGHT_BLOB_OUTPUT_DIR=/test-output/blob-report \ + CTRF_OUTPUT_FILE=/test-output/ctrf/ctrf-report.json + +RUN corepack enable && corepack prepare pnpm@10.26.2 --activate + +WORKDIR /workspace/e2e + +COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./ +RUN pnpm install --frozen-lockfile + +COPY --chown=pwuser:pwuser . . + +RUN mkdir -p /test-output && chown pwuser:pwuser /test-output + +USER pwuser + +ENTRYPOINT ["node", "./runner-entrypoint.mjs"] diff --git a/e2e/ci.yml b/e2e/ci.yml new file mode 100644 index 0000000..b9ade55 --- /dev/null +++ b/e2e/ci.yml @@ -0,0 +1,39 @@ +version: 1 + +runner: + dockerfile: Dockerfile + +playwright: + config: playwright.config.ts + +sut: + composeFile: compose.e2e.yml + baseUrl: http://127.0.0.1:4173 + +execution: + shards: 1 + artifactRetentionDays: 10 + +profiles: + pull-request: + projects: + - api + - chromium + labels: + - "@smoke" + labelMatch: any + + main: + projects: + - api + - chromium + labels: [] + labelMatch: all + + regression: + projects: + - api + - chromium + labels: + - "@regression" + labelMatch: any diff --git a/e2e/playwright.config.ts b/e2e/playwright.config.ts index 404e493..eed8675 100644 --- a/e2e/playwright.config.ts +++ b/e2e/playwright.config.ts @@ -1,10 +1,7 @@ import { defineConfig, devices } from "@playwright/test"; import { basename, dirname } from "node:path"; -const host = process.env.SUT_HOST ?? "127.0.0.1"; -const port = process.env.SUT_PORT ?? "4173"; -const localBaseUrl = `http://${host}:${port}`; -const baseURL = process.env.BASE_URL ?? localBaseUrl; +const baseURL = process.env.BASE_URL ?? "http://127.0.0.1:4173"; const isCi = Boolean(process.env.CI); const testResultsDir = process.env.E2E_RESULTS_DIR ?? @@ -53,16 +50,4 @@ export default defineConfig({ use: { ...devices["Desktop Chrome"] }, }, ], - webServer: process.env.BASE_URL - ? undefined - : { - command: "node ./sut/server.mjs", - env: { - SUT_HOST: host, - SUT_PORT: port, - }, - url: `${localBaseUrl}/health`, - reuseExistingServer: !isCi, - timeout: 30_000, - }, }); diff --git a/e2e/runner-entrypoint.mjs b/e2e/runner-entrypoint.mjs new file mode 100644 index 0000000..25971c8 --- /dev/null +++ b/e2e/runner-entrypoint.mjs @@ -0,0 +1,57 @@ +import { spawn } from "node:child_process"; +import { mkdirSync } from "node:fs"; +import { dirname } from "node:path"; + +const testOutputRoot = "/test-output"; +const environmentDefaults = { + CI: "true", + E2E_RESULTS_DIR: `${testOutputRoot}/test-results`, + PLAYWRIGHT_BLOB_OUTPUT_DIR: `${testOutputRoot}/blob-report`, + CTRF_OUTPUT_FILE: `${testOutputRoot}/ctrf/ctrf-report.json`, +}; + +for (const [name, value] of Object.entries(environmentDefaults)) { + process.env[name] ??= value; +} + +if (!process.env.BASE_URL) { + console.error("BASE_URL is required, for example http://host.docker.internal:4173"); + process.exit(2); +} + +for (const directory of [ + process.env.E2E_RESULTS_DIR, + process.env.PLAYWRIGHT_BLOB_OUTPUT_DIR, + dirname(process.env.CTRF_OUTPUT_FILE), +]) { + mkdirSync(directory, { recursive: true }); +} + +const playwright = spawn( + "./node_modules/.bin/playwright", + ["test", ...process.argv.slice(2)], + { + env: process.env, + stdio: "inherit", + }, +); + +for (const signal of ["SIGINT", "SIGTERM"]) { + process.once(signal, () => { + playwright.kill(signal); + }); +} + +playwright.once("error", (error) => { + console.error(`Unable to start Playwright: ${error.message}`); + process.exit(1); +}); + +playwright.once("exit", (code, signal) => { + if (signal) { + process.kill(process.pid, signal); + return; + } + + process.exit(code ?? 1); +}); diff --git a/e2e/tests/echo.api.spec.ts b/e2e/tests/echo.api.spec.ts index eafd91f..cc00918 100644 --- a/e2e/tests/echo.api.spec.ts +++ b/e2e/tests/echo.api.spec.ts @@ -8,46 +8,56 @@ interface EchoResponse { body: unknown; } -test("reports that the SUT is healthy", async ({ request }): Promise => { - const response = await request.get("/health"); +test( + "reports that the SUT is healthy", + { tag: "@smoke" }, + async ({ request }): Promise => { + const response = await request.get("/health"); - expect(response.ok()).toBe(true); - await expect(response.json()).resolves.toEqual({ status: "ok" }); -}); + expect(response.ok()).toBe(true); + await expect(response.json()).resolves.toEqual({ status: "ok" }); + }, +); -test("echoes a GET request", async ({ request }): Promise => { - const response = await request.get("/echo", { - params: { message: "hello from Playwright" }, - headers: { "x-unused-demonstration-header": "test" }, - }); - const payload = (await response.json()) as EchoResponse; +test( + "echoes a GET request", + { tag: "@smoke" }, + async ({ request }): Promise => { + const response = await request.get("/echo", { + params: { message: "hello from Playwright" }, + headers: { "x-unused-demonstration-header": "test" }, + }); + const payload = (await response.json()) as EchoResponse; - expect(response.ok()).toBe(true); - expect(payload).toMatchObject({ - method: "GET", - path: "/echo", - query: { message: "hello from Playwright" }, - body: null, - }); - expect(payload.headers.accept).toBe("*/*"); -}); + expect(response.ok()).toBe(true); + expect(payload).toMatchObject({ + method: "GET", + path: "/echo", + query: { message: "hello from Playwright" }, + body: null, + }); + expect(payload.headers.accept).toBe("*/*"); + }, +); -test("echoes a POST request with a JSON body", async ({ - request, -}): Promise => { - const requestBody = { - message: "hello from a real process boundary", - sequence: 42, - }; - const response = await request.post("/echo", { data: requestBody }); - const payload = (await response.json()) as EchoResponse; +test( + "echoes a POST request with a JSON body", + { tag: "@regression" }, + async ({ request }): Promise => { + const requestBody = { + message: "hello from a real process boundary", + sequence: 42, + }; + const response = await request.post("/echo", { data: requestBody }); + const payload = (await response.json()) as EchoResponse; - expect(response.ok()).toBe(true); - expect(payload).toMatchObject({ - method: "POST", - path: "/echo", - query: {}, - body: requestBody, - }); - expect(payload.headers["content-type"]).toContain("application/json"); -}); + expect(response.ok()).toBe(true); + expect(payload).toMatchObject({ + method: "POST", + path: "/echo", + query: {}, + body: requestBody, + }); + expect(payload.headers["content-type"]).toContain("application/json"); + }, +); diff --git a/e2e/tests/echo.browser.spec.ts b/e2e/tests/echo.browser.spec.ts index c693362..47ef816 100644 --- a/e2e/tests/echo.browser.spec.ts +++ b/e2e/tests/echo.browser.spec.ts @@ -1,11 +1,13 @@ import { expect, test } from "@playwright/test"; -test("navigates to the echo endpoint in Chromium", async ({ - page, -}): Promise => { - const response = await page.goto("/echo?source=chromium"); +test( + "navigates to the echo endpoint in Chromium", + { tag: ["@smoke", "@browser"] }, + async ({ page }): Promise => { + const response = await page.goto("/echo?source=chromium"); - expect(response?.ok()).toBe(true); - await expect(page.locator("body")).toContainText('"method":"GET"'); - await expect(page.locator("body")).toContainText('"source":"chromium"'); -}); + expect(response?.ok()).toBe(true); + await expect(page.locator("body")).toContainText('"method":"GET"'); + await expect(page.locator("body")).toContainText('"source":"chromium"'); + }, +); diff --git a/sut/.dockerignore b/sut/.dockerignore new file mode 100644 index 0000000..281787e --- /dev/null +++ b/sut/.dockerignore @@ -0,0 +1,3 @@ +* +!server.mjs +!Dockerfile diff --git a/sut/Dockerfile b/sut/Dockerfile new file mode 100644 index 0000000..2fe8480 --- /dev/null +++ b/sut/Dockerfile @@ -0,0 +1,18 @@ +FROM node:22.22.1-alpine3.23@sha256:8094c002d08262dba12645a3b4a15cd6cd627d30bc782f53229a2ec13ee22a00 + +ENV NODE_ENV=production \ + SUT_HOST=0.0.0.0 \ + SUT_PORT=4173 + +WORKDIR /app + +COPY --chown=node:node server.mjs ./server.mjs + +USER node + +EXPOSE 4173 + +HEALTHCHECK --interval=2s --timeout=2s --start-period=2s --retries=10 \ + CMD ["node", "-e", "fetch('http://127.0.0.1:4173/health').then(response => { if (!response.ok) process.exit(1) }).catch(() => process.exit(1))"] + +CMD ["node", "server.mjs"] diff --git a/e2e/sut/server.mjs b/sut/server.mjs similarity index 98% rename from e2e/sut/server.mjs rename to sut/server.mjs index 3849329..5dbed04 100644 --- a/e2e/sut/server.mjs +++ b/sut/server.mjs @@ -1,6 +1,6 @@ import { createServer } from "node:http"; -const host = process.env.SUT_HOST ?? "127.0.0.1"; +const host = process.env.SUT_HOST ?? "0.0.0.0"; const port = Number.parseInt(process.env.SUT_PORT ?? "4173", 10); const maximumBodyBytes = 1024 * 1024;