Run E2E against a Dockerized SUT

.github/actions/build-e2e-runner/action.yml

@@ -0,0 +1,82 @@
+name: Build E2E Runner
+description: Build and archive a content-addressed Playwright runner image.
+
+inputs:
+  working-directory:
+    description: Repository-relative Docker build context containing the E2E runner.
+    required: false
+    default: e2e
+  dockerfile:
+    description: Dockerfile path relative to working-directory.
+    required: false
+    default: Dockerfile
+  image-name:
+    description: Complete local Docker image reference, including an immutable tag.
+    required: true
+  archive-path:
+    description: Repository-relative output path for the zstd-compressed Docker image archive.
+    required: false
+    default: .e2e-runner/playwright-e2e-runner.tar.zst
+  content-hash:
+    description: SHA-256 of the E2E source tree used as the cache identity.
+    required: true
+
+outputs:
+  image-name:
+    description: Local Docker image reference saved in the archive.
+    value: ${{ steps.build.outputs.image-name }}
+  image-id:
+    description: Content-addressed Docker image ID.
+    value: ${{ steps.build.outputs.image-id }}
+  archive-path:
+    description: Absolute path to the portable .tar.zst Docker image archive.
+    value: ${{ steps.build.outputs.archive-path }}
+  archive-sha256:
+    description: SHA-256 digest of the compressed Docker archive.
+    value: ${{ steps.build.outputs.archive-sha256 }}
+
+runs:
+  using: composite
+  steps:
+    - id: build
+      name: Build and save runner image
+      shell: bash
+      env:
+        E2E_WORKING_DIRECTORY: ${{ inputs.working-directory }}
+        E2E_DOCKERFILE: ${{ inputs.dockerfile }}
+        E2E_RUNNER_IMAGE: ${{ inputs.image-name }}
+        E2E_ARCHIVE_PATH: ${{ inputs.archive-path }}
+        E2E_CONTENT_HASH: ${{ inputs.content-hash }}
+      run: |
+        set -euo pipefail
+
+        eval "$(node "$GITHUB_ACTION_PATH/prepare-build.mjs" \
+          "$GITHUB_WORKSPACE" \
+          "$E2E_WORKING_DIRECTORY" \
+          "$E2E_DOCKERFILE" \
+          "$E2E_RUNNER_IMAGE" \
+          "$E2E_ARCHIVE_PATH")"
+
+        docker build \
+          --file "$E2E_ABSOLUTE_DOCKERFILE" \
+          --tag "$E2E_RUNNER_IMAGE" \
+          "$E2E_BUILD_CONTEXT"
+
+        image_id="$(docker image inspect --format '{{.Id}}' "$E2E_RUNNER_IMAGE")"
+        if [[ ! "$image_id" =~ ^sha256:[a-f0-9]{64}$ ]]; then
+          echo "Unexpected Docker image ID: $image_id" >&2
+          exit 1
+        fi
+
+        command -v zstd > /dev/null
+        docker save "$E2E_RUNNER_IMAGE" | \
+          zstd -T0 -3 --force -o "$E2E_ABSOLUTE_ARCHIVE_PATH"
+        test -s "$E2E_ABSOLUTE_ARCHIVE_PATH"
+
+        node "$GITHUB_ACTION_PATH/runner-archive.mjs" \
+          "$GITHUB_WORKSPACE" \
+          "$E2E_ARCHIVE_PATH" \
+          "$E2E_RUNNER_IMAGE" \
+          "$E2E_CONTENT_HASH" \
+          "$GITHUB_OUTPUT" \
+          "$image_id"

.github/actions/build-e2e-runner/prepare-build.mjs

@@ -0,0 +1,99 @@
+import {
+  existsSync,
+  lstatSync,
+  mkdirSync,
+  realpathSync,
+  rmSync,
+} from "node:fs";
+import { basename, dirname, isAbsolute, relative, resolve, sep } from "node:path";
+
+const [
+  workspace,
+  requestedWorkingDirectory,
+  requestedDockerfile,
+  imageName,
+  requestedArchivePath,
+] = process.argv.slice(2);
+
+const assertRelative = (value, label) => {
+  if (!value || isAbsolute(value)) {
+    throw new Error(`${label} must be a non-empty repository-relative path`);
+  }
+  const relativePath = relative("/contract-root", resolve("/contract-root", value));
+  if (relativePath === "" || relativePath === ".." || relativePath.startsWith(`..${sep}`)) {
+    throw new Error(`${label} must resolve below its parent directory`);
+  }
+};
+
+assertRelative(requestedWorkingDirectory, "working-directory");
+assertRelative(requestedDockerfile, "dockerfile");
+assertRelative(requestedArchivePath, "archive-path");
+if (!imageName || /[\s\0]/.test(imageName)) {
+  throw new Error("image-name must be a non-empty Docker reference without whitespace");
+}
+
+const workspacePath = realpathSync(workspace);
+const buildContext = resolve(workspacePath, requestedWorkingDirectory);
+if (!existsSync(buildContext) || !lstatSync(buildContext).isDirectory()) {
+  throw new Error(`E2E runner build context does not exist: ${requestedWorkingDirectory}`);
+}
+const resolvedBuildContext = realpathSync(buildContext);
+const contextRelativePath = relative(workspacePath, resolvedBuildContext);
+if (contextRelativePath.startsWith("..") || isAbsolute(contextRelativePath)) {
+  throw new Error("working-directory must not escape GITHUB_WORKSPACE through a symlink");
+}
+
+const dockerfile = resolve(resolvedBuildContext, requestedDockerfile);
+if (!existsSync(dockerfile) || !lstatSync(dockerfile).isFile()) {
+  throw new Error(`E2E runner Dockerfile does not exist: ${requestedDockerfile}`);
+}
+const resolvedDockerfile = realpathSync(dockerfile);
+const dockerfileRelativePath = relative(resolvedBuildContext, resolvedDockerfile);
+if (dockerfileRelativePath.startsWith("..") || isAbsolute(dockerfileRelativePath)) {
+  throw new Error("dockerfile must not escape working-directory through a symlink");
+}
+
+const archivePath = resolve(workspacePath, requestedArchivePath);
+if (basename(archivePath) !== "playwright-e2e-runner.tar.zst") {
+  throw new Error("archive-path must end with playwright-e2e-runner.tar.zst");
+}
+const archiveRelativePath = relative(workspacePath, archivePath);
+if (archiveRelativePath.startsWith("..") || isAbsolute(archiveRelativePath)) {
+  throw new Error("archive-path must remain inside GITHUB_WORKSPACE");
+}
+let existingArchiveAncestor = archivePath;
+while (!existsSync(existingArchiveAncestor)) {
+  const parent = dirname(existingArchiveAncestor);
+  if (parent === existingArchiveAncestor) {
+    throw new Error("archive-path has no existing parent directory");
+  }
+  existingArchiveAncestor = parent;
+}
+const resolvedArchiveAncestor = realpathSync(existingArchiveAncestor);
+const archiveAncestorRelativePath = relative(
+  workspacePath,
+  resolvedArchiveAncestor,
+);
+if (
+  archiveAncestorRelativePath.startsWith("..") ||
+  isAbsolute(archiveAncestorRelativePath)
+) {
+  throw new Error("archive-path must not traverse a symlink outside GITHUB_WORKSPACE");
+}
+mkdirSync(dirname(archivePath), { recursive: true });
+for (const outputPath of [archivePath, resolve(dirname(archivePath), "metadata.json")]) {
+  if (existsSync(outputPath) && lstatSync(outputPath).isDirectory()) {
+    throw new Error(`runner output path must not be a directory: ${outputPath}`);
+  }
+  rmSync(outputPath, { force: true });
+}
+
+const quoteForShell = (value) => `'${value.replaceAll("'", `'"'"'`)}'`;
+for (const [name, value] of Object.entries({
+  E2E_BUILD_CONTEXT: resolvedBuildContext,
+  E2E_ABSOLUTE_DOCKERFILE: resolvedDockerfile,
+  E2E_RUNNER_IMAGE: imageName,
+  E2E_ABSOLUTE_ARCHIVE_PATH: archivePath,
+})) {
+  process.stdout.write(`${name}=${quoteForShell(value)}\n`);
+}

.github/actions/build-e2e-runner/runner-archive.mjs

@@ -0,0 +1,106 @@
+import {
+  appendFileSync,
+  createReadStream,
+  existsSync,
+  lstatSync,
+  realpathSync,
+  statSync,
+  writeFileSync,
+} from "node:fs";
+import { createHash } from "node:crypto";
+import { basename, dirname, isAbsolute, relative, resolve, sep } from "node:path";
+
+const [
+  workspace,
+  requestedArchivePath,
+  expectedImageName,
+  expectedContentHash,
+  outputFile,
+  imageId,
+] = process.argv.slice(2);
+
+const fail = (message) => {
+  throw new Error(message);
+};
+
+if (!/^[a-f0-9]{64}$/.test(expectedContentHash)) {
+  fail("content-hash must be a lowercase SHA-256 digest");
+}
+if (!expectedImageName || /[\s\0]/.test(expectedImageName)) {
+  fail("image-name must be a non-empty Docker reference without whitespace");
+}
+if (!requestedArchivePath || isAbsolute(requestedArchivePath)) {
+  fail("archive-path must be repository-relative");
+}
+
+const workspacePath = realpathSync(workspace);
+const archivePath = resolve(workspacePath, requestedArchivePath);
+if (basename(archivePath) !== "playwright-e2e-runner.tar.zst") {
+  fail("archive-path must end with playwright-e2e-runner.tar.zst");
+}
+const relativeArchive = relative(workspacePath, archivePath);
+if (
+  relativeArchive === "" ||
+  relativeArchive === ".." ||
+  relativeArchive.startsWith(`..${sep}`) ||
+  isAbsolute(relativeArchive)
+) {
+  fail("archive-path must remain inside GITHUB_WORKSPACE");
+}
+if (!existsSync(archivePath) || !lstatSync(archivePath).isFile()) {
+  fail(`runner archive does not exist: ${requestedArchivePath}`);
+}
+if (lstatSync(archivePath).isSymbolicLink()) {
+  fail("runner archive must not be a symbolic link");
+}
+
+const digestFile = async (path) => {
+  const hash = createHash("sha256");
+  const stream = createReadStream(path);
+  for await (const chunk of stream) hash.update(chunk);
+  return hash.digest("hex");
+};
+
+const archiveDigest = await digestFile(archivePath);
+const archiveSize = statSync(archivePath).size;
+const archiveDirectory = realpathSync(dirname(archivePath));
+const relativeArchiveDirectory = relative(workspacePath, archiveDirectory);
+if (
+  relativeArchiveDirectory === ".." ||
+  relativeArchiveDirectory.startsWith(`..${sep}`) ||
+  isAbsolute(relativeArchiveDirectory)
+) {
+  fail("runner archive directory must remain inside GITHUB_WORKSPACE");
+}
+const metadataPath = resolve(archiveDirectory, "metadata.json");
+if (existsSync(metadataPath) && lstatSync(metadataPath).isSymbolicLink()) {
+  fail("runner metadata must not be a symbolic link");
+}
+
+if (!/^sha256:[a-f0-9]{64}$/.test(imageId ?? "")) {
+  fail("image-id must be a content-addressed Docker image ID");
+}
+
+const metadata = {
+  schemaVersion: 1,
+  contentHash: expectedContentHash,
+  imageName: expectedImageName,
+  imageId,
+  archive: {
+    file: "playwright-e2e-runner.tar.zst",
+    sha256: archiveDigest,
+    size: archiveSize,
+  },
+};
+writeFileSync(metadataPath, `${JSON.stringify(metadata, null, 2)}\n`, {
+  mode: 0o600,
+});
+
+for (const [name, value] of [
+  ["image-name", metadata.imageName],
+  ["image-id", metadata.imageId],
+  ["archive-path", archivePath],
+  ["archive-sha256", archiveDigest],
+]) {
+  appendFileSync(outputFile, `${name}=${value}\n`);
+}

.github/actions/finalize-e2e-verdict/action.yml → .github/actions/finalize-e2e-result/action.yml

@@ -1,9 +1,9 @@
-name: Finalize E2E Verdict
-description: Enforce the evidence verdict and all upstream GitHub Actions job results.
+name: Finalize E2E Result
+description: Enforce the merged E2E result and all upstream GitHub Actions job results.
 
 inputs:
-  verdict-file:
-    description: Path to the downloaded verdict.json, absolute or relative to GITHUB_WORKSPACE.
+  result-file:
+    description: Path to the downloaded result.json, absolute or relative to GITHUB_WORKSPACE.
     required: true
   shard-job-result:
     description: GitHub Actions result of the shard matrix job.
@@ -17,29 +17,29 @@ inputs:
     default: success
 
 outputs:
-  verdict:
-    description: Final enforced verdict.
-    value: ${{ steps.finalize.outputs.verdict }}
+  result:
+    description: Final enforced result.
+    value: ${{ steps.finalize.outputs.result }}
   primary-failure:
-    description: Primary failure message, empty when the verdict passed.
+    description: Primary failure message, empty when the result passed.
     value: ${{ steps.finalize.outputs.primary-failure }}
 
 runs:
   using: composite
   steps:
-    - name: Enforce final E2E verdict
+    - name: Enforce final E2E result
       id: finalize
       shell: bash
       env:
-        E2E_VERDICT_FILE: ${{ inputs.verdict-file }}
+        E2E_RESULT_FILE: ${{ inputs.result-file }}
         E2E_SHARD_JOB_RESULT: ${{ inputs.shard-job-result }}
         E2E_MERGE_JOB_RESULT: ${{ inputs.merge-job-result }}
         E2E_PUBLICATION_RESULT: ${{ inputs.publication-result }}
       run: |
         set -euo pipefail
         node "$GITHUB_ACTION_PATH/finalize.mjs" \
           "$GITHUB_WORKSPACE" \
-          "$E2E_VERDICT_FILE" \
+          "$E2E_RESULT_FILE" \
           "$E2E_SHARD_JOB_RESULT" \
           "$E2E_MERGE_JOB_RESULT" \
           "$E2E_PUBLICATION_RESULT" \