From 7a4f268f06bd72223e1eeb78f5cd8806eb71afd9 Mon Sep 17 00:00:00 2001 From: Ryan Kelly <94ryan.kelly@gmail.com> Date: Sat, 1 Aug 2026 20:55:09 +0100 Subject: [PATCH] ci(web): fail smoke test on WebGL upload errors + baseline-generation workflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Godot 4.7.1 regression shipped a build whose entire UI rendered as garbled boxes, yet the smoke test passed — it only proves the engine boots. Two follow-ups: - smoke.mjs now collects 'WebGL: INVALID_*' console errors and fails after boot if any occurred. Baseline-free, and verified against live deploys: fails on the broken 4.7.1 prod build, passes on the good 4.7.0 preview build. - visual-baselines.yml: one-click ubuntu-latest run of the visual suite with --update-snapshots against a deployed URL, uploading the PNGs as an artifact — the missing step blocking MAZE_VISUAL_BASELINES=1 (baselines must be linux-generated to match CI). Co-Authored-By: Claude Fable 5 --- .github/smoke/smoke.mjs | 22 +++++++++ .github/workflows/visual-baselines.yml | 63 ++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 .github/workflows/visual-baselines.yml diff --git a/.github/smoke/smoke.mjs b/.github/smoke/smoke.mjs index d7abf6c..3ddc4f1 100644 --- a/.github/smoke/smoke.mjs +++ b/.github/smoke/smoke.mjs @@ -14,6 +14,7 @@ if (!url) { const BOOT_TIMEOUT_MS = 150_000; // ~96 MB payload + .NET WASM init const fatalErrors = []; +const webglErrors = []; const browser = await chromium.launch(); try { @@ -22,6 +23,15 @@ try { const t = String(e); if (/abort|Aborted|RuntimeError|unreachable|out of memory/i.test(t)) fatalErrors.push(t); }); + // WebGL upload errors corrupt what's drawn without crashing anything, so a build can + // pass every boot check while rendering garbage. Real case: the Godot 4.7.1 export drew + // the entire UI as garbled boxes (broken font-atlas uploads) while booting "successfully" + // — the only signal was console spam of exactly these messages. Collected here, asserted + // after boot below. + page.on("console", (m) => { + const t = m.text(); + if (/^WebGL: (INVALID_|out of memory)/.test(t) && webglErrors.length < 20) webglErrors.push(t); + }); console.log(`→ Opening ${url}`); const resp = await page.goto(url, { waitUntil: "domcontentloaded", timeout: 60_000 }); @@ -78,6 +88,18 @@ try { throw new Error("Fatal runtime error(s):\n" + fatalErrors.join("\n")); } + // Let a few frames render after boot, then assert the GPU upload path is clean. + // Deduplicated: Chrome repeats the same message per draw call, and one is enough proof. + await page.waitForTimeout(5_000); + if (webglErrors.length) { + const unique = [...new Set(webglErrors)]; + throw new Error( + `WebGL error(s) during boot/first frames — the build renders corrupted output ` + + `even though it boots:\n` + unique.join("\n"), + ); + } + console.log(" WebGL upload path clean (no INVALID_* console errors)"); + // Confirm the deploy is serving the build this run produced, not a stale or partially // replaced one. Vercel promotes an alias after upload, so "deploy succeeded" and "the new // build is what visitors get" are two different claims — this checks the second. diff --git a/.github/workflows/visual-baselines.yml b/.github/workflows/visual-baselines.yml new file mode 100644 index 0000000..1845514 --- /dev/null +++ b/.github/workflows/visual-baselines.yml @@ -0,0 +1,63 @@ +name: Generate Visual Baselines + +# One-click implementation of docs/VISUAL_REGRESSION.md → "Generating the baselines". +# +# Baselines must be produced on the SAME platform CI compares on (ubuntu-latest) — +# Playwright suffixes snapshots per-platform, so PNGs generated on macOS/Windows would +# never match. The web build itself can only be produced on Windows (patched editor), +# so this runs the visual suite with --update-snapshots against an already-DEPLOYED +# build and uploads the PNGs as an artifact. +# +# Procedure: +# 1. Make sure the URL serves a KNOWN-GOOD build of a commit that includes the test +# bridge (seeded URLs need scripts/testing/TestBridge.cs). Verify text renders +# correctly by eye first — a baseline captured from a broken build makes the +# breakage the expected result. +# 2. Dispatch this workflow with that URL (defaults to production). +# 3. Download the "visual-baselines" artifact, INSPECT EVERY PNG, then commit them +# to tests/visual/maze.spec.ts-snapshots/ and set MAZE_VISUAL_BASELINES: "1" in +# web-export.yml in the SAME commit. + +on: + workflow_dispatch: + inputs: + maze_url: + description: "Deployed build to baseline (must be known-good and bridge-capable)" + required: false + default: "https://maze.ryankelly.dev" + +permissions: + contents: read + +jobs: + baselines: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Node + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: "20" + + - name: Install Playwright (chromium) + working-directory: tests/visual + run: | + npm ci + npx playwright install --with-deps chromium + + - name: Generate baselines with --update-snapshots + working-directory: tests/visual + env: + MAZE_URL: ${{ github.event.inputs.maze_url }} + MAZE_TEST_BRIDGE: "1" + MAZE_VISUAL_BASELINES: "1" + run: npx playwright test --project=maze --update-snapshots + + - name: Upload baseline PNGs + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: visual-baselines + path: tests/visual/maze.spec.ts-snapshots/ + if-no-files-found: error