diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 000000000..319617403 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,62 @@ +name: Tests + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +concurrency: + group: tests-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + unit: + name: Unit tests (Vitest) + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v6 + - uses: oven-sh/setup-bun@v2 + - run: bun install --frozen-lockfile + - run: bun run test --run + + e2e: + name: Export end-to-end (Playwright) + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v6 + - uses: oven-sh/setup-bun@v2 + - run: bun install --frozen-lockfile + + # ffmpeg makes the input fixtures and ffprobe checks the exported files. + - name: Install ffmpeg + run: sudo apt-get update && sudo apt-get install -y --no-install-recommends ffmpeg + + - name: Make video fixtures + run: scripts/make-fixtures.sh --small + + # Telemetry off: CI must never report to Datadog. + - name: Build the static site + run: bun run build + env: + NEXT_PUBLIC_DATADOG_ENV: "off" + + - name: Install Chromium for Playwright + run: bunx playwright install --with-deps chromium + + # Served with COOP/COEP (multi-threaded core, as in production) and + # without (single-threaded core); see playwright.config.ts. + - name: Run Playwright + run: bunx playwright test + + - name: Upload Playwright report + if: failure() + uses: actions/upload-artifact@v4 + with: + name: playwright-results + path: test-results/ + retention-days: 7 diff --git a/.gitignore b/.gitignore index 1d982f0b4..cc717af46 100644 --- a/.gitignore +++ b/.gitignore @@ -50,3 +50,10 @@ chromatic.log # Local Netlify folder .netlify + +# generated video fixtures (scripts/make-fixtures.sh) +/test/fixtures/ + +# playwright +/test-results/ +/playwright-report/ diff --git a/bun.lock b/bun.lock index c91ebf99a..ffc20a3a5 100644 --- a/bun.lock +++ b/bun.lock @@ -22,6 +22,7 @@ "wasm-feature-detect": "^1.8.0", }, "devDependencies": { + "@playwright/test": "1.64.1", "@storybook/addon-a11y": "^10.5.10", "@storybook/addon-docs": "^10.5.10", "@storybook/addon-themes": "^10.5.10", @@ -566,6 +567,8 @@ "@oxc-resolver/binding-win32-x64-msvc": ["@oxc-resolver/binding-win32-x64-msvc@11.21.2", "", { "os": "win32", "cpu": "x64" }, "sha512-VPoCAhKvCQTlG7vxqaBXcmuvbh77BfnGXj8g0pbvVXpm1F/R8rDVwqIWcEbMzrI1JvJlm8v7T9uMVQb6UctMRg=="], + "@playwright/test": ["@playwright/test@1.64.1", "", { "dependencies": { "playwright": "1.64.1" }, "bin": { "playwright": "cli.js" } }, "sha512-IaYGml/nSOiZWROfPtuIL7neuu9qDOZ3TNvNljUIrhDMxiGTgPjzj/g9p9RxpS2tSepPDRY+UwAaEeiKtUvIAg=="], + "@pmmmwh/react-refresh-webpack-plugin": ["@pmmmwh/react-refresh-webpack-plugin@0.5.17", "", { "dependencies": { "ansi-html": "^0.0.9", "core-js-pure": "^3.23.3", "error-stack-parser": "^2.0.6", "html-entities": "^2.1.0", "loader-utils": "^2.0.4", "schema-utils": "^4.2.0", "source-map": "^0.7.3" }, "peerDependencies": { "@types/webpack": "4.x || 5.x", "react-refresh": ">=0.10.0 <1.0.0", "sockjs-client": "^1.4.0", "type-fest": ">=0.17.0 <5.0.0", "webpack": ">=4.43.0 <6.0.0", "webpack-dev-server": "3.x || 4.x || 5.x", "webpack-hot-middleware": "2.x", "webpack-plugin-serve": "0.x || 1.x" }, "optionalPeers": ["@types/webpack", "sockjs-client", "type-fest", "webpack-dev-server", "webpack-hot-middleware", "webpack-plugin-serve"] }, "sha512-tXDyE1/jzFsHXjhRZQ3hMl0IVhYe5qula43LDWIhVfjp9G/nT5OQY5AORVOrkEGAUltBJOfOWeETbmhm6kHhuQ=="], "@rolldown/binding-android-arm-eabi": ["@rolldown/binding-android-arm-eabi@1.2.6", "", { "os": "android", "cpu": "arm" }, "sha512-b+jTcARdTiFLI6jB4a5XjTm0RWd6KcRfQj/I2356fxUZemiho9zQLxo0RtCuMDAyKcLo6cEltkgbQp6d1+sjjQ=="], @@ -1626,6 +1629,10 @@ "pkg-dir": ["pkg-dir@7.0.0", "", { "dependencies": { "find-up": "^6.3.0" } }, "sha512-Ie9z/WINcxxLp27BKOCHGde4ITq9UklYKDzVo1nhk5sqGEXU3FpkwP5GM2voTGJkGd9B3Otl+Q4uwSOeSUtOBA=="], + "playwright": ["playwright@1.64.1", "", { "dependencies": { "playwright-core": "1.64.1" }, "bin": { "playwright": "cli.js" } }, "sha512-HCrVhHr5QQuwYoswEUGTb7gGp522yflbv/ST8nF4sFWGuLjQCt17Q/EubpGGSNKYn1tsTCPZxqHhHCfVURu7UA=="], + + "playwright-core": ["playwright-core@1.64.1", "", { "bin": { "playwright-core": "cli.js" } }, "sha512-BC4s8y6SEVA6Q1rSxZTto8hI1KUI3xXlHGZgvGqP4IqPdI/a6FX9H29IDNz7vKo9l21b6LfYosScPL6M3xxyDQ=="], + "possible-typed-array-names": ["possible-typed-array-names@1.1.0", "", {}, "sha512-/+5VFTchJDoVj3bhoqi6UeymcD00DAwb1nJwamzPvHEszJ4FpF6SNNbUbOS8yI56qHzdV8eK0qEfOSiodkTdxg=="], "postcss": ["postcss@8.5.26", "", { "dependencies": { "nanoid": "^3.3.17", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-u82N74LFzG8ca+dD8puPnplTXoGH4fTPpVGuIbt36G3qvNlkvfD0lEAZSxaly3KX8TS/L1A1gsCEmvKmBcVbkQ=="], diff --git a/docs/DATADOG.md b/docs/DATADOG.md index 72e99746e..99b7662a5 100644 --- a/docs/DATADOG.md +++ b/docs/DATADOG.md @@ -44,8 +44,8 @@ its own, so we can see which browser and device combinations fail. |---|---|---| | `export_started` | Export clicked and settings validated | `format`, `resolution`, `input_size`, `clip_length`, `has_text`, `has_music`, `has_image_overlay`, `is_retry`, `retry_after` (retries only: the previous failure's category) | | `export_succeeded` | Terminal | the context above, plus `engine`, `fell_back` (true after an engine fallback), `durationMs` (click to done), `encode_ms`, `output_size` | -| `export_failed` | Terminal | the context, plus `engine`, `durationMs`, `stage` (`engine` / `encode`), `category` | -| `export_cancelled` | Terminal: the user cancelled | the context, plus `engine`, `durationMs`, `stage` | +| `export_failed` | Terminal | the context, plus `engine`, `durationMs`, `stage` (`engine` / `encode`), `category`, and `stall_reason` for stalls | +| `export_cancelled` | Terminal: the user cancelled | the context, plus `engine`, `durationMs`, `stage`, `stuck` (true when progress hadn't changed for 60 s: the user gave up on a hang, and monitors count it as a failure) | | `engine_fallback` | The multi-threaded engine stalled; the same export restarts automatically on the single-threaded engine. Not terminal: the attempt's outcome follows | the context, plus `from: multi`, `to: single`, `stall_reason` | | `ffmpeg_load_success` | A worker loaded the engine (once per worker) | `engine` (`multi` / `single`, what actually loaded), `engine_requested`, `single_reason` (`not_isolated` / `after_stall` / `mt_load_failed`, single-threaded only), `load_number` (`first` / `repeat`), `durationMs` | | `ffmpeg_load_failure` | A worker failed to load (not sent when the user cancelled) | `engine_requested`, `load_number`, `category` (`network` / `integrity` / `wasm` / `worker_script` / `unknown`), `durationMs` | @@ -69,12 +69,21 @@ as part of the schema. | Category | Meaning | |---|---| | `engine_load` | The engine never became ready (any failure in the `engine` stage) | -| `timeout` | No progress for 60 s; the next attempt uses the single-threaded engine | +| `timeout` | The stall watchdog stopped the export (see `stall_reason`); the next attempt uses the single-threaded engine | | `unsupported_format` | FFmpeg couldn't decode the input | | `out_of_memory` | An allocation failed, or a `RangeError` | | `encoder_crash` | A WASM trap in the encoder (`memory access out of bounds`, `unreachable`) | | `network` | A network error after the engine started | | `encode_error` | Any other FFmpeg failure | +| `unknown_duration` | The browser couldn't tell how long the clip is (`clip_length: unknown`), and the export then stalled or failed. Usually a live recording with no duration header that the seek-to-end fallback couldn't resolve | + +`export_failed.stall_reason` (stalls only): + +| Reason | Meaning | +|---|---| +| `no_activity` | No heartbeat or log line for 60 s: the engine is wedged (the multi-threaded thread-pool deadlock looked like this) | +| `no_progress` | FFmpeg kept logging, but the encoded time didn't change for 120 s ("stuck at 100%") | +| `overrun` | Encoding ran past 1.25 × the expected length + 15 s, for example looped music that never stops | ### Guarantees (covered by tests) @@ -125,10 +134,25 @@ Created through the API on 2026-11-03 (site `datadoghq.com`): | Item | ID | |---|---| | Dashboard "Reframe — export reliability" | [`2tw-pge-qkk`](https://app.datadoghq.com/dashboard/2tw-pge-qkk) | -| Monitor: exports failing in production (> 10 `export_failed` in 1 h; warning at 5) | `340137974` | +| Monitor: exports not completing (started − succeeded − ordinary cancellations > 10 in 1 h; warning at 5). Counts failures, stuck cancellations (`stuck: true`) and abandoned exports | `340137974` | | Monitor: FFmpeg engine failing to load (> 5 `ffmpeg_load_failure` in 1 h) | `340138002` | | Monitor: cross-origin isolation missing (> 3 loads with `single_reason:not_isolated` in 1 h) | `340138004` | | Monitor: frontend errors spike (> 25 uncaught errors in 1 h) | `340138012` | +| Monitor: exports start but none succeed (> 3 `export_started` and 0 `export_succeeded` in 6 h) | `340168191` | +| RUM-based metrics `reframe.export.started` and `reframe.export.succeeded` (production action counts, from 2026-11-03) | — | +| SLO: 95% of started exports succeed, 30 days (and 7 days), warning at 97% | `0ac26b24a9a156729eaa2bdab83d07ff` | +| Monitor: SLO error budget burning > 6× over 6 h and 30 min | `340187598` | + +Why the "none succeed" and "not completing" monitors exist: during the +multi-threaded engine outage (2026-10-22 to 2026-11-03) every export hung, +and hung exports produced no `export_failed` at all, only cancellations or +nothing. The original failure monitor counted only `export_failed`, so it +stayed OK the whole time. + +The SLO counts every start that didn't end in success, so user +cancellations count against it as well; that is deliberate (a cancel is +usually an export that was too slow). With today's traffic a single failure +moves it a lot; treat it as meaningful after a few hundred exports. The monitors only watch `env:production`, never page anyone, and email the maintainer. Their thresholds are absolute counts, chosen for today's low diff --git a/docs/datadog/export-reliability.dashboard.json b/docs/datadog/export-reliability.dashboard.json index 3ebee75c5..53b2a31b9 100644 --- a/docs/datadog/export-reliability.dashboard.json +++ b/docs/datadog/export-reliability.dashboard.json @@ -1,5 +1,5 @@ { - "title": "Reframe — export reliability", + "title": "Reframe \u2014 export reliability", "description": "Export outcomes from Reframe's browser RUM events (src/lib/exportTelemetry.ts). Success rate = export_succeeded / (export_succeeded + export_failed); cancellations are excluded. Treat groups with few finished exports as noise. Setup and facets: docs/DATADOG.md.", "layout_type": "ordered", "template_variables": [ @@ -1009,6 +1009,56 @@ } ] } + }, + { + "definition": { + "title": "Stalled exports by reason and engine", + "type": "query_table", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "rum", + "name": "stalled", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "count" + }, + "search": { + "query": "@type:action service:reframe $env @action.target.name:export_failed @context.category:(timeout OR unknown_duration)" + }, + "group_by": [ + { + "facet": "@context.stall_reason", + "limit": 3, + "sort": { + "aggregation": "count", + "order": "desc" + } + }, + { + "facet": "@context.engine", + "limit": 3, + "sort": { + "aggregation": "count", + "order": "desc" + } + } + ] + } + ], + "formulas": [ + { + "formula": "stalled", + "alias": "Stalled exports" + } + ] + } + ] + } } ] } diff --git a/package.json b/package.json index 8cf6b3057..5037164c9 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "wasm-feature-detect": "^1.8.0" }, "devDependencies": { + "@playwright/test": "1.64.1", "@storybook/addon-a11y": "^10.5.10", "@storybook/addon-docs": "^10.5.10", "@storybook/addon-themes": "^10.5.10", diff --git a/playwright.config.ts b/playwright.config.ts new file mode 100644 index 000000000..533b7f532 --- /dev/null +++ b/playwright.config.ts @@ -0,0 +1,44 @@ +import { defineConfig, devices } from "@playwright/test"; + +// End-to-end export tests against the production build (out/), served with +// the same cross-origin isolation headers as Netlify. +// +// scripts/make-fixtures.sh --small +// NEXT_PUBLIC_DATADOG_ENV=off bun run build +// bunx playwright test +// +// Two projects: "multi-threaded" (COOP/COEP set, so @ffmpeg/core-mt loads, +// as on the live site) and "single-threaded" (no isolation headers). +export default defineConfig({ + testDir: "test/e2e", + // Each test runs real FFmpeg.wasm encodes; one worker keeps CPU contention + // from making timings meaningless. + workers: 1, + fullyParallel: false, + timeout: 240_000, + expect: { timeout: 15_000 }, + retries: process.env.CI ? 1 : 0, + reporter: process.env.CI ? [["list"], ["junit", { outputFile: "test-results/e2e-junit.xml" }]] : "list", + use: { + ...devices["Desktop Chrome"], + acceptDownloads: true, + trace: "retain-on-failure", + }, + projects: [ + { name: "multi-threaded", use: { baseURL: "http://127.0.0.1:4173" } }, + { name: "single-threaded", use: { baseURL: "http://127.0.0.1:4174" } }, + ], + webServer: [ + { + command: "node scripts/serve-isolated.mjs 4173", + url: "http://127.0.0.1:4173", + reuseExistingServer: !process.env.CI, + }, + { + command: "node scripts/serve-isolated.mjs 4174", + url: "http://127.0.0.1:4174", + env: { ISOLATED: "0" }, + reuseExistingServer: !process.env.CI, + }, + ], +}); diff --git a/scripts/make-fixtures.sh b/scripts/make-fixtures.sh new file mode 100755 index 000000000..495a63284 --- /dev/null +++ b/scripts/make-fixtures.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# Recreates the video fixtures used by the end-to-end export tests. +# Output goes to test/fixtures/ (gitignored). Needs the ffmpeg CLI +# (macOS: `brew install ffmpeg`, Ubuntu: `sudo apt-get install ffmpeg`). +# +# scripts/make-fixtures.sh # all fixtures +# scripts/make-fixtures.sh --small # everything except the ~300 MB clip +set -euo pipefail + +cd "$(dirname "$0")/.." +OUT=test/fixtures +mkdir -p "$OUT" + +command -v ffmpeg >/dev/null || { echo "ffmpeg not found; install it first." >&2; exit 1; } + +ff() { ffmpeg -hide_banner -loglevel error -y "$@"; } + +# A moving test pattern with a tone, so trims, speed changes and audio edits +# are visible in the output. +video() { echo "testsrc2=size=$1:rate=30:duration=$2"; } +tone() { echo "sine=frequency=440:sample_rate=48000:duration=$1"; } + +echo "normal.mp4: 1280x720 H.264/AAC, 10 s" +ff -f lavfi -i "$(video 1280x720 10)" -f lavfi -i "$(tone 10)" \ + -c:v libx264 -pix_fmt yuv420p -preset veryfast -c:a aac -b:a 128k \ + -movflags +faststart -shortest "$OUT/normal.mp4" + +# Browser screen recordings (MediaRecorder) are written as a live stream: the +# WebM header has no duration and there are no cues, so