diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e76088a3..5b5ce1ea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,20 +1,97 @@ name: CI +# A pull request tests what it changes: the packages it touches and those that depend on them, +# Firefox and WebKit only when the player, the runtime or the demo changed, and without the +# player's browser files that wait out real time. A push to main tests what it changed with every file; the +# weekly run and a run by hand test everything. A change to the docs alone starts no run. + on: push: branches: - main + paths-ignore: + - 'design/**' + - 'docs/**' + - '*.md' + - LICENSE pull_request: branches: - main + paths-ignore: + - 'design/**' + - 'docs/**' + - '*.md' + - LICENSE + # Everything, and the week's caches saved on main, which pull requests restore. + schedule: + - cron: '0 5 * * 1' workflow_dispatch: permissions: contents: read jobs: + # What the run tests. `select` picks the packages for pnpm: `-r` for all of them, or by name the + # ones the change touches and those that depend on them, as pnpm's `...[]` reads it. A + # file outside packages/, apps/ and skills/ that is not docs (the lockfile, a workflow, the + # root's package.json) tests everything: what it reaches cannot be read off its path. + scope: + runs-on: ubuntu-latest + outputs: + select: ${{ steps.scope.outputs.select }} + demo: ${{ steps.scope.outputs.demo }} + engines: ${{ steps.scope.outputs.engines }} + player: ${{ steps.scope.outputs.player }} + vscode: ${{ steps.scope.outputs.vscode }} + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 2 + - uses: pnpm/action-setup@v6 + # bash by name, which adds pipefail: a pnpm that fails must fail the run, not pick nothing. + - id: scope + shell: bash + env: + EVENT: ${{ github.event_name }} + BEFORE: ${{ github.event.before }} + run: | + everything() { + printf '%s\n' select=-r demo=true engines=true player=true vscode=true | tee -a "$GITHUB_OUTPUT" + exit 0 + } + case "$EVENT" in + # A pull request's run checks out the merge commit, whose first parent is main. + pull_request) base=HEAD^1 ;; + # Where main was before the push; a new or forced branch may have nothing to compare. + push) git fetch -q --depth=1 origin "$BEFORE" || everything; base=$BEFORE ;; + *) everything ;; + esac + changed=$(git diff --name-only "$base" HEAD) + if grep -qvE '^(packages|apps|skills|design|docs)/|^[^/]+\.md$|^LICENSE$' <<< "$changed"; then everything; fi + # pnpm counts a file outside the packages as the root's, whose test script tests them all. + picked=$(pnpm --filter "...[$base]" ls --depth -1 --json | jq -r --arg root "$PWD" '.[] | select(.path != $root) | .name') + # The skills are tested from apps/skills, and h5p-author's ships in the author package. + if grep -q '^skills/' <<< "$changed"; then + picked=$(printf '%s\n' $picked @missing-elements/h5p-author @missing-elements/h5p-skills | sort -u) + fi + has() { grep -qx "$1" <<< "$picked" && echo true || echo false; } + select= + for name in $picked; do select="$select --filter $name"; done + # Firefox and WebKit run the player's browser suite when the player changed, and the + # installable app's tests when either did. + engines=$(has @missing-elements/h5p-offline-player) + if grep -q '^apps/demo/' <<< "$changed"; then engines=true; fi + printf '%s\n' "select=$select" "demo=$(has @missing-elements/h5p-demo)" "engines=$engines" \ + "player=$(has @missing-elements/h5p-offline-player)" "vscode=$(has h5p-preview)" | tee -a "$GITHUB_OUTPUT" + test: + needs: scope + if: needs.scope.outputs.select != '' runs-on: ubuntu-latest + env: + SELECT: ${{ needs.scope.outputs.select }} + # The player's slow browser files stay out of a pull request's run (vite.config.ts). + H5P_TEST_SKIP_SLOW: ${{ github.event_name == 'pull_request' && '1' || '' }} steps: - uses: actions/checkout@v5 - uses: pnpm/action-setup@v6 @@ -25,14 +102,17 @@ jobs: - run: pnpm install --frozen-lockfile # The demo first, from a checkout with nothing built, as Vercel builds it. Run after `pnpm # build`, it found the player's dist/ there, and passed while the deploy of #123 failed. - - run: pnpm build:demo + - if: needs.scope.outputs.demo == 'true' + run: pnpm build:demo # Without --with-deps: the runner image ships Chrome and Firefox, so the libraries they link # against are there, and what apt would add (fonts, Xvfb, FFmpeg's codecs) a headless suite that # decodes no video does not use. Some runners reach Ubuntu's mirror at 40 kB/s, where that took # up to 20 minutes (#73). - run: pnpm --filter @missing-elements/h5p-offline-player exec playwright install chromium + # Every package whatever the change, in seconds: each one's check writes the types the + # packages after it read. - run: pnpm typecheck - - run: pnpm test + - run: pnpm $SELECT --workspace-concurrency=1 test - run: pnpm build # The player's browser suite and the installable app's tests in the other two engines; the job above runs @@ -42,7 +122,12 @@ jobs: # just after them — each found here first. engines: name: browser suite (${{ matrix.browser }}) + needs: scope + if: needs.scope.outputs.engines == 'true' runs-on: ubuntu-latest + env: + H5P_TEST_BROWSER: ${{ matrix.browser }} + H5P_TEST_SKIP_SLOW: ${{ github.event_name == 'pull_request' && '1' || '' }} strategy: fail-fast: false matrix: @@ -91,14 +176,16 @@ jobs: # rather than every week's. - if: matrix.deps run: sudo apt-get autoclean - - run: pnpm test:engine - env: - H5P_TEST_BROWSER: ${{ matrix.browser }} + - if: needs.scope.outputs.player == 'true' + run: pnpm --filter @missing-elements/h5p-offline-player test:engine + - run: pnpm --filter @missing-elements/h5p-demo test # The VS Code extension in a real VS Code, on a display Xvfb gives it, and the .vsix the owner # publishes, built from what the tests ran against. The unit tests run in the job above. vscode: name: VS Code extension + needs: scope + if: needs.scope.outputs.vscode == 'true' runs-on: ubuntu-latest steps: - uses: actions/checkout@v5 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c370caae..6be98f56 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -49,8 +49,13 @@ pnpm build `pnpm test:browser` exercises the player in Chromium, including its Service Worker and Jobs worker. Run it for changes to playback, storage, worker, -network, or browser-facing behavior. CI also runs the suite, and the demo's -tests, in Firefox and WebKit; for a change to the Service Worker, the Jobs +network, or browser-facing behavior. CI also runs the suite in Firefox and +WebKit when the player or the runtime changes, and the demo's tests there when +either of them or the demo changes. A pull request's run tests only the +packages it changes and those that depend on them, and leaves out three slow +browser files that main's run has; a change outside the packages, such as the +lockfile or a workflow, tests everything. For a change to the Service Worker, +the Jobs worker, storage or the demo's pages, run them there too, one engine at a time, after installing them with `pnpm --filter @missing-elements/h5p-offline-player exec playwright install firefox webkit`: diff --git a/design/testing.md b/design/testing.md index eec27f29..e00e1439 100644 --- a/design/testing.md +++ b/design/testing.md @@ -4,31 +4,34 @@ How the repository is tested: the engines, CI's browsers, the owner's rule on wh ## Engines, CI and what is tested -The `browser` project runs in Chromium unless `H5P_TEST_BROWSER` names `firefox` or `webkit`, and -`pnpm test:engine` runs it alone, which is what CI does for those two (the `engines` job, one -engine per job; the `test` job covers Chromium). One engine per run, never a list: the dev -server's switches — a host's outage, the worker version it serves — are shared, and two engines -running one file at once flipped them under each other. `browser-emulated` stays on Chromium, -since CDP is Chromium's. Added on 2026-10-13, after Safari had only ever been checked by hand; -the first runs found the three engine differences under *A body that streams holds its fetch -event open*, *An untrusted package plays as data* (the worker's own script in its policy) and *A -downloaded archive is read back before it is reported done*. Playwright's WebKit is WebKit's -desktop port and not iOS Safari: the app's menu filled an iPhone's screen in Safari on 2026-10-13 -and drew correctly in Playwright's WebKit at every phone size. - -CI installs Chromium and Firefox without their system packages (#73, 2026-10-23): the runner image -ships both browsers, so the libraries they link against are there, and what `--with-deps` adds — -fonts, Xvfb, FFmpeg's codecs — a headless suite that decodes no video does not use. WebKit needs -its packages, GStreamer and GTK 4 among them, 117 MB from Ubuntu's mirror, and the `engines` job -keeps them in the Actions cache (#77): `/var/cache/apt/archives`, keyed by ISO week with any -earlier week as the fallback, and `apt-get autoclean` after the install so one version of each -package stays. Measured that morning: `azure.archive.ubuntu.com` served the package lists at -12 MB/s and the packages at 40 kB/s — 115 MB in 47 minutes, in three runs of three, against 30 to -60 seconds in the eight `main` runs before. The image's apt failover (actions/runner-images#14594) -fires on a stall, not on a trickle, which is actions/runner-images#14823, open; the cache restores -from GitHub's storage instead, 111 MB in 3 seconds, and the install then took 26 seconds. A cache -saved by a pull request's run is that pull request's: `main`'s first run after the merge fills the -one every later pull request reads. +**CI tests what a change reaches.** A pull request's run tests the packages it changes and those +that depend on them, as pnpm's `...[]` filter reads the change. A change under `skills/` +adds the author package and `apps/skills`. Any other file outside the packages (the lockfile, a +workflow, the root's `package.json`) tests everything, unless it is one of the docs below. Every +package that has a `typecheck` script runs it whatever the change, since each package's check +writes the types the next one reads. A push to main tests what it changed the same way. The +weekly run and a run by hand test everything, and a change to `design/`, `docs/`, a root `.md` or +`LICENSE` alone starts no run. The `scope` job in `ci.yml` decides. + +**A pull request's run leaves out the player's browser files that wait out real time.** These are +`stalling-host`, `worker-update` and `large-media`, named in the player's `vite.config.ts` and +dropped under `H5P_TEST_SKIP_SLOW`: about two minutes of test time in each engine. Main's run after +the merge has them, so wait for it before publishing the player. + +The `browser` project runs in Chromium unless `H5P_TEST_BROWSER` names `firefox` or `webkit`. +`pnpm test:engine` runs it, then the app's tests. The `engines` job runs in those two engines when +the player, the runtime or the demo changed: the app's tests every time, and the player's suite only +when the player or the runtime changed. One engine per run, never a list: the dev server's switches +(a host's outage, the worker version it serves) are shared, and two engines running one file at once +flipped them under each other. `browser-emulated` stays on Chromium, since CDP is Chromium's. +Playwright's WebKit is WebKit's desktop port, not iOS Safari. + +CI installs Chromium and Firefox without their system packages. The runner image has what they +link against, and `--with-deps` adds fonts, Xvfb and codecs that a headless suite does not use, +from a mirror some runners reach at 40 kB/s (#73). WebKit needs its 117 MB of packages, which the +`engines` job keeps in the Actions cache, keyed by ISO week, running `apt-get autoclean` after the +install (#77). A pull request's cache is its own; main's runs, the weekly one among them, fill the +one pull requests read. **What is tested, by the owner's rule of 2026-10-17: the packages and their source, and of the demo app only the installable app.** The demo's pages — the player page, the examples, `/embed`, diff --git a/packages/player/vite.config.ts b/packages/player/vite.config.ts index bd9ba172..493a27a8 100644 --- a/packages/player/vite.config.ts +++ b/packages/player/vite.config.ts @@ -1,6 +1,7 @@ /// import { resolve } from 'node:path' import { defineConfig } from 'vite' +import { configDefaults } from 'vitest/config' import { playwright } from '@vitest/browser-playwright' import { devServiceWorkerPlugin, jobsWorkerPlugin, noRangeFixturesPlugin } from './vite.plugins' @@ -9,6 +10,13 @@ import { devServiceWorkerPlugin, jobsWorkerPlugin, noRangeFixturesPlugin } from * app (`apps/demo/vite.config.ts`), which is the dev server, needs the same ones. */ +/** + * The browser files that wait out real time: a host silent past the stall bound, a worker + * update's deadlines, 20 MB archives. `H5P_TEST_SKIP_SLOW` leaves them out, as a pull request's + * run does; a run on main has them. + */ +const SLOW = ['tests/browser/stalling-host.test.ts', 'tests/browser/worker-update.test.ts', 'tests/browser/large-media.test.ts'] + export default defineConfig({ plugins: [jobsWorkerPlugin(), devServiceWorkerPlugin(), noRangeFixturesPlugin()], @@ -42,6 +50,7 @@ export default defineConfig({ test: { name: 'browser', include: ['tests/browser/**/*.test.ts'], + exclude: process.env.H5P_TEST_SKIP_SLOW ? [...configDefaults.exclude, ...SLOW] : configDefaults.exclude, browser: browserProject(engine()) } },