From efa5401f12a32733c2274376d97c1eca0153f347 Mon Sep 17 00:00:00 2001 From: Aleksandr Shevchuk Date: Tue, 17 Nov 2026 14:41:12 +0200 Subject: [PATCH 1/2] ci: test what a change reaches, not everything on every push (#224) A pull request tests the packages it changes and those that depend on them, as pnpm's ...[] filter reads the change; skills/ adds the author package and apps/skills, and any other file outside the packages that is not docs tests everything. Firefox and WebKit run when the player, the runtime or the demo changed, the VS Code job when the extension or what it carries did. A pull request's run leaves out the player's three browser files that wait out real time (H5P_TEST_SKIP_SLOW); a push to main tests what it changed with every file, and a weekly run tests everything. A change to the docs alone starts no run. --- .github/workflows/ci.yml | 97 ++++++++++++++++++++++++++++++++-- CONTRIBUTING.md | 5 +- design/testing.md | 53 ++++++++++--------- packages/player/vite.config.ts | 9 ++++ 4 files changed, 133 insertions(+), 31 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e76088a3..5b5ce1ea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,20 +1,97 @@ name: CI +# A pull request tests what it changes: the packages it touches and those that depend on them, +# Firefox and WebKit only when the player, the runtime or the demo changed, and without the +# player's browser files that wait out real time. A push to main tests what it changed with every file; the +# weekly run and a run by hand test everything. A change to the docs alone starts no run. + on: push: branches: - main + paths-ignore: + - 'design/**' + - 'docs/**' + - '*.md' + - LICENSE pull_request: branches: - main + paths-ignore: + - 'design/**' + - 'docs/**' + - '*.md' + - LICENSE + # Everything, and the week's caches saved on main, which pull requests restore. + schedule: + - cron: '0 5 * * 1' workflow_dispatch: permissions: contents: read jobs: + # What the run tests. `select` picks the packages for pnpm: `-r` for all of them, or by name the + # ones the change touches and those that depend on them, as pnpm's `...[]` reads it. A + # file outside packages/, apps/ and skills/ that is not docs (the lockfile, a workflow, the + # root's package.json) tests everything: what it reaches cannot be read off its path. + scope: + runs-on: ubuntu-latest + outputs: + select: ${{ steps.scope.outputs.select }} + demo: ${{ steps.scope.outputs.demo }} + engines: ${{ steps.scope.outputs.engines }} + player: ${{ steps.scope.outputs.player }} + vscode: ${{ steps.scope.outputs.vscode }} + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 2 + - uses: pnpm/action-setup@v6 + # bash by name, which adds pipefail: a pnpm that fails must fail the run, not pick nothing. + - id: scope + shell: bash + env: + EVENT: ${{ github.event_name }} + BEFORE: ${{ github.event.before }} + run: | + everything() { + printf '%s\n' select=-r demo=true engines=true player=true vscode=true | tee -a "$GITHUB_OUTPUT" + exit 0 + } + case "$EVENT" in + # A pull request's run checks out the merge commit, whose first parent is main. + pull_request) base=HEAD^1 ;; + # Where main was before the push; a new or forced branch may have nothing to compare. + push) git fetch -q --depth=1 origin "$BEFORE" || everything; base=$BEFORE ;; + *) everything ;; + esac + changed=$(git diff --name-only "$base" HEAD) + if grep -qvE '^(packages|apps|skills|design|docs)/|^[^/]+\.md$|^LICENSE$' <<< "$changed"; then everything; fi + # pnpm counts a file outside the packages as the root's, whose test script tests them all. + picked=$(pnpm --filter "...[$base]" ls --depth -1 --json | jq -r --arg root "$PWD" '.[] | select(.path != $root) | .name') + # The skills are tested from apps/skills, and h5p-author's ships in the author package. + if grep -q '^skills/' <<< "$changed"; then + picked=$(printf '%s\n' $picked @missing-elements/h5p-author @missing-elements/h5p-skills | sort -u) + fi + has() { grep -qx "$1" <<< "$picked" && echo true || echo false; } + select= + for name in $picked; do select="$select --filter $name"; done + # Firefox and WebKit run the player's browser suite when the player changed, and the + # installable app's tests when either did. + engines=$(has @missing-elements/h5p-offline-player) + if grep -q '^apps/demo/' <<< "$changed"; then engines=true; fi + printf '%s\n' "select=$select" "demo=$(has @missing-elements/h5p-demo)" "engines=$engines" \ + "player=$(has @missing-elements/h5p-offline-player)" "vscode=$(has h5p-preview)" | tee -a "$GITHUB_OUTPUT" + test: + needs: scope + if: needs.scope.outputs.select != '' runs-on: ubuntu-latest + env: + SELECT: ${{ needs.scope.outputs.select }} + # The player's slow browser files stay out of a pull request's run (vite.config.ts). + H5P_TEST_SKIP_SLOW: ${{ github.event_name == 'pull_request' && '1' || '' }} steps: - uses: actions/checkout@v5 - uses: pnpm/action-setup@v6 @@ -25,14 +102,17 @@ jobs: - run: pnpm install --frozen-lockfile # The demo first, from a checkout with nothing built, as Vercel builds it. Run after `pnpm # build`, it found the player's dist/ there, and passed while the deploy of #123 failed. - - run: pnpm build:demo + - if: needs.scope.outputs.demo == 'true' + run: pnpm build:demo # Without --with-deps: the runner image ships Chrome and Firefox, so the libraries they link # against are there, and what apt would add (fonts, Xvfb, FFmpeg's codecs) a headless suite that # decodes no video does not use. Some runners reach Ubuntu's mirror at 40 kB/s, where that took # up to 20 minutes (#73). - run: pnpm --filter @missing-elements/h5p-offline-player exec playwright install chromium + # Every package whatever the change, in seconds: each one's check writes the types the + # packages after it read. - run: pnpm typecheck - - run: pnpm test + - run: pnpm $SELECT --workspace-concurrency=1 test - run: pnpm build # The player's browser suite and the installable app's tests in the other two engines; the job above runs @@ -42,7 +122,12 @@ jobs: # just after them — each found here first. engines: name: browser suite (${{ matrix.browser }}) + needs: scope + if: needs.scope.outputs.engines == 'true' runs-on: ubuntu-latest + env: + H5P_TEST_BROWSER: ${{ matrix.browser }} + H5P_TEST_SKIP_SLOW: ${{ github.event_name == 'pull_request' && '1' || '' }} strategy: fail-fast: false matrix: @@ -91,14 +176,16 @@ jobs: # rather than every week's. - if: matrix.deps run: sudo apt-get autoclean - - run: pnpm test:engine - env: - H5P_TEST_BROWSER: ${{ matrix.browser }} + - if: needs.scope.outputs.player == 'true' + run: pnpm --filter @missing-elements/h5p-offline-player test:engine + - run: pnpm --filter @missing-elements/h5p-demo test # The VS Code extension in a real VS Code, on a display Xvfb gives it, and the .vsix the owner # publishes, built from what the tests ran against. The unit tests run in the job above. vscode: name: VS Code extension + needs: scope + if: needs.scope.outputs.vscode == 'true' runs-on: ubuntu-latest steps: - uses: actions/checkout@v5 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c370caae..79c37f7e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -50,7 +50,10 @@ pnpm build `pnpm test:browser` exercises the player in Chromium, including its Service Worker and Jobs worker. Run it for changes to playback, storage, worker, network, or browser-facing behavior. CI also runs the suite, and the demo's -tests, in Firefox and WebKit; for a change to the Service Worker, the Jobs +tests, in Firefox and WebKit when the player or the demo changes. A pull +request's run tests only the packages it changes and those that depend on them, +and leaves out three slow browser files that main's run has. For a change to +the Service Worker, the Jobs worker, storage or the demo's pages, run them there too, one engine at a time, after installing them with `pnpm --filter @missing-elements/h5p-offline-player exec playwright install firefox webkit`: diff --git a/design/testing.md b/design/testing.md index eec27f29..d7db06d2 100644 --- a/design/testing.md +++ b/design/testing.md @@ -4,31 +4,34 @@ How the repository is tested: the engines, CI's browsers, the owner's rule on wh ## Engines, CI and what is tested -The `browser` project runs in Chromium unless `H5P_TEST_BROWSER` names `firefox` or `webkit`, and -`pnpm test:engine` runs it alone, which is what CI does for those two (the `engines` job, one -engine per job; the `test` job covers Chromium). One engine per run, never a list: the dev -server's switches — a host's outage, the worker version it serves — are shared, and two engines -running one file at once flipped them under each other. `browser-emulated` stays on Chromium, -since CDP is Chromium's. Added on 2026-10-13, after Safari had only ever been checked by hand; -the first runs found the three engine differences under *A body that streams holds its fetch -event open*, *An untrusted package plays as data* (the worker's own script in its policy) and *A -downloaded archive is read back before it is reported done*. Playwright's WebKit is WebKit's -desktop port and not iOS Safari: the app's menu filled an iPhone's screen in Safari on 2026-10-13 -and drew correctly in Playwright's WebKit at every phone size. - -CI installs Chromium and Firefox without their system packages (#73, 2026-10-23): the runner image -ships both browsers, so the libraries they link against are there, and what `--with-deps` adds — -fonts, Xvfb, FFmpeg's codecs — a headless suite that decodes no video does not use. WebKit needs -its packages, GStreamer and GTK 4 among them, 117 MB from Ubuntu's mirror, and the `engines` job -keeps them in the Actions cache (#77): `/var/cache/apt/archives`, keyed by ISO week with any -earlier week as the fallback, and `apt-get autoclean` after the install so one version of each -package stays. Measured that morning: `azure.archive.ubuntu.com` served the package lists at -12 MB/s and the packages at 40 kB/s — 115 MB in 47 minutes, in three runs of three, against 30 to -60 seconds in the eight `main` runs before. The image's apt failover (actions/runner-images#14594) -fires on a stall, not on a trickle, which is actions/runner-images#14823, open; the cache restores -from GitHub's storage instead, 111 MB in 3 seconds, and the install then took 26 seconds. A cache -saved by a pull request's run is that pull request's: `main`'s first run after the merge fills the -one every later pull request reads. +**CI tests what a change reaches.** A pull request's run tests the packages it changes and those +that depend on them, as pnpm's `...[]` filter reads the change. A change under `skills/` +adds the author package and `apps/skills`. A file anywhere else that is not docs (the lockfile, a +workflow, the root's `package.json`) tests everything. Type checking covers every package +whatever the change, since each package's check writes the types the next one reads. A push to +main tests what it changed the same way. The weekly run and a run by hand test everything, and a +change to `design/`, `docs/` or a root `.md` alone starts no run. The `scope` job in `ci.yml` +decides. + +**A pull request's run leaves out the player's browser files that wait out real time.** These are +`stalling-host`, `worker-update` and `large-media`, named in the player's `vite.config.ts` and +dropped under `H5P_TEST_SKIP_SLOW`: about two minutes of test time in each engine. Main's run after +the merge has them, so wait for it before publishing the player. + +The `browser` project runs in Chromium unless `H5P_TEST_BROWSER` names `firefox` or `webkit`. +`pnpm test:engine` runs it, then the app's tests. In those two engines the `engines` job runs the +suite when the player or the runtime changed, and the app's tests then or when the demo did. One engine +per run, never a list: the dev server's switches (a host's outage, the worker version it serves) +are shared, and two engines running one file at once flipped them under each other. +`browser-emulated` stays on Chromium, since CDP is Chromium's. Playwright's WebKit is WebKit's +desktop port, not iOS Safari. + +CI installs Chromium and Firefox without their system packages. The runner image has what they +link against, and `--with-deps` adds fonts, Xvfb and codecs that a headless suite does not use, +from a mirror some runners reach at 40 kB/s (#73). WebKit needs its 117 MB of packages, which the +`engines` job keeps in the Actions cache, keyed by ISO week, running `apt-get autoclean` after the +install (#77). A pull request's cache is its own; main's runs, the weekly one among them, fill the +one pull requests read. **What is tested, by the owner's rule of 2026-10-17: the packages and their source, and of the demo app only the installable app.** The demo's pages — the player page, the examples, `/embed`, diff --git a/packages/player/vite.config.ts b/packages/player/vite.config.ts index bd9ba172..493a27a8 100644 --- a/packages/player/vite.config.ts +++ b/packages/player/vite.config.ts @@ -1,6 +1,7 @@ /// import { resolve } from 'node:path' import { defineConfig } from 'vite' +import { configDefaults } from 'vitest/config' import { playwright } from '@vitest/browser-playwright' import { devServiceWorkerPlugin, jobsWorkerPlugin, noRangeFixturesPlugin } from './vite.plugins' @@ -9,6 +10,13 @@ import { devServiceWorkerPlugin, jobsWorkerPlugin, noRangeFixturesPlugin } from * app (`apps/demo/vite.config.ts`), which is the dev server, needs the same ones. */ +/** + * The browser files that wait out real time: a host silent past the stall bound, a worker + * update's deadlines, 20 MB archives. `H5P_TEST_SKIP_SLOW` leaves them out, as a pull request's + * run does; a run on main has them. + */ +const SLOW = ['tests/browser/stalling-host.test.ts', 'tests/browser/worker-update.test.ts', 'tests/browser/large-media.test.ts'] + export default defineConfig({ plugins: [jobsWorkerPlugin(), devServiceWorkerPlugin(), noRangeFixturesPlugin()], @@ -42,6 +50,7 @@ export default defineConfig({ test: { name: 'browser', include: ['tests/browser/**/*.test.ts'], + exclude: process.env.H5P_TEST_SKIP_SLOW ? [...configDefaults.exclude, ...SLOW] : configDefaults.exclude, browser: browserProject(engine()) } }, From 139b9b5f4e98d54220265e65aeacb3944d4d2000 Mon Sep 17 00:00:00 2001 From: Aleksandr Shevchuk Date: Tue, 17 Nov 2026 14:49:22 +0200 Subject: [PATCH 2/2] docs: say exactly what each CI job runs, LICENSE among the docs (#224) The engines job runs when the player, the runtime or the demo changed, the app's tests every time and the player's suite only for the first two. A change outside the packages tests everything unless it is docs, LICENSE included, and typecheck runs in every package that has the script. --- CONTRIBUTING.md | 12 +++++++----- design/testing.md | 24 ++++++++++++------------ 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 79c37f7e..6be98f56 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -49,11 +49,13 @@ pnpm build `pnpm test:browser` exercises the player in Chromium, including its Service Worker and Jobs worker. Run it for changes to playback, storage, worker, -network, or browser-facing behavior. CI also runs the suite, and the demo's -tests, in Firefox and WebKit when the player or the demo changes. A pull -request's run tests only the packages it changes and those that depend on them, -and leaves out three slow browser files that main's run has. For a change to -the Service Worker, the Jobs +network, or browser-facing behavior. CI also runs the suite in Firefox and +WebKit when the player or the runtime changes, and the demo's tests there when +either of them or the demo changes. A pull request's run tests only the +packages it changes and those that depend on them, and leaves out three slow +browser files that main's run has; a change outside the packages, such as the +lockfile or a workflow, tests everything. For a change to the Service Worker, +the Jobs worker, storage or the demo's pages, run them there too, one engine at a time, after installing them with `pnpm --filter @missing-elements/h5p-offline-player exec playwright install firefox webkit`: diff --git a/design/testing.md b/design/testing.md index d7db06d2..e00e1439 100644 --- a/design/testing.md +++ b/design/testing.md @@ -6,12 +6,12 @@ How the repository is tested: the engines, CI's browsers, the owner's rule on wh **CI tests what a change reaches.** A pull request's run tests the packages it changes and those that depend on them, as pnpm's `...[]` filter reads the change. A change under `skills/` -adds the author package and `apps/skills`. A file anywhere else that is not docs (the lockfile, a -workflow, the root's `package.json`) tests everything. Type checking covers every package -whatever the change, since each package's check writes the types the next one reads. A push to -main tests what it changed the same way. The weekly run and a run by hand test everything, and a -change to `design/`, `docs/` or a root `.md` alone starts no run. The `scope` job in `ci.yml` -decides. +adds the author package and `apps/skills`. Any other file outside the packages (the lockfile, a +workflow, the root's `package.json`) tests everything, unless it is one of the docs below. Every +package that has a `typecheck` script runs it whatever the change, since each package's check +writes the types the next one reads. A push to main tests what it changed the same way. The +weekly run and a run by hand test everything, and a change to `design/`, `docs/`, a root `.md` or +`LICENSE` alone starts no run. The `scope` job in `ci.yml` decides. **A pull request's run leaves out the player's browser files that wait out real time.** These are `stalling-host`, `worker-update` and `large-media`, named in the player's `vite.config.ts` and @@ -19,12 +19,12 @@ dropped under `H5P_TEST_SKIP_SLOW`: about two minutes of test time in each engin the merge has them, so wait for it before publishing the player. The `browser` project runs in Chromium unless `H5P_TEST_BROWSER` names `firefox` or `webkit`. -`pnpm test:engine` runs it, then the app's tests. In those two engines the `engines` job runs the -suite when the player or the runtime changed, and the app's tests then or when the demo did. One engine -per run, never a list: the dev server's switches (a host's outage, the worker version it serves) -are shared, and two engines running one file at once flipped them under each other. -`browser-emulated` stays on Chromium, since CDP is Chromium's. Playwright's WebKit is WebKit's -desktop port, not iOS Safari. +`pnpm test:engine` runs it, then the app's tests. The `engines` job runs in those two engines when +the player, the runtime or the demo changed: the app's tests every time, and the player's suite only +when the player or the runtime changed. One engine per run, never a list: the dev server's switches +(a host's outage, the worker version it serves) are shared, and two engines running one file at once +flipped them under each other. `browser-emulated` stays on Chromium, since CDP is Chromium's. +Playwright's WebKit is WebKit's desktop port, not iOS Safari. CI installs Chromium and Firefox without their system packages. The runner image has what they link against, and `--with-deps` adds fonts, Xvfb and codecs that a headless suite does not use,