Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
97 changes: 92 additions & 5 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
@@ -1,20 +1,97 @@
name: CI

# A pull request tests what it changes: the packages it touches and those that depend on them,
# Firefox and WebKit only when the player, the runtime or the demo changed, and without the
# player's browser files that wait out real time. A push to main tests what it changed with every file; the
# weekly run and a run by hand test everything. A change to the docs alone starts no run.

on:
push:
branches:
- main
paths-ignore:
- 'design/**'
- 'docs/**'
- '*.md'
- LICENSE
pull_request:
branches:
- main
paths-ignore:
- 'design/**'
- 'docs/**'
- '*.md'
- LICENSE
# Everything, and the week's caches saved on main, which pull requests restore.
schedule:
- cron: '0 5 * * 1'
workflow_dispatch:

permissions:
contents: read

jobs:
# What the run tests. `select` picks the packages for pnpm: `-r` for all of them, or by name the
# ones the change touches and those that depend on them, as pnpm's `...[<commit>]` reads it. A
# file outside packages/, apps/ and skills/ that is not docs (the lockfile, a workflow, the
# root's package.json) tests everything: what it reaches cannot be read off its path.
scope:
runs-on: ubuntu-latest
outputs:
select: ${{ steps.scope.outputs.select }}
demo: ${{ steps.scope.outputs.demo }}
engines: ${{ steps.scope.outputs.engines }}
player: ${{ steps.scope.outputs.player }}
vscode: ${{ steps.scope.outputs.vscode }}
steps:
- uses: actions/checkout@v5
with:
fetch-depth: 2
- uses: pnpm/action-setup@v6
# bash by name, which adds pipefail: a pnpm that fails must fail the run, not pick nothing.
- id: scope
shell: bash
env:
EVENT: ${{ github.event_name }}
BEFORE: ${{ github.event.before }}
run: |
everything() {
printf '%s\n' select=-r demo=true engines=true player=true vscode=true | tee -a "$GITHUB_OUTPUT"
exit 0
}
case "$EVENT" in
# A pull request's run checks out the merge commit, whose first parent is main.
pull_request) base=HEAD^1 ;;
# Where main was before the push; a new or forced branch may have nothing to compare.
push) git fetch -q --depth=1 origin "$BEFORE" || everything; base=$BEFORE ;;
*) everything ;;
esac
changed=$(git diff --name-only "$base" HEAD)
if grep -qvE '^(packages|apps|skills|design|docs)/|^[^/]+\.md$|^LICENSE$' <<< "$changed"; then everything; fi
# pnpm counts a file outside the packages as the root's, whose test script tests them all.
picked=$(pnpm --filter "...[$base]" ls --depth -1 --json | jq -r --arg root "$PWD" '.[] | select(.path != $root) | .name')
# The skills are tested from apps/skills, and h5p-author's ships in the author package.
if grep -q '^skills/' <<< "$changed"; then
picked=$(printf '%s\n' $picked @missing-elements/h5p-author @missing-elements/h5p-skills | sort -u)
fi
has() { grep -qx "$1" <<< "$picked" && echo true || echo false; }
select=
for name in $picked; do select="$select --filter $name"; done
# Firefox and WebKit run the player's browser suite when the player changed, and the
# installable app's tests when either did.
engines=$(has @missing-elements/h5p-offline-player)
if grep -q '^apps/demo/' <<< "$changed"; then engines=true; fi
printf '%s\n' "select=$select" "demo=$(has @missing-elements/h5p-demo)" "engines=$engines" \
"player=$(has @missing-elements/h5p-offline-player)" "vscode=$(has h5p-preview)" | tee -a "$GITHUB_OUTPUT"

test:
needs: scope
if: needs.scope.outputs.select != ''
runs-on: ubuntu-latest
env:
SELECT: ${{ needs.scope.outputs.select }}
# The player's slow browser files stay out of a pull request's run (vite.config.ts).
H5P_TEST_SKIP_SLOW: ${{ github.event_name == 'pull_request' && '1' || '' }}
steps:
- uses: actions/checkout@v5
- uses: pnpm/action-setup@v6
Expand All @@ -25,14 +102,17 @@ jobs:
- run: pnpm install --frozen-lockfile
# The demo first, from a checkout with nothing built, as Vercel builds it. Run after `pnpm
# build`, it found the player's dist/ there, and passed while the deploy of #123 failed.
- run: pnpm build:demo
- if: needs.scope.outputs.demo == 'true'
run: pnpm build:demo
# Without --with-deps: the runner image ships Chrome and Firefox, so the libraries they link
# against are there, and what apt would add (fonts, Xvfb, FFmpeg's codecs) a headless suite that
# decodes no video does not use. Some runners reach Ubuntu's mirror at 40 kB/s, where that took
# up to 20 minutes (#73).
- run: pnpm --filter @missing-elements/h5p-offline-player exec playwright install chromium
# Every package whatever the change, in seconds: each one's check writes the types the
# packages after it read.
- run: pnpm typecheck
- run: pnpm test
- run: pnpm $SELECT --workspace-concurrency=1 test
Comment thread
alekswebnet marked this conversation as resolved.
Resolved
- run: pnpm build

# The player's browser suite and the installable app's tests in the other two engines; the job above runs
Expand All @@ -42,7 +122,12 @@ jobs:
# just after them — each found here first.
engines:
name: browser suite (${{ matrix.browser }})
needs: scope
if: needs.scope.outputs.engines == 'true'
runs-on: ubuntu-latest
env:
H5P_TEST_BROWSER: ${{ matrix.browser }}
H5P_TEST_SKIP_SLOW: ${{ github.event_name == 'pull_request' && '1' || '' }}
strategy:
fail-fast: false
matrix:
Expand Down Expand Up @@ -91,14 +176,16 @@ jobs:
# rather than every week's.
- if: matrix.deps
run: sudo apt-get autoclean
- run: pnpm test:engine
env:
H5P_TEST_BROWSER: ${{ matrix.browser }}
- if: needs.scope.outputs.player == 'true'
run: pnpm --filter @missing-elements/h5p-offline-player test:engine
- run: pnpm --filter @missing-elements/h5p-demo test

# The VS Code extension in a real VS Code, on a display Xvfb gives it, and the .vsix the owner
# publishes, built from what the tests ran against. The unit tests run in the job above.
vscode:
name: VS Code extension
needs: scope
if: needs.scope.outputs.vscode == 'true'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v5
Expand Down
9 changes: 7 additions & 2 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,13 @@ pnpm build

`pnpm test:browser` exercises the player in Chromium, including its Service
Worker and Jobs worker. Run it for changes to playback, storage, worker,
network, or browser-facing behavior. CI also runs the suite, and the demo's
tests, in Firefox and WebKit; for a change to the Service Worker, the Jobs
network, or browser-facing behavior. CI also runs the suite in Firefox and
WebKit when the player or the runtime changes, and the demo's tests there when
either of them or the demo changes. A pull request's run tests only the
packages it changes and those that depend on them, and leaves out three slow
browser files that main's run has; a change outside the packages, such as the
lockfile or a workflow, tests everything. For a change to the Service Worker,
the Jobs
worker, storage or the demo's pages, run them there too, one engine at a time,
after installing them with
`pnpm --filter @missing-elements/h5p-offline-player exec playwright install firefox webkit`:
Expand Down
53 changes: 28 additions & 25 deletions design/testing.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,31 +4,34 @@ How the repository is tested: the engines, CI's browsers, the owner's rule on wh

## Engines, CI and what is tested

The `browser` project runs in Chromium unless `H5P_TEST_BROWSER` names `firefox` or `webkit`, and
`pnpm test:engine` runs it alone, which is what CI does for those two (the `engines` job, one
engine per job; the `test` job covers Chromium). One engine per run, never a list: the dev
server's switches — a host's outage, the worker version it serves — are shared, and two engines
running one file at once flipped them under each other. `browser-emulated` stays on Chromium,
since CDP is Chromium's. Added on 2026-10-13, after Safari had only ever been checked by hand;
the first runs found the three engine differences under *A body that streams holds its fetch
event open*, *An untrusted package plays as data* (the worker's own script in its policy) and *A
downloaded archive is read back before it is reported done*. Playwright's WebKit is WebKit's
desktop port and not iOS Safari: the app's menu filled an iPhone's screen in Safari on 2026-10-13
and drew correctly in Playwright's WebKit at every phone size.

CI installs Chromium and Firefox without their system packages (#73, 2026-10-23): the runner image
ships both browsers, so the libraries they link against are there, and what `--with-deps` adds —
fonts, Xvfb, FFmpeg's codecs — a headless suite that decodes no video does not use. WebKit needs
its packages, GStreamer and GTK 4 among them, 117 MB from Ubuntu's mirror, and the `engines` job
keeps them in the Actions cache (#77): `/var/cache/apt/archives`, keyed by ISO week with any
earlier week as the fallback, and `apt-get autoclean` after the install so one version of each
package stays. Measured that morning: `azure.archive.ubuntu.com` served the package lists at
12 MB/s and the packages at 40 kB/s — 115 MB in 47 minutes, in three runs of three, against 30 to
60 seconds in the eight `main` runs before. The image's apt failover (actions/runner-images#14594)
fires on a stall, not on a trickle, which is actions/runner-images#14823, open; the cache restores
from GitHub's storage instead, 111 MB in 3 seconds, and the install then took 26 seconds. A cache
saved by a pull request's run is that pull request's: `main`'s first run after the merge fills the
one every later pull request reads.
**CI tests what a change reaches.** A pull request's run tests the packages it changes and those
that depend on them, as pnpm's `...[<commit>]` filter reads the change. A change under `skills/`
adds the author package and `apps/skills`. Any other file outside the packages (the lockfile, a
workflow, the root's `package.json`) tests everything, unless it is one of the docs below. Every
package that has a `typecheck` script runs it whatever the change, since each package's check
writes the types the next one reads. A push to main tests what it changed the same way. The
weekly run and a run by hand test everything, and a change to `design/`, `docs/`, a root `.md` or
`LICENSE` alone starts no run. The `scope` job in `ci.yml` decides.

**A pull request's run leaves out the player's browser files that wait out real time.** These are
`stalling-host`, `worker-update` and `large-media`, named in the player's `vite.config.ts` and
dropped under `H5P_TEST_SKIP_SLOW`: about two minutes of test time in each engine. Main's run after
the merge has them, so wait for it before publishing the player.

The `browser` project runs in Chromium unless `H5P_TEST_BROWSER` names `firefox` or `webkit`.
`pnpm test:engine` runs it, then the app's tests. The `engines` job runs in those two engines when
the player, the runtime or the demo changed: the app's tests every time, and the player's suite only
when the player or the runtime changed. One engine per run, never a list: the dev server's switches
(a host's outage, the worker version it serves) are shared, and two engines running one file at once
flipped them under each other. `browser-emulated` stays on Chromium, since CDP is Chromium's.
Playwright's WebKit is WebKit's desktop port, not iOS Safari.

CI installs Chromium and Firefox without their system packages. The runner image has what they
link against, and `--with-deps` adds fonts, Xvfb and codecs that a headless suite does not use,
from a mirror some runners reach at 40 kB/s (#73). WebKit needs its 117 MB of packages, which the
`engines` job keeps in the Actions cache, keyed by ISO week, running `apt-get autoclean` after the
install (#77). A pull request's cache is its own; main's runs, the weekly one among them, fill the
one pull requests read.

**What is tested, by the owner's rule of 2026-10-17: the packages and their source, and of the
demo app only the installable app.** The demo's pages — the player page, the examples, `/embed`,
Expand Down
9 changes: 9 additions & 0 deletions packages/player/vite.config.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
/// <reference types="vitest/config" />
import { resolve } from 'node:path'
import { defineConfig } from 'vite'
import { configDefaults } from 'vitest/config'
import { playwright } from '@vitest/browser-playwright'
import { devServiceWorkerPlugin, jobsWorkerPlugin, noRangeFixturesPlugin } from './vite.plugins'

Expand All @@ -9,6 +10,13 @@ import { devServiceWorkerPlugin, jobsWorkerPlugin, noRangeFixturesPlugin } from
* app (`apps/demo/vite.config.ts`), which is the dev server, needs the same ones.
*/

/**
* The browser files that wait out real time: a host silent past the stall bound, a worker
* update's deadlines, 20 MB archives. `H5P_TEST_SKIP_SLOW` leaves them out, as a pull request's
* run does; a run on main has them.
*/
const SLOW = ['tests/browser/stalling-host.test.ts', 'tests/browser/worker-update.test.ts', 'tests/browser/large-media.test.ts']

export default defineConfig({
plugins: [jobsWorkerPlugin(), devServiceWorkerPlugin(), noRangeFixturesPlugin()],

Expand Down Expand Up @@ -42,6 +50,7 @@ export default defineConfig({
test: {
name: 'browser',
include: ['tests/browser/**/*.test.ts'],
exclude: process.env.H5P_TEST_SKIP_SLOW ? [...configDefaults.exclude, ...SLOW] : configDefaults.exclude,
browser: browserProject(engine())
}
},
Expand Down
Loading