Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions .github/workflows/libraries.yml
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,20 @@ jobs:
git status --short -- packages/libraries | head -20
exit 1
fi
# The same for one part: a part whose library did not change is written as it was, so a
# new hash would make every browser download it again and give every package played
# with it a new revision, for no library's sake (#116).
stray=""
for part in $(git diff --name-only -- packages/libraries/parts | sed -n 's|^packages/libraries/parts/\(.*\)\.h5p$|\1|p'); do
case "$changes" in
*"+$part "* | *"-$part "*) ;;
*) stray="$stray $part" ;;
esac
done
if [ -n "$stray" ]; then
echo "::error::These parts changed, but their libraries in libraries.txt did not:$stray. Compare the archives' entries before releasing anything."
exit 1
fi

version=$(node -e "
const fs = require('fs'), path = 'packages/libraries/package.json'
Expand Down
15 changes: 11 additions & 4 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -154,9 +154,10 @@ packages/libraries/ @missing-elements/h5p-libraries — the hub's runtime
SHA-256; index.js, the `pack` export a bundler reads, a static `new URL` per part,
and index.d.ts; libraries.h5p, the whole pack as one archive; libraries.txt, the
licences. All committed, built by apps/demo/scripts/build-libraries.mjs (`pnpm
libraries`), the parts split from the pack by scripts/parts.mjs (`splitPack`, also
`pnpm parts` there; the player's fixture builder uses it too). The demo pages import
`pack`; jsDelivr serves the package versioned with CORS
libraries`), the pack written by scripts/parts.mjs (`writePack`, which keeps an
unchanged library's entries from the pack before, #116) and split into parts by
it (`splitPack`, also `pnpm parts` there; the player's fixture builder uses it
too). The demo pages import `pack`; jsDelivr serves the package versioned with CORS

packages/normalize/ @missing-elements/h5p-normalize — the package rewriter, published as a command
bin/h5p-normalize.mjs the command
Expand Down Expand Up @@ -1211,7 +1212,13 @@ element, and the element acts. That relay is why `frame-document.ts` has a `mess
first refresh after the split rewrote all 106 archives with no library changed (#48, closed
unmerged; #51). `tests/unit/pack-bytes.test.ts` in the player builds in four zones, and the
workflow stops with an error when the files change and no library's line in `libraries.txt`
does. Measured on the demo's packages: 2 to 16 parts, 0.04 to 1.5 MB, 4.0 MB for formulas
does. A library whose files are unchanged keeps its entries from the committed pack, order
and compressed bytes included (`writePack`, #116): the build takes each library from the first
hub bundle that carries its newest version, in that bundle's order, and the refresh of
2026-11-02 (#111) rewrote 30 parts with no library changed, because the hub had rebuilt Branching
Scenario's bundle, the first to carry them, with their files in another order. The workflow
also stops when a part changes and its library's line does not, and the test holds both cases.
Measured on the demo's packages: 2 to 16 parts, 0.04 to 1.5 MB, 4.0 MB for formulas
(MathDisplay is 2.8 MB), against 14 MB whole. **Planning is the worker's** (`planFromPack` in
`mount.ts`, the rules in `sw/pack-plan.ts`): an `index` that offers the pack (`pack` on the
message: `{ pinned: true }` for `LIBRARY_PACK_BASE`, whose index the worker carries compiled in
Expand Down
30 changes: 10 additions & 20 deletions apps/demo/scripts/build-libraries.mjs
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
import { openAsBlob } from 'node:fs'
import { writeFile } from 'node:fs/promises'
import { readFile, writeFile } from 'node:fs/promises'
import { join, resolve } from 'node:path'
import { BlobReader, BlobWriter, TextReader, Uint8ArrayReader, Uint8ArrayWriter, ZipReader, ZipWriter, configure } from '@zip.js/zip.js'
import { BlobReader, Uint8ArrayWriter, ZipReader, configure } from '@zip.js/zip.js'
import { formatBytes } from '@missing-elements/h5p-normalize'
import { FIXED_DATE, splitPack, writePackModule } from '../../../packages/libraries/scripts/parts.mjs'
import { splitPack, writePack, writePackModule } from '../../../packages/libraries/scripts/parts.mjs'
import { HUB, download as cachedDownload, hubBundle as cachedBundle } from './lib/hub.mjs'

/**
* Builds the library pack, `@missing-elements/h5p-libraries`: `packages/libraries/libraries.h5p`,
* the same split into one archive per library under `parts/` with its `index.json` and the
* `index.js` a bundler reads (`splitPack` and `writePackModule` in that package's
* `index.js` a bundler reads (`writePack`, `splitPack` and `writePackModule` in that package's
* `scripts/parts.mjs`), and the licence notice, `packages/libraries/libraries.txt`. Exports from
* h5p.com and h5p.org carry no libraries, and the player fetches the parts such a package needs;
* it also plays the pack's newer version of a library a package carries.
Expand Down Expand Up @@ -157,22 +157,12 @@ if (unknown.length > 0) {
process.exit(1)
}

// A bundle registered to supply libraries is never held to its own manifest, but the player
// refuses any archive without an h5p.json.
// A fixed date, the same in any time zone (see `FIXED_DATE`), so the pack's bytes change only when
// a library does: the weekly refresh (.github/workflows/libraries.yml) opens a pull request only
// on a real difference.
const writer = new ZipWriter(new BlobWriter('application/zip'), { lastModDate: FIXED_DATE, extendedTimestamp: false })
await writer.add(
'h5p.json',
new TextReader(
JSON.stringify({ title: 'H5P Offline Player library pack', language: 'und', mainLibrary: '', preloadedDependencies: [], embedTypes: ['div'] })
)
)
for (const { files } of libraries) {
for (const [name, bytes] of files) await writer.add(name, new Uint8ArrayReader(bytes))
}
const pack = new Uint8Array(await (await writer.close()).arrayBuffer())
// Written with a fixed date, the same in any time zone (see `FIXED_DATE`), and with a library's
// entries kept from the committed pack while its files are the same, so the pack's bytes change
// only where a library does: the weekly refresh (.github/workflows/libraries.yml) opens a pull
// request only on a real difference, and commits only the parts of the libraries that changed.
const previous = await readFile(join(outDir, 'libraries.h5p')).then((bytes) => new Uint8Array(bytes), () => undefined)
const pack = await writePack(libraries, previous)
await writeFile(join(outDir, 'libraries.h5p'), pack)
const parts = await splitPack(pack, join(outDir, 'parts'))
await writePackModule(parts, outDir)
Expand Down
61 changes: 56 additions & 5 deletions packages/libraries/scripts/parts.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -15,20 +15,21 @@ import { BlobWriter, TextReader, Uint8ArrayReader, Uint8ArrayWriter, ZipReader,
* static `new URL` for every part, the one shape Vite, Rollup and webpack 5 emit as an asset.
*
* Each part copies its entries from the pack as they are, compressed bytes and dates included,
* so the parts change exactly when the pack does and the weekly refresh commits only the
* so the parts change exactly when the pack does, and the pack keeps a library's entries from the
* pack before it when its files are the same (`writePack`), so the weekly refresh commits only the
* libraries that changed. `pnpm parts` splits the committed `libraries.h5p`;
* `apps/demo/scripts/build-libraries.mjs` calls `splitPack` after writing a new one.
* `apps/demo/scripts/build-libraries.mjs` writes a new one with `writePack` and splits it.
*/

configure({ useWebWorkers: false })

const packageDir = resolve(import.meta.dirname, '..')

/**
* The `h5p.json` every part carries. The player refuses an archive without one, and a part,
* registered to supply libraries, is never held to what it declares.
* The `h5p.json` the pack and every part carry. The player refuses an archive without one, and an
* archive registered to supply libraries is never held to what it declares.
*/
const PART_MANIFEST = JSON.stringify({ title: 'H5P Offline Player library pack', language: 'und', mainLibrary: '', preloadedDependencies: [], embedTypes: ['div'] })
const PART_MANIFEST =JSON.stringify({ title: 'H5P Offline Player library pack', language: 'und', mainLibrary: '', preloadedDependencies: [], embedTypes: ['div'] })

/**
* The date every entry of the pack and its parts is written with, so the bytes change only when a
Expand All @@ -48,6 +49,56 @@ export const FIXED_DATE = new Date(2000, 0, 1, 2)
*/
const MIN_PART_SIZE = 8 * 1024 + 1

/**
* Writes the pack: its `h5p.json`, then each library's files, in the order given. A library whose
* files are those its folder holds in `previous`, the pack this one replaces, keeps that pack's
* entries as they are, their order and compressed bytes included, so its part comes out as before.
* The build takes each library from the first hub bundle that carries its newest version, with
* its files in that bundle's order, and the hub lists them in an order of its own: the refresh of
* 2026-11-02 rewrote 30 parts whose libraries had not changed, because the new bundle of Branching
* Scenario, the first to carry them, listed their files differently (#116).
* @param {{ folder: string, files: Map<string, Uint8Array> }[]} libraries each folder's files, by their names in the pack
* @param {Uint8Array} [previous] the pack this one replaces
*/
export async function writePack(libraries, previous) {
const reader = previous ? new ZipReader(new Uint8ArrayReader(previous)) : null
const before = new Map()
for (const entry of reader ? await reader.getEntries() : []) {
const slash = entry.filename.indexOf('/')
if (entry.directory || slash < 0) continue
const folder = entry.filename.slice(0, slash)
if (!before.has(folder)) before.set(folder, [])
before.get(folder).push(entry)
}

const writer = new ZipWriter(new BlobWriter('application/zip'), { lastModDate: FIXED_DATE, extendedTimestamp: false })
await writer.add('h5p.json', new TextReader(PART_MANIFEST))
for (const { folder, files } of libraries) {
const kept = before.get(folder)
if (kept && (await holds(kept, files))) {
for (const entry of kept) {
const raw = await entry.getData(new Uint8ArrayWriter(), { passThrough: true })
await writer.add(entry.filename, new Uint8ArrayReader(raw), { passThrough: true, entry })
}
} else {
for (const [name, bytes] of files) await writer.add(name, new Uint8ArrayReader(bytes))
}
}
await reader?.close()
return new Uint8Array(await (await writer.close()).arrayBuffer())
}

/** Whether a folder's entries in a pack hold exactly these files, by name and bytes. */
async function holds(entries, files) {
if (entries.length !== files.size) return false
for (const entry of entries) {
const bytes = files.get(entry.filename)
if (!bytes || bytes.length !== entry.uncompressedSize) return false
if (Buffer.compare(await entry.getData(new Uint8ArrayWriter()), bytes) !== 0) return false
}
return true
}

/**
* Writes `<outDir>/<folder>.h5p` for every library folder in the pack and `<outDir>/index.json`,
* replacing whatever `outDir` held. Returns the index's entries.
Expand Down
72 changes: 60 additions & 12 deletions packages/player/tests/unit/pack-bytes.test.ts
Original file line number Diff line number Diff line change
@@ -1,13 +1,18 @@
import { execFileSync } from 'node:child_process'
import { resolve } from 'node:path'
import { mkdtemp } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join, resolve } from 'node:path'
import { pathToFileURL } from 'node:url'
import { describe, expect, it } from 'vitest'
import { splitPack, writePack } from '../../../libraries/scripts/parts.mjs'

/**
* The library pack's bytes, which the player pins by hash, depend on nothing but its libraries. A
* zip records a wall-clock time with no zone, and the pack was once written with a date built
* from UTC fields: the pack built at UTC+2 said 02:00 and CI's said 00:00, so the first weekly
* refresh after the split rewrote every archive with no library changed (#48, #51).
* refresh after the split rewrote every archive with no library changed (#48, #51). Nor do they
* depend on the order a hub bundle lists a library's files in, which changed when Branching
* Scenario's bundle was rebuilt and rewrote the 30 parts it is the first to carry (#116).
*
* Each time zone runs in a process of its own: whether setting `TZ` inside a running test worker
* moves its clock is the engine's business, and a child process starts in the zone it is given.
Expand All @@ -16,22 +21,24 @@ import { describe, expect, it } from 'vitest'
const librariesDir = resolve(import.meta.dirname, '../../../libraries')
const parts = pathToFileURL(resolve(librariesDir, 'scripts/parts.mjs')).href

// A pack of one library, written the way apps/demo/scripts/build-libraries.mjs writes the real
// one, split by the real `splitPack`; prints the bytes' hashes and the first entry's DOS time and date.
// A pack of one library, written by the `writePack` that apps/demo/scripts/build-libraries.mjs
// writes the real one with, split by the real `splitPack`; prints the bytes' hashes and the first
// entry's DOS time and date.
const build = `
import { createHash } from 'node:crypto'
import { mkdtemp, readFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { BlobWriter, TextReader, ZipWriter, configure } from '@zip.js/zip.js'
import { FIXED_DATE, splitPack } from ${JSON.stringify(parts)}
import { splitPack, writePack } from ${JSON.stringify(parts)}

configure({ useWebWorkers: false })
const writer = new ZipWriter(new BlobWriter('application/zip'), { lastModDate: FIXED_DATE, extendedTimestamp: false })
await writer.add('h5p.json', new TextReader('{}'))
await writer.add('H5P.Tiny-1.0/library.json', new TextReader(JSON.stringify({ machineName: 'H5P.Tiny', majorVersion: 1, minorVersion: 0, patchVersion: 3 })))
await writer.add('H5P.Tiny-1.0/tiny.js', new TextReader('H5P.Tiny = function () {}'))
const pack = new Uint8Array(await (await writer.close()).arrayBuffer())
const text = (value) => new TextEncoder().encode(value)
const pack = await writePack([{
folder: 'H5P.Tiny-1.0',
files: new Map([
['H5P.Tiny-1.0/library.json', text(JSON.stringify({ machineName: 'H5P.Tiny', majorVersion: 1, minorVersion: 0, patchVersion: 3 }))],
['H5P.Tiny-1.0/tiny.js', text('H5P.Tiny = function () {}')]
])
}])

const out = await mkdtemp(join(tmpdir(), 'pack-bytes-'))
const [{ sha256 }] = await splitPack(pack, out)
Expand Down Expand Up @@ -68,4 +75,45 @@ describe('the library pack', () => {
expect(time).toBe(2 << 11)
expect(date).toBe(((2000 - 1980) << 9) | (1 << 5) | 1)
}, 30_000)

// Two libraries' files, as a hub bundle lists them, and the same listed the other way round.
const text = (value: string) => new TextEncoder().encode(value)
const tiny = (patchVersion: number, script: string) => ({
folder: 'H5P.Tiny-1.0',
files: new Map([
['H5P.Tiny-1.0/library.json', text(JSON.stringify({ machineName: 'H5P.Tiny', majorVersion: 1, minorVersion: 0, patchVersion }))],
['H5P.Tiny-1.0/tiny.js', text(script)],
['H5P.Tiny-1.0/tiny.css', text('.h5p-tiny { color: teal }')]
])
})
const other = () => ({
folder: 'H5P.Other-1.0',
files: new Map([
['H5P.Other-1.0/library.json', text(JSON.stringify({ machineName: 'H5P.Other', majorVersion: 1, minorVersion: 0, patchVersion: 7 }))],
['H5P.Other-1.0/other.js', text('H5P.Other = function () {}')],
['H5P.Other-1.0/language/de.json', text('{"semantics":[]}')]
])
})
const reversed = ({ folder, files }: { folder: string; files: Map<string, Uint8Array> }) => ({ folder, files: new Map([...files].reverse()) })
/** Each part's SHA-256, by folder. */
async function partsOf(pack: Uint8Array) {
const index = await splitPack(pack, await mkdtemp(join(tmpdir(), 'pack-parts-')))
return Object.fromEntries(index.map(({ folder, sha256 }: { folder: string; sha256: string }) => [folder, sha256]))
}

it('keeps the entries of a library whose files are the same, in whatever order a bundle lists them', async () => {
const before = await writePack([other(), tiny(1, 'H5P.Tiny = function () {}')])
const again = [reversed(other()), reversed(tiny(1, 'H5P.Tiny = function () {}'))]
expect(Buffer.compare(await writePack(again, before), before)).toBe(0)
// Without the pack before, the bundle's order is the order written: the refresh of #116.
expect(Buffer.compare(await writePack(again), before)).not.toBe(0)
})

it("writes again only the library that changed, so only its part's hash moves", async () => {
const before = await writePack([other(), tiny(1, 'H5P.Tiny = function () {}')])
const after = await writePack([reversed(other()), reversed(tiny(2, 'H5P.Tiny = function (params) {}'))], before)
const [partsBefore, partsAfter] = [await partsOf(before), await partsOf(after)]
expect(partsAfter['H5P.Other-1.0']).toBe(partsBefore['H5P.Other-1.0'])
expect(partsAfter['H5P.Tiny-1.0']).not.toBe(partsBefore['H5P.Tiny-1.0'])
})
})
Loading