diff --git a/.github/workflows/libraries.yml b/.github/workflows/libraries.yml index b4760980..a7650c10 100644 --- a/.github/workflows/libraries.yml +++ b/.github/workflows/libraries.yml @@ -51,6 +51,20 @@ jobs: git status --short -- packages/libraries | head -20 exit 1 fi + # The same for one part: a part whose library did not change is written as it was, so a + # new hash would make every browser download it again and give every package played + # with it a new revision, for no library's sake (#116). + stray="" + for part in $(git diff --name-only -- packages/libraries/parts | sed -n 's|^packages/libraries/parts/\(.*\)\.h5p$|\1|p'); do + case "$changes" in + *"+$part "* | *"-$part "*) ;; + *) stray="$stray $part" ;; + esac + done + if [ -n "$stray" ]; then + echo "::error::These parts changed, but their libraries in libraries.txt did not:$stray. Compare the archives' entries before releasing anything." + exit 1 + fi version=$(node -e " const fs = require('fs'), path = 'packages/libraries/package.json' diff --git a/AGENTS.md b/AGENTS.md index 5a06b368..f6ef32f8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -154,9 +154,10 @@ packages/libraries/ @missing-elements/h5p-libraries — the hub's runtime SHA-256; index.js, the `pack` export a bundler reads, a static `new URL` per part, and index.d.ts; libraries.h5p, the whole pack as one archive; libraries.txt, the licences. All committed, built by apps/demo/scripts/build-libraries.mjs (`pnpm - libraries`), the parts split from the pack by scripts/parts.mjs (`splitPack`, also - `pnpm parts` there; the player's fixture builder uses it too). The demo pages import - `pack`; jsDelivr serves the package versioned with CORS + libraries`), the pack written by scripts/parts.mjs (`writePack`, which keeps an + unchanged library's entries from the pack before, #116) and split into parts by + it (`splitPack`, also `pnpm parts` there; the player's fixture builder uses it + too). The demo pages import `pack`; jsDelivr serves the package versioned with CORS packages/normalize/ @missing-elements/h5p-normalize — the package rewriter, published as a command bin/h5p-normalize.mjs the command @@ -1211,7 +1212,13 @@ element, and the element acts. That relay is why `frame-document.ts` has a `mess first refresh after the split rewrote all 106 archives with no library changed (#48, closed unmerged; #51). `tests/unit/pack-bytes.test.ts` in the player builds in four zones, and the workflow stops with an error when the files change and no library's line in `libraries.txt` - does. Measured on the demo's packages: 2 to 16 parts, 0.04 to 1.5 MB, 4.0 MB for formulas + does. A library whose files are unchanged keeps its entries from the committed pack, order + and compressed bytes included (`writePack`, #116): the build takes each library from the first + hub bundle that carries its newest version, in that bundle's order, and the refresh of + 2026-11-02 (#111) rewrote 30 parts with no library changed, because the hub had rebuilt Branching + Scenario's bundle, the first to carry them, with their files in another order. The workflow + also stops when a part changes and its library's line does not, and the test holds both cases. + Measured on the demo's packages: 2 to 16 parts, 0.04 to 1.5 MB, 4.0 MB for formulas (MathDisplay is 2.8 MB), against 14 MB whole. **Planning is the worker's** (`planFromPack` in `mount.ts`, the rules in `sw/pack-plan.ts`): an `index` that offers the pack (`pack` on the message: `{ pinned: true }` for `LIBRARY_PACK_BASE`, whose index the worker carries compiled in diff --git a/apps/demo/scripts/build-libraries.mjs b/apps/demo/scripts/build-libraries.mjs index 61b741c3..43d0fc85 100644 --- a/apps/demo/scripts/build-libraries.mjs +++ b/apps/demo/scripts/build-libraries.mjs @@ -1,15 +1,15 @@ import { openAsBlob } from 'node:fs' -import { writeFile } from 'node:fs/promises' +import { readFile, writeFile } from 'node:fs/promises' import { join, resolve } from 'node:path' -import { BlobReader, BlobWriter, TextReader, Uint8ArrayReader, Uint8ArrayWriter, ZipReader, ZipWriter, configure } from '@zip.js/zip.js' +import { BlobReader, Uint8ArrayWriter, ZipReader, configure } from '@zip.js/zip.js' import { formatBytes } from '@missing-elements/h5p-normalize' -import { FIXED_DATE, splitPack, writePackModule } from '../../../packages/libraries/scripts/parts.mjs' +import { splitPack, writePack, writePackModule } from '../../../packages/libraries/scripts/parts.mjs' import { HUB, download as cachedDownload, hubBundle as cachedBundle } from './lib/hub.mjs' /** * Builds the library pack, `@missing-elements/h5p-libraries`: `packages/libraries/libraries.h5p`, * the same split into one archive per library under `parts/` with its `index.json` and the - * `index.js` a bundler reads (`splitPack` and `writePackModule` in that package's + * `index.js` a bundler reads (`writePack`, `splitPack` and `writePackModule` in that package's * `scripts/parts.mjs`), and the licence notice, `packages/libraries/libraries.txt`. Exports from * h5p.com and h5p.org carry no libraries, and the player fetches the parts such a package needs; * it also plays the pack's newer version of a library a package carries. @@ -157,22 +157,12 @@ if (unknown.length > 0) { process.exit(1) } -// A bundle registered to supply libraries is never held to its own manifest, but the player -// refuses any archive without an h5p.json. -// A fixed date, the same in any time zone (see `FIXED_DATE`), so the pack's bytes change only when -// a library does: the weekly refresh (.github/workflows/libraries.yml) opens a pull request only -// on a real difference. -const writer = new ZipWriter(new BlobWriter('application/zip'), { lastModDate: FIXED_DATE, extendedTimestamp: false }) -await writer.add( - 'h5p.json', - new TextReader( - JSON.stringify({ title: 'H5P Offline Player library pack', language: 'und', mainLibrary: '', preloadedDependencies: [], embedTypes: ['div'] }) - ) -) -for (const { files } of libraries) { - for (const [name, bytes] of files) await writer.add(name, new Uint8ArrayReader(bytes)) -} -const pack = new Uint8Array(await (await writer.close()).arrayBuffer()) +// Written with a fixed date, the same in any time zone (see `FIXED_DATE`), and with a library's +// entries kept from the committed pack while its files are the same, so the pack's bytes change +// only where a library does: the weekly refresh (.github/workflows/libraries.yml) opens a pull +// request only on a real difference, and commits only the parts of the libraries that changed. +const previous = await readFile(join(outDir, 'libraries.h5p')).then((bytes) => new Uint8Array(bytes), () => undefined) +const pack = await writePack(libraries, previous) await writeFile(join(outDir, 'libraries.h5p'), pack) const parts = await splitPack(pack, join(outDir, 'parts')) await writePackModule(parts, outDir) diff --git a/packages/libraries/scripts/parts.mjs b/packages/libraries/scripts/parts.mjs index 027d1781..c7c076c0 100644 --- a/packages/libraries/scripts/parts.mjs +++ b/packages/libraries/scripts/parts.mjs @@ -15,9 +15,10 @@ import { BlobWriter, TextReader, Uint8ArrayReader, Uint8ArrayWriter, ZipReader, * static `new URL` for every part, the one shape Vite, Rollup and webpack 5 emit as an asset. * * Each part copies its entries from the pack as they are, compressed bytes and dates included, - * so the parts change exactly when the pack does and the weekly refresh commits only the + * so the parts change exactly when the pack does, and the pack keeps a library's entries from the + * pack before it when its files are the same (`writePack`), so the weekly refresh commits only the * libraries that changed. `pnpm parts` splits the committed `libraries.h5p`; - * `apps/demo/scripts/build-libraries.mjs` calls `splitPack` after writing a new one. + * `apps/demo/scripts/build-libraries.mjs` writes a new one with `writePack` and splits it. */ configure({ useWebWorkers: false }) @@ -25,10 +26,10 @@ configure({ useWebWorkers: false }) const packageDir = resolve(import.meta.dirname, '..') /** - * The `h5p.json` every part carries. The player refuses an archive without one, and a part, - * registered to supply libraries, is never held to what it declares. + * The `h5p.json` the pack and every part carry. The player refuses an archive without one, and an + * archive registered to supply libraries is never held to what it declares. */ -const PART_MANIFEST = JSON.stringify({ title: 'H5P Offline Player library pack', language: 'und', mainLibrary: '', preloadedDependencies: [], embedTypes: ['div'] }) +const PART_MANIFEST =JSON.stringify({ title: 'H5P Offline Player library pack', language: 'und', mainLibrary: '', preloadedDependencies: [], embedTypes: ['div'] }) /** * The date every entry of the pack and its parts is written with, so the bytes change only when a @@ -48,6 +49,56 @@ export const FIXED_DATE = new Date(2000, 0, 1, 2) */ const MIN_PART_SIZE = 8 * 1024 + 1 +/** + * Writes the pack: its `h5p.json`, then each library's files, in the order given. A library whose + * files are those its folder holds in `previous`, the pack this one replaces, keeps that pack's + * entries as they are, their order and compressed bytes included, so its part comes out as before. + * The build takes each library from the first hub bundle that carries its newest version, with + * its files in that bundle's order, and the hub lists them in an order of its own: the refresh of + * 2026-11-02 rewrote 30 parts whose libraries had not changed, because the new bundle of Branching + * Scenario, the first to carry them, listed their files differently (#116). + * @param {{ folder: string, files: Map }[]} libraries each folder's files, by their names in the pack + * @param {Uint8Array} [previous] the pack this one replaces + */ +export async function writePack(libraries, previous) { + const reader = previous ? new ZipReader(new Uint8ArrayReader(previous)) : null + const before = new Map() + for (const entry of reader ? await reader.getEntries() : []) { + const slash = entry.filename.indexOf('/') + if (entry.directory || slash < 0) continue + const folder = entry.filename.slice(0, slash) + if (!before.has(folder)) before.set(folder, []) + before.get(folder).push(entry) + } + + const writer = new ZipWriter(new BlobWriter('application/zip'), { lastModDate: FIXED_DATE, extendedTimestamp: false }) + await writer.add('h5p.json', new TextReader(PART_MANIFEST)) + for (const { folder, files } of libraries) { + const kept = before.get(folder) + if (kept && (await holds(kept, files))) { + for (const entry of kept) { + const raw = await entry.getData(new Uint8ArrayWriter(), { passThrough: true }) + await writer.add(entry.filename, new Uint8ArrayReader(raw), { passThrough: true, entry }) + } + } else { + for (const [name, bytes] of files) await writer.add(name, new Uint8ArrayReader(bytes)) + } + } + await reader?.close() + return new Uint8Array(await (await writer.close()).arrayBuffer()) +} + +/** Whether a folder's entries in a pack hold exactly these files, by name and bytes. */ +async function holds(entries, files) { + if (entries.length !== files.size) return false + for (const entry of entries) { + const bytes = files.get(entry.filename) + if (!bytes || bytes.length !== entry.uncompressedSize) return false + if (Buffer.compare(await entry.getData(new Uint8ArrayWriter()), bytes) !== 0) return false + } + return true +} + /** * Writes `/.h5p` for every library folder in the pack and `/index.json`, * replacing whatever `outDir` held. Returns the index's entries. diff --git a/packages/player/tests/unit/pack-bytes.test.ts b/packages/player/tests/unit/pack-bytes.test.ts index a3d89532..c1de6948 100644 --- a/packages/player/tests/unit/pack-bytes.test.ts +++ b/packages/player/tests/unit/pack-bytes.test.ts @@ -1,13 +1,18 @@ import { execFileSync } from 'node:child_process' -import { resolve } from 'node:path' +import { mkdtemp } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' import { pathToFileURL } from 'node:url' import { describe, expect, it } from 'vitest' +import { splitPack, writePack } from '../../../libraries/scripts/parts.mjs' /** * The library pack's bytes, which the player pins by hash, depend on nothing but its libraries. A * zip records a wall-clock time with no zone, and the pack was once written with a date built * from UTC fields: the pack built at UTC+2 said 02:00 and CI's said 00:00, so the first weekly - * refresh after the split rewrote every archive with no library changed (#48, #51). + * refresh after the split rewrote every archive with no library changed (#48, #51). Nor do they + * depend on the order a hub bundle lists a library's files in, which changed when Branching + * Scenario's bundle was rebuilt and rewrote the 30 parts it is the first to carry (#116). * * Each time zone runs in a process of its own: whether setting `TZ` inside a running test worker * moves its clock is the engine's business, and a child process starts in the zone it is given. @@ -16,22 +21,24 @@ import { describe, expect, it } from 'vitest' const librariesDir = resolve(import.meta.dirname, '../../../libraries') const parts = pathToFileURL(resolve(librariesDir, 'scripts/parts.mjs')).href -// A pack of one library, written the way apps/demo/scripts/build-libraries.mjs writes the real -// one, split by the real `splitPack`; prints the bytes' hashes and the first entry's DOS time and date. +// A pack of one library, written by the `writePack` that apps/demo/scripts/build-libraries.mjs +// writes the real one with, split by the real `splitPack`; prints the bytes' hashes and the first +// entry's DOS time and date. const build = ` import { createHash } from 'node:crypto' import { mkdtemp, readFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' -import { BlobWriter, TextReader, ZipWriter, configure } from '@zip.js/zip.js' -import { FIXED_DATE, splitPack } from ${JSON.stringify(parts)} +import { splitPack, writePack } from ${JSON.stringify(parts)} -configure({ useWebWorkers: false }) -const writer = new ZipWriter(new BlobWriter('application/zip'), { lastModDate: FIXED_DATE, extendedTimestamp: false }) -await writer.add('h5p.json', new TextReader('{}')) -await writer.add('H5P.Tiny-1.0/library.json', new TextReader(JSON.stringify({ machineName: 'H5P.Tiny', majorVersion: 1, minorVersion: 0, patchVersion: 3 }))) -await writer.add('H5P.Tiny-1.0/tiny.js', new TextReader('H5P.Tiny = function () {}')) -const pack = new Uint8Array(await (await writer.close()).arrayBuffer()) +const text = (value) => new TextEncoder().encode(value) +const pack = await writePack([{ + folder: 'H5P.Tiny-1.0', + files: new Map([ + ['H5P.Tiny-1.0/library.json', text(JSON.stringify({ machineName: 'H5P.Tiny', majorVersion: 1, minorVersion: 0, patchVersion: 3 }))], + ['H5P.Tiny-1.0/tiny.js', text('H5P.Tiny = function () {}')] + ]) +}]) const out = await mkdtemp(join(tmpdir(), 'pack-bytes-')) const [{ sha256 }] = await splitPack(pack, out) @@ -68,4 +75,45 @@ describe('the library pack', () => { expect(time).toBe(2 << 11) expect(date).toBe(((2000 - 1980) << 9) | (1 << 5) | 1) }, 30_000) + + // Two libraries' files, as a hub bundle lists them, and the same listed the other way round. + const text = (value: string) => new TextEncoder().encode(value) + const tiny = (patchVersion: number, script: string) => ({ + folder: 'H5P.Tiny-1.0', + files: new Map([ + ['H5P.Tiny-1.0/library.json', text(JSON.stringify({ machineName: 'H5P.Tiny', majorVersion: 1, minorVersion: 0, patchVersion }))], + ['H5P.Tiny-1.0/tiny.js', text(script)], + ['H5P.Tiny-1.0/tiny.css', text('.h5p-tiny { color: teal }')] + ]) + }) + const other = () => ({ + folder: 'H5P.Other-1.0', + files: new Map([ + ['H5P.Other-1.0/library.json', text(JSON.stringify({ machineName: 'H5P.Other', majorVersion: 1, minorVersion: 0, patchVersion: 7 }))], + ['H5P.Other-1.0/other.js', text('H5P.Other = function () {}')], + ['H5P.Other-1.0/language/de.json', text('{"semantics":[]}')] + ]) + }) + const reversed = ({ folder, files }: { folder: string; files: Map }) => ({ folder, files: new Map([...files].reverse()) }) + /** Each part's SHA-256, by folder. */ + async function partsOf(pack: Uint8Array) { + const index = await splitPack(pack, await mkdtemp(join(tmpdir(), 'pack-parts-'))) + return Object.fromEntries(index.map(({ folder, sha256 }: { folder: string; sha256: string }) => [folder, sha256])) + } + + it('keeps the entries of a library whose files are the same, in whatever order a bundle lists them', async () => { + const before = await writePack([other(), tiny(1, 'H5P.Tiny = function () {}')]) + const again = [reversed(other()), reversed(tiny(1, 'H5P.Tiny = function () {}'))] + expect(Buffer.compare(await writePack(again, before), before)).toBe(0) + // Without the pack before, the bundle's order is the order written: the refresh of #116. + expect(Buffer.compare(await writePack(again), before)).not.toBe(0) + }) + + it("writes again only the library that changed, so only its part's hash moves", async () => { + const before = await writePack([other(), tiny(1, 'H5P.Tiny = function () {}')]) + const after = await writePack([reversed(other()), reversed(tiny(2, 'H5P.Tiny = function (params) {}'))], before) + const [partsBefore, partsAfter] = [await partsOf(before), await partsOf(after)] + expect(partsAfter['H5P.Other-1.0']).toBe(partsBefore['H5P.Other-1.0']) + expect(partsAfter['H5P.Tiny-1.0']).not.toBe(partsBefore['H5P.Tiny-1.0']) + }) })