From 362a66e658afec7122c5e98282429f1946dfd9aa Mon Sep 17 00:00:00 2001 From: Aleksandr Shevchuk Date: Tue, 3 Nov 2026 01:47:07 +0200 Subject: [PATCH] fix(libraries): keep an unchanged library's entries from the pack before (#116) The refresh that opened #111 rewrote 30 parts whose libraries had not changed: the build takes each library from the first hub bundle that carries its newest version, in that bundle's order, and the hub had rebuilt Branching Scenario's bundle, the first to carry them, with their files in another order. A part's id is its hash and the libraries revision covers entry order, so every browser would download those parts again and every package played with them would get a new revision. writePack in packages/libraries/scripts/parts.mjs writes the pack, and a library whose files are byte for byte those its folder holds in the committed pack keeps that pack's entries as they are, as splitPack copies entries into the parts. build-libraries.mjs writes the pack with it. The library pack workflow also stops when a part changes and its library's line in libraries.txt does not. Rebuilt from today's hub, the pack changes the five updated libraries' parts alone, each byte for byte the part #111 has. The committed pack, rebuilt from its own files in reverse order, comes out byte for byte. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/libraries.yml | 14 ++++ AGENTS.md | 15 ++-- apps/demo/scripts/build-libraries.mjs | 30 +++----- packages/libraries/scripts/parts.mjs | 61 ++++++++++++++-- packages/player/tests/unit/pack-bytes.test.ts | 72 +++++++++++++++---- 5 files changed, 151 insertions(+), 41 deletions(-) diff --git a/.github/workflows/libraries.yml b/.github/workflows/libraries.yml index b4760980..a7650c10 100644 --- a/.github/workflows/libraries.yml +++ b/.github/workflows/libraries.yml @@ -51,6 +51,20 @@ jobs: git status --short -- packages/libraries | head -20 exit 1 fi + # The same for one part: a part whose library did not change is written as it was, so a + # new hash would make every browser download it again and give every package played + # with it a new revision, for no library's sake (#116). + stray="" + for part in $(git diff --name-only -- packages/libraries/parts | sed -n 's|^packages/libraries/parts/\(.*\)\.h5p$|\1|p'); do + case "$changes" in + *"+$part "* | *"-$part "*) ;; + *) stray="$stray $part" ;; + esac + done + if [ -n "$stray" ]; then + echo "::error::These parts changed, but their libraries in libraries.txt did not:$stray. Compare the archives' entries before releasing anything." + exit 1 + fi version=$(node -e " const fs = require('fs'), path = 'packages/libraries/package.json' diff --git a/AGENTS.md b/AGENTS.md index 5a06b368..f6ef32f8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -154,9 +154,10 @@ packages/libraries/ @missing-elements/h5p-libraries — the hub's runtime SHA-256; index.js, the `pack` export a bundler reads, a static `new URL` per part, and index.d.ts; libraries.h5p, the whole pack as one archive; libraries.txt, the licences. All committed, built by apps/demo/scripts/build-libraries.mjs (`pnpm - libraries`), the parts split from the pack by scripts/parts.mjs (`splitPack`, also - `pnpm parts` there; the player's fixture builder uses it too). The demo pages import - `pack`; jsDelivr serves the package versioned with CORS + libraries`), the pack written by scripts/parts.mjs (`writePack`, which keeps an + unchanged library's entries from the pack before, #116) and split into parts by + it (`splitPack`, also `pnpm parts` there; the player's fixture builder uses it + too). The demo pages import `pack`; jsDelivr serves the package versioned with CORS packages/normalize/ @missing-elements/h5p-normalize — the package rewriter, published as a command bin/h5p-normalize.mjs the command @@ -1211,7 +1212,13 @@ element, and the element acts. That relay is why `frame-document.ts` has a `mess first refresh after the split rewrote all 106 archives with no library changed (#48, closed unmerged; #51). `tests/unit/pack-bytes.test.ts` in the player builds in four zones, and the workflow stops with an error when the files change and no library's line in `libraries.txt` - does. Measured on the demo's packages: 2 to 16 parts, 0.04 to 1.5 MB, 4.0 MB for formulas + does. A library whose files are unchanged keeps its entries from the committed pack, order + and compressed bytes included (`writePack`, #116): the build takes each library from the first + hub bundle that carries its newest version, in that bundle's order, and the refresh of + 2026-11-02 (#111) rewrote 30 parts with no library changed, because the hub had rebuilt Branching + Scenario's bundle, the first to carry them, with their files in another order. The workflow + also stops when a part changes and its library's line does not, and the test holds both cases. + Measured on the demo's packages: 2 to 16 parts, 0.04 to 1.5 MB, 4.0 MB for formulas (MathDisplay is 2.8 MB), against 14 MB whole. **Planning is the worker's** (`planFromPack` in `mount.ts`, the rules in `sw/pack-plan.ts`): an `index` that offers the pack (`pack` on the message: `{ pinned: true }` for `LIBRARY_PACK_BASE`, whose index the worker carries compiled in diff --git a/apps/demo/scripts/build-libraries.mjs b/apps/demo/scripts/build-libraries.mjs index 61b741c3..43d0fc85 100644 --- a/apps/demo/scripts/build-libraries.mjs +++ b/apps/demo/scripts/build-libraries.mjs @@ -1,15 +1,15 @@ import { openAsBlob } from 'node:fs' -import { writeFile } from 'node:fs/promises' +import { readFile, writeFile } from 'node:fs/promises' import { join, resolve } from 'node:path' -import { BlobReader, BlobWriter, TextReader, Uint8ArrayReader, Uint8ArrayWriter, ZipReader, ZipWriter, configure } from '@zip.js/zip.js' +import { BlobReader, Uint8ArrayWriter, ZipReader, configure } from '@zip.js/zip.js' import { formatBytes } from '@missing-elements/h5p-normalize' -import { FIXED_DATE, splitPack, writePackModule } from '../../../packages/libraries/scripts/parts.mjs' +import { splitPack, writePack, writePackModule } from '../../../packages/libraries/scripts/parts.mjs' import { HUB, download as cachedDownload, hubBundle as cachedBundle } from './lib/hub.mjs' /** * Builds the library pack, `@missing-elements/h5p-libraries`: `packages/libraries/libraries.h5p`, * the same split into one archive per library under `parts/` with its `index.json` and the - * `index.js` a bundler reads (`splitPack` and `writePackModule` in that package's + * `index.js` a bundler reads (`writePack`, `splitPack` and `writePackModule` in that package's * `scripts/parts.mjs`), and the licence notice, `packages/libraries/libraries.txt`. Exports from * h5p.com and h5p.org carry no libraries, and the player fetches the parts such a package needs; * it also plays the pack's newer version of a library a package carries. @@ -157,22 +157,12 @@ if (unknown.length > 0) { process.exit(1) } -// A bundle registered to supply libraries is never held to its own manifest, but the player -// refuses any archive without an h5p.json. -// A fixed date, the same in any time zone (see `FIXED_DATE`), so the pack's bytes change only when -// a library does: the weekly refresh (.github/workflows/libraries.yml) opens a pull request only -// on a real difference. -const writer = new ZipWriter(new BlobWriter('application/zip'), { lastModDate: FIXED_DATE, extendedTimestamp: false }) -await writer.add( - 'h5p.json', - new TextReader( - JSON.stringify({ title: 'H5P Offline Player library pack', language: 'und', mainLibrary: '', preloadedDependencies: [], embedTypes: ['div'] }) - ) -) -for (const { files } of libraries) { - for (const [name, bytes] of files) await writer.add(name, new Uint8ArrayReader(bytes)) -} -const pack = new Uint8Array(await (await writer.close()).arrayBuffer()) +// Written with a fixed date, the same in any time zone (see `FIXED_DATE`), and with a library's +// entries kept from the committed pack while its files are the same, so the pack's bytes change +// only where a library does: the weekly refresh (.github/workflows/libraries.yml) opens a pull +// request only on a real difference, and commits only the parts of the libraries that changed. +const previous = await readFile(join(outDir, 'libraries.h5p')).then((bytes) => new Uint8Array(bytes), () => undefined) +const pack = await writePack(libraries, previous) await writeFile(join(outDir, 'libraries.h5p'), pack) const parts = await splitPack(pack, join(outDir, 'parts')) await writePackModule(parts, outDir) diff --git a/packages/libraries/scripts/parts.mjs b/packages/libraries/scripts/parts.mjs index 027d1781..c7c076c0 100644 --- a/packages/libraries/scripts/parts.mjs +++ b/packages/libraries/scripts/parts.mjs @@ -15,9 +15,10 @@ import { BlobWriter, TextReader, Uint8ArrayReader, Uint8ArrayWriter, ZipReader, * static `new URL` for every part, the one shape Vite, Rollup and webpack 5 emit as an asset. * * Each part copies its entries from the pack as they are, compressed bytes and dates included, - * so the parts change exactly when the pack does and the weekly refresh commits only the + * so the parts change exactly when the pack does, and the pack keeps a library's entries from the + * pack before it when its files are the same (`writePack`), so the weekly refresh commits only the * libraries that changed. `pnpm parts` splits the committed `libraries.h5p`; - * `apps/demo/scripts/build-libraries.mjs` calls `splitPack` after writing a new one. + * `apps/demo/scripts/build-libraries.mjs` writes a new one with `writePack` and splits it. */ configure({ useWebWorkers: false }) @@ -25,10 +26,10 @@ configure({ useWebWorkers: false }) const packageDir = resolve(import.meta.dirname, '..') /** - * The `h5p.json` every part carries. The player refuses an archive without one, and a part, - * registered to supply libraries, is never held to what it declares. + * The `h5p.json` the pack and every part carry. The player refuses an archive without one, and an + * archive registered to supply libraries is never held to what it declares. */ -const PART_MANIFEST = JSON.stringify({ title: 'H5P Offline Player library pack', language: 'und', mainLibrary: '', preloadedDependencies: [], embedTypes: ['div'] }) +const PART_MANIFEST =JSON.stringify({ title: 'H5P Offline Player library pack', language: 'und', mainLibrary: '', preloadedDependencies: [], embedTypes: ['div'] }) /** * The date every entry of the pack and its parts is written with, so the bytes change only when a @@ -48,6 +49,56 @@ export const FIXED_DATE = new Date(2000, 0, 1, 2) */ const MIN_PART_SIZE = 8 * 1024 + 1 +/** + * Writes the pack: its `h5p.json`, then each library's files, in the order given. A library whose + * files are those its folder holds in `previous`, the pack this one replaces, keeps that pack's + * entries as they are, their order and compressed bytes included, so its part comes out as before. + * The build takes each library from the first hub bundle that carries its newest version, with + * its files in that bundle's order, and the hub lists them in an order of its own: the refresh of + * 2026-11-02 rewrote 30 parts whose libraries had not changed, because the new bundle of Branching + * Scenario, the first to carry them, listed their files differently (#116). + * @param {{ folder: string, files: Map }[]} libraries each folder's files, by their names in the pack + * @param {Uint8Array} [previous] the pack this one replaces + */ +export async function writePack(libraries, previous) { + const reader = previous ? new ZipReader(new Uint8ArrayReader(previous)) : null + const before = new Map() + for (const entry of reader ? await reader.getEntries() : []) { + const slash = entry.filename.indexOf('/') + if (entry.directory || slash < 0) continue + const folder = entry.filename.slice(0, slash) + if (!before.has(folder)) before.set(folder, []) + before.get(folder).push(entry) + } + + const writer = new ZipWriter(new BlobWriter('application/zip'), { lastModDate: FIXED_DATE, extendedTimestamp: false }) + await writer.add('h5p.json', new TextReader(PART_MANIFEST)) + for (const { folder, files } of libraries) { + const kept = before.get(folder) + if (kept && (await holds(kept, files))) { + for (const entry of kept) { + const raw = await entry.getData(new Uint8ArrayWriter(), { passThrough: true }) + await writer.add(entry.filename, new Uint8ArrayReader(raw), { passThrough: true, entry }) + } + } else { + for (const [name, bytes] of files) await writer.add(name, new Uint8ArrayReader(bytes)) + } + } + await reader?.close() + return new Uint8Array(await (await writer.close()).arrayBuffer()) +} + +/** Whether a folder's entries in a pack hold exactly these files, by name and bytes. */ +async function holds(entries, files) { + if (entries.length !== files.size) return false + for (const entry of entries) { + const bytes = files.get(entry.filename) + if (!bytes || bytes.length !== entry.uncompressedSize) return false + if (Buffer.compare(await entry.getData(new Uint8ArrayWriter()), bytes) !== 0) return false + } + return true +} + /** * Writes `/.h5p` for every library folder in the pack and `/index.json`, * replacing whatever `outDir` held. Returns the index's entries. diff --git a/packages/player/tests/unit/pack-bytes.test.ts b/packages/player/tests/unit/pack-bytes.test.ts index a3d89532..c1de6948 100644 --- a/packages/player/tests/unit/pack-bytes.test.ts +++ b/packages/player/tests/unit/pack-bytes.test.ts @@ -1,13 +1,18 @@ import { execFileSync } from 'node:child_process' -import { resolve } from 'node:path' +import { mkdtemp } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' import { pathToFileURL } from 'node:url' import { describe, expect, it } from 'vitest' +import { splitPack, writePack } from '../../../libraries/scripts/parts.mjs' /** * The library pack's bytes, which the player pins by hash, depend on nothing but its libraries. A * zip records a wall-clock time with no zone, and the pack was once written with a date built * from UTC fields: the pack built at UTC+2 said 02:00 and CI's said 00:00, so the first weekly - * refresh after the split rewrote every archive with no library changed (#48, #51). + * refresh after the split rewrote every archive with no library changed (#48, #51). Nor do they + * depend on the order a hub bundle lists a library's files in, which changed when Branching + * Scenario's bundle was rebuilt and rewrote the 30 parts it is the first to carry (#116). * * Each time zone runs in a process of its own: whether setting `TZ` inside a running test worker * moves its clock is the engine's business, and a child process starts in the zone it is given. @@ -16,22 +21,24 @@ import { describe, expect, it } from 'vitest' const librariesDir = resolve(import.meta.dirname, '../../../libraries') const parts = pathToFileURL(resolve(librariesDir, 'scripts/parts.mjs')).href -// A pack of one library, written the way apps/demo/scripts/build-libraries.mjs writes the real -// one, split by the real `splitPack`; prints the bytes' hashes and the first entry's DOS time and date. +// A pack of one library, written by the `writePack` that apps/demo/scripts/build-libraries.mjs +// writes the real one with, split by the real `splitPack`; prints the bytes' hashes and the first +// entry's DOS time and date. const build = ` import { createHash } from 'node:crypto' import { mkdtemp, readFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' -import { BlobWriter, TextReader, ZipWriter, configure } from '@zip.js/zip.js' -import { FIXED_DATE, splitPack } from ${JSON.stringify(parts)} +import { splitPack, writePack } from ${JSON.stringify(parts)} -configure({ useWebWorkers: false }) -const writer = new ZipWriter(new BlobWriter('application/zip'), { lastModDate: FIXED_DATE, extendedTimestamp: false }) -await writer.add('h5p.json', new TextReader('{}')) -await writer.add('H5P.Tiny-1.0/library.json', new TextReader(JSON.stringify({ machineName: 'H5P.Tiny', majorVersion: 1, minorVersion: 0, patchVersion: 3 }))) -await writer.add('H5P.Tiny-1.0/tiny.js', new TextReader('H5P.Tiny = function () {}')) -const pack = new Uint8Array(await (await writer.close()).arrayBuffer()) +const text = (value) => new TextEncoder().encode(value) +const pack = await writePack([{ + folder: 'H5P.Tiny-1.0', + files: new Map([ + ['H5P.Tiny-1.0/library.json', text(JSON.stringify({ machineName: 'H5P.Tiny', majorVersion: 1, minorVersion: 0, patchVersion: 3 }))], + ['H5P.Tiny-1.0/tiny.js', text('H5P.Tiny = function () {}')] + ]) +}]) const out = await mkdtemp(join(tmpdir(), 'pack-bytes-')) const [{ sha256 }] = await splitPack(pack, out) @@ -68,4 +75,45 @@ describe('the library pack', () => { expect(time).toBe(2 << 11) expect(date).toBe(((2000 - 1980) << 9) | (1 << 5) | 1) }, 30_000) + + // Two libraries' files, as a hub bundle lists them, and the same listed the other way round. + const text = (value: string) => new TextEncoder().encode(value) + const tiny = (patchVersion: number, script: string) => ({ + folder: 'H5P.Tiny-1.0', + files: new Map([ + ['H5P.Tiny-1.0/library.json', text(JSON.stringify({ machineName: 'H5P.Tiny', majorVersion: 1, minorVersion: 0, patchVersion }))], + ['H5P.Tiny-1.0/tiny.js', text(script)], + ['H5P.Tiny-1.0/tiny.css', text('.h5p-tiny { color: teal }')] + ]) + }) + const other = () => ({ + folder: 'H5P.Other-1.0', + files: new Map([ + ['H5P.Other-1.0/library.json', text(JSON.stringify({ machineName: 'H5P.Other', majorVersion: 1, minorVersion: 0, patchVersion: 7 }))], + ['H5P.Other-1.0/other.js', text('H5P.Other = function () {}')], + ['H5P.Other-1.0/language/de.json', text('{"semantics":[]}')] + ]) + }) + const reversed = ({ folder, files }: { folder: string; files: Map }) => ({ folder, files: new Map([...files].reverse()) }) + /** Each part's SHA-256, by folder. */ + async function partsOf(pack: Uint8Array) { + const index = await splitPack(pack, await mkdtemp(join(tmpdir(), 'pack-parts-'))) + return Object.fromEntries(index.map(({ folder, sha256 }: { folder: string; sha256: string }) => [folder, sha256])) + } + + it('keeps the entries of a library whose files are the same, in whatever order a bundle lists them', async () => { + const before = await writePack([other(), tiny(1, 'H5P.Tiny = function () {}')]) + const again = [reversed(other()), reversed(tiny(1, 'H5P.Tiny = function () {}'))] + expect(Buffer.compare(await writePack(again, before), before)).toBe(0) + // Without the pack before, the bundle's order is the order written: the refresh of #116. + expect(Buffer.compare(await writePack(again), before)).not.toBe(0) + }) + + it("writes again only the library that changed, so only its part's hash moves", async () => { + const before = await writePack([other(), tiny(1, 'H5P.Tiny = function () {}')]) + const after = await writePack([reversed(other()), reversed(tiny(2, 'H5P.Tiny = function (params) {}'))], before) + const [partsBefore, partsAfter] = [await partsOf(before), await partsOf(after)] + expect(partsAfter['H5P.Other-1.0']).toBe(partsBefore['H5P.Other-1.0']) + expect(partsAfter['H5P.Tiny-1.0']).not.toBe(partsBefore['H5P.Tiny-1.0']) + }) })