From 3a3286e92ee70e5641627a19e669f6249a9106d9 Mon Sep 17 00:00:00 2001 From: Aleksandr Shevchuk Date: Fri, 6 Nov 2026 22:17:25 +0200 Subject: [PATCH] feat(normalize): write named library folders after the media normalizeArchive takes `last`, a list of library folders to write after everything else, the media included, each whole and in the order the source had them. Without it the order is as before. It is for libraries a package carries and nothing reads as it plays, such as the editor libraries an H5P site's upload check asks for (#146): sorted with the rest they come before content.json, and a host that ignores Range waits for them before the frame boots (#148). --- design/normalize.md | 15 +++- packages/normalize/README.md | 8 +++ packages/normalize/lib/normalize.mjs | 30 ++++++-- packages/normalize/tests/normalize.test.ts | 80 ++++++++++++++++++++++ packages/normalize/tests/package-types.ts | 7 +- 5 files changed, 132 insertions(+), 8 deletions(-) diff --git a/design/normalize.md b/design/normalize.md index efbad3b9..c3596a7d 100644 --- a/design/normalize.md +++ b/design/normalize.md @@ -7,7 +7,7 @@ publishes it. It is its own package, `packages/normalize`, published as `lib/normalize.mjs` holds the policy, `lib/mp4-faststart.mjs` the remux and `lib/zip-writer.mjs` the output side; `bin/h5p-normalize.mjs` is the command. Its documented API is `normalizeArchive`, with `formatBytes` and `archiveFingerprint` re-exported, so the other modules stay free to change; -the demo's content build is the one caller in this repository. Its `prepublishOnly` runs its +the demo's content build and h5p-author's Node build are the callers in this repository. Its `prepublishOnly` runs its typecheck and tests, and its `prepack` copies the root LICENSE in. Five things about it that the code does not say by itself: @@ -26,7 +26,18 @@ it that the code does not say by itself: deflate stream cannot be seeked; a stored one is read in place in the source archive. - **Library folders stay whole; only `content/` media moves to the end.** The forward index treats a folder as finished once a later folder has begun, so a library's own png pulled to the - end would be reported absent after its library had "arrived". + end would be reported absent after its library had "arrived". A caller can also name folders to + write after the media, `last` (2026-11-06, #148): libraries a package carries and nothing reads + as it plays, which h5p-author and the demo's packages carry for an H5P site's upload check + (#146) and which sorted before `content.json` like any library, so that a host that ignores + `Range` had them to wait for. Only the caller knows which they are, so the command has no flag + and the normalizer reads no `library.json`; `content` is never moved and a name the archive + lacks is ignored. Measured on the demo's video with them (Interactive Video, 5.2 MB, its content + ending at 4.1 MB in the library order and at 1.5 with `last`) from a host paced to 1 MB/s that + ignores `Range`: ready at 2.3 s with 2.3 MB sent, where it was 4.5 s with 4.4 MB. From a host + that honours `Range` the boot is the same, 4.5 s either way: the warm pulls every text run + before `ready`, and the trailing run is a span of its own, eight requests for seven. Writing + the run last does not shorten that; trimming the warm would, and is the player's. - **It drops what the player would drop.** Directory entries, names that are not plain relative paths, duplicates after normalisation — read with `filenameValidation: 'tolerant'` for the same reason the player uses it. `plainRelativeName` is a copy of the player's `normalizeEntryName`, diff --git a/packages/normalize/README.md b/packages/normalize/README.md index 5f946c20..adfdadec 100644 --- a/packages/normalize/README.md +++ b/packages/normalize/README.md @@ -42,10 +42,18 @@ import { normalizeArchive, formatBytes } from '@missing-elements/h5p-normalize' const report = await normalizeArchive({ input: 'course.h5p', output: 'course.normalized.h5p', // omit for a dry run + last: ['H5PEditor.Wizard-1.2'], // library folders to write after everything else, the media too onEntry: (result) => console.log(result.name, result.action) }) ``` +`last` names library folders a package carries and nothing reads as it plays, such as the editor +libraries an H5P site's upload check asks for. They are written after the media, each whole, so a +host that ignores `Range` does not wait for them before the frame boots; without `last`, every +library folder comes before `content/`, as it always did. `content` is never moved, a name the +archive has no folder for is ignored, and the command has no flag for it: only the package's author +knows which folders those are. + TypeScript declarations ship with the package: `Report` and `EntryResult` describe what it returns. Node only, not a browser. `crc32(bytes, previous)` from `@missing-elements/h5p-normalize/crc32` diff --git a/packages/normalize/lib/normalize.mjs b/packages/normalize/lib/normalize.mjs index 19e98132..980b8706 100644 --- a/packages/normalize/lib/normalize.mjs +++ b/packages/normalize/lib/normalize.mjs @@ -9,7 +9,9 @@ * then nothing decodes before the last byte lands. * - Entries are ordered `h5p.json`, then the library folders, then `content/`, with content * media last. On a host that ignores `Range` the player boots from the local headers as they - * arrive, and this order gets the libraries in before the media. + * arrive, and this order gets the libraries in before the media. A caller can name library + * folders that a package carries and nothing reads as it plays (`last`): those go after the + * media, so that such a host does not wait for them. * - Text the runtime parses — scripts, styles, JSON — is deflated where an exporter had stored * it. Everything else is copied byte for byte. * @@ -132,15 +134,25 @@ export function plainRelativeName(raw) { * media last. Library folders are kept intact rather than having their images pulled to the * end: the player treats a folder as complete once a later folder has begun, and a straggler * would then be reported missing. + * + * A folder in `last` goes after all of that, the media too, whole and in the order the source + * introduced them (#148): a library the package carries and nothing reads as it plays, such as + * the editor libraries an H5P site's upload check asks for. `content` is never moved, and a name + * the archive has no folder for is ignored. * @internal * * @template {{ name: string, target: TargetForm }} T * @param {T[]} entries in source order + * @param {Iterable} [last] top-level folders to write after everything else * @returns {T[]} */ -export function orderEntries(entries) { +export function orderEntries(entries, last = []) { + const trailing = new Set(last) + trailing.delete('content') /** @type {Map} */ const folderRank = new Map() + /** @type {Map} */ + const trailingRank = new Map() const keyed = entries.map((entry, index) => { const { name, target } = entry const slash = name.indexOf('/') @@ -150,7 +162,10 @@ export function orderEntries(entries) { key = [0, name === 'h5p.json' ? 0 : 1, index] } else { const folder = name.slice(0, slash) - if (folder !== 'content') { + if (trailing.has(folder)) { + if (!trailingRank.has(folder)) trailingRank.set(folder, trailingRank.size) + key = [4, /** @type {number} */ (trailingRank.get(folder)), index] + } else if (folder !== 'content') { if (!folderRank.has(folder)) folderRank.set(folder, folderRank.size) key = [1, /** @type {number} */ (folderRank.get(folder)), index] } else if (target === 'store') { @@ -174,9 +189,14 @@ export function orderEntries(entries) { * @param {string} options.input path of the source archive * @param {string} [options.output] path to write; omitted for a dry run * @param {(result: EntryResult) => void} [options.onEntry] called as each entry is finished + * @param {readonly string[] | ReadonlySet} [options.last] library folders, by name, to write + * after everything else, the media included: what the package carries and nothing reads as it + * plays. The rest of the order is as above; a name the archive has no folder for is ignored * @returns {Promise} */ -export async function normalizeArchive({ input, output, onEntry }) { +export async function normalizeArchive({ input, output, onEntry, last }) { + // A string is iterable, and would be read as one folder per letter. + if (typeof last === 'string') throw new TypeError('`last` takes a list of folder names, not one name') const source = await open(input, 'r') /** @type {string | null} */ let tempDir = null @@ -229,7 +249,7 @@ export async function normalizeArchive({ input, output, onEntry }) { /** @type {EntryResult[]} */ const results = [] let index = 0 - for (const item of orderEntries(planned)) { + for (const item of orderEntries(planned, last)) { let result try { result = await processEntry(item, { source, tempDir, writer, index: index++ }) diff --git a/packages/normalize/tests/normalize.test.ts b/packages/normalize/tests/normalize.test.ts index 59d22a19..efb8128f 100644 --- a/packages/normalize/tests/normalize.test.ts +++ b/packages/normalize/tests/normalize.test.ts @@ -133,6 +133,61 @@ describe('normalizeArchive', () => { expect(report.warnings).toEqual([]) }) + it('writes the library folders named in `last` after everything else, the media included, and changes nothing but the order (#148)', async () => { + // A package as h5p-author writes one with an editor library: carried for an H5P site's upload + // check, and read by nothing as the package plays. + const writer = new ZipWriter(new BlobWriter('application/zip')) + await writer.add('h5p.json', text({ mainLibrary: 'H5P.Foo' })) + await writer.add('H5P.Foo-1.0/library.json', text(libraryJson)) + await writer.add('H5PEditor.Foo-1.0/library.json', text({ ...libraryJson, machineName: 'H5PEditor.Foo', runnable: 0 })) + await writer.add('H5PEditor.Foo-1.0/editor.js', new TextReader(script)) + await writer.add('H5PEditor.Foo-1.0/icon.png', new Uint8ArrayReader(icon), { level: 0 }) + await writer.add('content/content.json', text({})) + await writer.add('content/images/x.png', new Uint8ArrayReader(png), { level: 0 }) + const carried = join(dir, 'carried.h5p') + await writeFile(carried, new Uint8Array(await (await writer.close()).arrayBuffer())) + + // Without `last`, as before: every library folder ahead of the content. + const plain = await normalizeArchive({ input: carried, output: join(dir, 'carried.plain.h5p') }) + expect(plain.entries.map((entry) => entry.name)).toEqual([ + 'h5p.json', + 'H5P.Foo-1.0/library.json', + 'H5PEditor.Foo-1.0/library.json', + 'H5PEditor.Foo-1.0/editor.js', + 'H5PEditor.Foo-1.0/icon.png', + 'content/content.json', + 'content/images/x.png' + ]) + + const output = join(dir, 'carried.last.h5p') + const report = await normalizeArchive({ input: carried, output, last: ['H5PEditor.Foo-1.0'] }) + const { entries, data } = await entriesOf(output) + const order = [ + 'h5p.json', + 'H5P.Foo-1.0/library.json', + 'content/content.json', + 'content/images/x.png', + 'H5PEditor.Foo-1.0/library.json', + 'H5PEditor.Foo-1.0/editor.js', + 'H5PEditor.Foo-1.0/icon.png' + ] + expect(entries.map((entry) => entry.filename)).toEqual(order) + expect(report.entries.map((entry) => entry.name)).toEqual(order) + // The folder stays whole, its png beside its scripts, and every entry is what it was without `last`. + const forms = ({ entries }: typeof plain) => Object.fromEntries(entries.map((entry) => [entry.name, [entry.target, entry.action]])) + expect(forms(report)).toEqual(forms(plain)) + expect(data.get('H5PEditor.Foo-1.0/icon.png')).toEqual(icon) + expect(new TextDecoder().decode(data.get('H5PEditor.Foo-1.0/editor.js'))).toBe(script) + expect(entries.every((entry) => entry.bitFlag?.dataDescriptor === false)).toBe(true) + expect(report.bytesOut).toBe((await stat(output)).size) + + // `content` is never moved, and a name the archive has no folder for is ignored. + const odd = await normalizeArchive({ input: carried, last: ['content', 'H5P.Missing-1.0'] }) + expect(odd.entries.map((entry) => entry.name)).toEqual(plain.entries.map((entry) => entry.name)) + // One name is not a list: a string would be read as a folder per letter. + await expect(normalizeArchive({ input: carried, last: 'H5PEditor.Foo-1.0' as unknown as string[] })).rejects.toThrow(/list of folder names/) + }) + it('reports the same plan on a dry run and writes nothing', async () => { const report = await normalizeArchive({ input }) expect(report.bytesOut).toBeNull() @@ -269,6 +324,31 @@ describe('orderEntries', () => { 'content/video.mp4' ]) }) + + it('puts the folders in `last` after the media, each whole, in the order the source had them (#148)', () => { + const ordered = orderEntries( + named( + 'H5PEditor.B-1.0/b.js', + 'content/video.mp4', + 'content/content.json', + 'H5P.A-1.0/a.js', + 'H5PEditor.A-1.0/a.js', + 'H5PEditor.B-1.0/img/logo.png', + 'h5p.json' + ), + // `content` stays where it is, and a folder the entries lack is ignored. + ['H5PEditor.A-1.0', 'H5PEditor.B-1.0', 'content', 'H5P.Missing-1.0'] + ).map((entry) => entry.name) + expect(ordered).toEqual([ + 'h5p.json', + 'H5P.A-1.0/a.js', + 'content/content.json', + 'content/video.mp4', + 'H5PEditor.B-1.0/b.js', + 'H5PEditor.B-1.0/img/logo.png', + 'H5PEditor.A-1.0/a.js' + ]) + }) }) describe('targetFormOf', () => { diff --git a/packages/normalize/tests/package-types.ts b/packages/normalize/tests/package-types.ts index ae1af614..fe51b07e 100644 --- a/packages/normalize/tests/package-types.ts +++ b/packages/normalize/tests/package-types.ts @@ -19,10 +19,15 @@ export async function asACallerWould(): Promise { // @ts-expect-error the source is named `input` await normalizeArchive({ file: 'in.h5p' }) + // Library folders to write after everything else, the media included (#148). + const trailing: Report = await normalizeArchive({ input: 'in.h5p', output: 'out.h5p', last: ['H5PEditor.Wizard-1.2'] }) + // @ts-expect-error a list of folder names, not one name + await normalizeArchive({ input: 'in.h5p', last: 'H5PEditor.Wizard-1.2' }) + const checked: number = crc32(new Uint8Array(4), crc32(new Uint8Array(4))) // @ts-expect-error bytes, not text crc32('h5p.json') - void [moved, saved, orderEntries, checked] + void [moved, saved, orderEntries, checked, trailing] return `${report.revision} ${revision}` }