diff --git a/design/normalize.md b/design/normalize.md index efbad3b9..c3596a7d 100644 --- a/design/normalize.md +++ b/design/normalize.md @@ -7,7 +7,7 @@ publishes it. It is its own package, `packages/normalize`, published as `lib/normalize.mjs` holds the policy, `lib/mp4-faststart.mjs` the remux and `lib/zip-writer.mjs` the output side; `bin/h5p-normalize.mjs` is the command. Its documented API is `normalizeArchive`, with `formatBytes` and `archiveFingerprint` re-exported, so the other modules stay free to change; -the demo's content build is the one caller in this repository. Its `prepublishOnly` runs its +the demo's content build and h5p-author's Node build are the callers in this repository. Its `prepublishOnly` runs its typecheck and tests, and its `prepack` copies the root LICENSE in. Five things about it that the code does not say by itself: @@ -26,7 +26,18 @@ it that the code does not say by itself: deflate stream cannot be seeked; a stored one is read in place in the source archive. - **Library folders stay whole; only `content/` media moves to the end.** The forward index treats a folder as finished once a later folder has begun, so a library's own png pulled to the - end would be reported absent after its library had "arrived". + end would be reported absent after its library had "arrived". A caller can also name folders to + write after the media, `last` (2026-11-06, #148): libraries a package carries and nothing reads + as it plays, which h5p-author and the demo's packages carry for an H5P site's upload check + (#146) and which sorted before `content.json` like any library, so that a host that ignores + `Range` had them to wait for. Only the caller knows which they are, so the command has no flag + and the normalizer reads no `library.json`; `content` is never moved and a name the archive + lacks is ignored. Measured on the demo's video with them (Interactive Video, 5.2 MB, its content + ending at 4.1 MB in the library order and at 1.5 with `last`) from a host paced to 1 MB/s that + ignores `Range`: ready at 2.3 s with 2.3 MB sent, where it was 4.5 s with 4.4 MB. From a host + that honours `Range` the boot is the same, 4.5 s either way: the warm pulls every text run + before `ready`, and the trailing run is a span of its own, eight requests for seven. Writing + the run last does not shorten that; trimming the warm would, and is the player's. - **It drops what the player would drop.** Directory entries, names that are not plain relative paths, duplicates after normalisation — read with `filenameValidation: 'tolerant'` for the same reason the player uses it. `plainRelativeName` is a copy of the player's `normalizeEntryName`, diff --git a/packages/normalize/README.md b/packages/normalize/README.md index 5f946c20..adfdadec 100644 --- a/packages/normalize/README.md +++ b/packages/normalize/README.md @@ -42,10 +42,18 @@ import { normalizeArchive, formatBytes } from '@missing-elements/h5p-normalize' const report = await normalizeArchive({ input: 'course.h5p', output: 'course.normalized.h5p', // omit for a dry run + last: ['H5PEditor.Wizard-1.2'], // library folders to write after everything else, the media too onEntry: (result) => console.log(result.name, result.action) }) ``` +`last` names library folders a package carries and nothing reads as it plays, such as the editor +libraries an H5P site's upload check asks for. They are written after the media, each whole, so a +host that ignores `Range` does not wait for them before the frame boots; without `last`, every +library folder comes before `content/`, as it always did. `content` is never moved, a name the +archive has no folder for is ignored, and the command has no flag for it: only the package's author +knows which folders those are. + TypeScript declarations ship with the package: `Report` and `EntryResult` describe what it returns. Node only, not a browser. `crc32(bytes, previous)` from `@missing-elements/h5p-normalize/crc32` diff --git a/packages/normalize/lib/normalize.mjs b/packages/normalize/lib/normalize.mjs index 19e98132..980b8706 100644 --- a/packages/normalize/lib/normalize.mjs +++ b/packages/normalize/lib/normalize.mjs @@ -9,7 +9,9 @@ * then nothing decodes before the last byte lands. * - Entries are ordered `h5p.json`, then the library folders, then `content/`, with content * media last. On a host that ignores `Range` the player boots from the local headers as they - * arrive, and this order gets the libraries in before the media. + * arrive, and this order gets the libraries in before the media. A caller can name library + * folders that a package carries and nothing reads as it plays (`last`): those go after the + * media, so that such a host does not wait for them. * - Text the runtime parses — scripts, styles, JSON — is deflated where an exporter had stored * it. Everything else is copied byte for byte. * @@ -132,15 +134,25 @@ export function plainRelativeName(raw) { * media last. Library folders are kept intact rather than having their images pulled to the * end: the player treats a folder as complete once a later folder has begun, and a straggler * would then be reported missing. + * + * A folder in `last` goes after all of that, the media too, whole and in the order the source + * introduced them (#148): a library the package carries and nothing reads as it plays, such as + * the editor libraries an H5P site's upload check asks for. `content` is never moved, and a name + * the archive has no folder for is ignored. * @internal * * @template {{ name: string, target: TargetForm }} T * @param {T[]} entries in source order + * @param {Iterable} [last] top-level folders to write after everything else * @returns {T[]} */ -export function orderEntries(entries) { +export function orderEntries(entries, last = []) { + const trailing = new Set(last) + trailing.delete('content') /** @type {Map} */ const folderRank = new Map() + /** @type {Map} */ + const trailingRank = new Map() const keyed = entries.map((entry, index) => { const { name, target } = entry const slash = name.indexOf('/') @@ -150,7 +162,10 @@ export function orderEntries(entries) { key = [0, name === 'h5p.json' ? 0 : 1, index] } else { const folder = name.slice(0, slash) - if (folder !== 'content') { + if (trailing.has(folder)) { + if (!trailingRank.has(folder)) trailingRank.set(folder, trailingRank.size) + key = [4, /** @type {number} */ (trailingRank.get(folder)), index] + } else if (folder !== 'content') { if (!folderRank.has(folder)) folderRank.set(folder, folderRank.size) key = [1, /** @type {number} */ (folderRank.get(folder)), index] } else if (target === 'store') { @@ -174,9 +189,14 @@ export function orderEntries(entries) { * @param {string} options.input path of the source archive * @param {string} [options.output] path to write; omitted for a dry run * @param {(result: EntryResult) => void} [options.onEntry] called as each entry is finished + * @param {readonly string[] | ReadonlySet} [options.last] library folders, by name, to write + * after everything else, the media included: what the package carries and nothing reads as it + * plays. The rest of the order is as above; a name the archive has no folder for is ignored * @returns {Promise} */ -export async function normalizeArchive({ input, output, onEntry }) { +export async function normalizeArchive({ input, output, onEntry, last }) { + // A string is iterable, and would be read as one folder per letter. + if (typeof last === 'string') throw new TypeError('`last` takes a list of folder names, not one name') const source = await open(input, 'r') /** @type {string | null} */ let tempDir = null @@ -229,7 +249,7 @@ export async function normalizeArchive({ input, output, onEntry }) { /** @type {EntryResult[]} */ const results = [] let index = 0 - for (const item of orderEntries(planned)) { + for (const item of orderEntries(planned, last)) { let result try { result = await processEntry(item, { source, tempDir, writer, index: index++ }) diff --git a/packages/normalize/tests/normalize.test.ts b/packages/normalize/tests/normalize.test.ts index 59d22a19..efb8128f 100644 --- a/packages/normalize/tests/normalize.test.ts +++ b/packages/normalize/tests/normalize.test.ts @@ -133,6 +133,61 @@ describe('normalizeArchive', () => { expect(report.warnings).toEqual([]) }) + it('writes the library folders named in `last` after everything else, the media included, and changes nothing but the order (#148)', async () => { + // A package as h5p-author writes one with an editor library: carried for an H5P site's upload + // check, and read by nothing as the package plays. + const writer = new ZipWriter(new BlobWriter('application/zip')) + await writer.add('h5p.json', text({ mainLibrary: 'H5P.Foo' })) + await writer.add('H5P.Foo-1.0/library.json', text(libraryJson)) + await writer.add('H5PEditor.Foo-1.0/library.json', text({ ...libraryJson, machineName: 'H5PEditor.Foo', runnable: 0 })) + await writer.add('H5PEditor.Foo-1.0/editor.js', new TextReader(script)) + await writer.add('H5PEditor.Foo-1.0/icon.png', new Uint8ArrayReader(icon), { level: 0 }) + await writer.add('content/content.json', text({})) + await writer.add('content/images/x.png', new Uint8ArrayReader(png), { level: 0 }) + const carried = join(dir, 'carried.h5p') + await writeFile(carried, new Uint8Array(await (await writer.close()).arrayBuffer())) + + // Without `last`, as before: every library folder ahead of the content. + const plain = await normalizeArchive({ input: carried, output: join(dir, 'carried.plain.h5p') }) + expect(plain.entries.map((entry) => entry.name)).toEqual([ + 'h5p.json', + 'H5P.Foo-1.0/library.json', + 'H5PEditor.Foo-1.0/library.json', + 'H5PEditor.Foo-1.0/editor.js', + 'H5PEditor.Foo-1.0/icon.png', + 'content/content.json', + 'content/images/x.png' + ]) + + const output = join(dir, 'carried.last.h5p') + const report = await normalizeArchive({ input: carried, output, last: ['H5PEditor.Foo-1.0'] }) + const { entries, data } = await entriesOf(output) + const order = [ + 'h5p.json', + 'H5P.Foo-1.0/library.json', + 'content/content.json', + 'content/images/x.png', + 'H5PEditor.Foo-1.0/library.json', + 'H5PEditor.Foo-1.0/editor.js', + 'H5PEditor.Foo-1.0/icon.png' + ] + expect(entries.map((entry) => entry.filename)).toEqual(order) + expect(report.entries.map((entry) => entry.name)).toEqual(order) + // The folder stays whole, its png beside its scripts, and every entry is what it was without `last`. + const forms = ({ entries }: typeof plain) => Object.fromEntries(entries.map((entry) => [entry.name, [entry.target, entry.action]])) + expect(forms(report)).toEqual(forms(plain)) + expect(data.get('H5PEditor.Foo-1.0/icon.png')).toEqual(icon) + expect(new TextDecoder().decode(data.get('H5PEditor.Foo-1.0/editor.js'))).toBe(script) + expect(entries.every((entry) => entry.bitFlag?.dataDescriptor === false)).toBe(true) + expect(report.bytesOut).toBe((await stat(output)).size) + + // `content` is never moved, and a name the archive has no folder for is ignored. + const odd = await normalizeArchive({ input: carried, last: ['content', 'H5P.Missing-1.0'] }) + expect(odd.entries.map((entry) => entry.name)).toEqual(plain.entries.map((entry) => entry.name)) + // One name is not a list: a string would be read as a folder per letter. + await expect(normalizeArchive({ input: carried, last: 'H5PEditor.Foo-1.0' as unknown as string[] })).rejects.toThrow(/list of folder names/) + }) + it('reports the same plan on a dry run and writes nothing', async () => { const report = await normalizeArchive({ input }) expect(report.bytesOut).toBeNull() @@ -269,6 +324,31 @@ describe('orderEntries', () => { 'content/video.mp4' ]) }) + + it('puts the folders in `last` after the media, each whole, in the order the source had them (#148)', () => { + const ordered = orderEntries( + named( + 'H5PEditor.B-1.0/b.js', + 'content/video.mp4', + 'content/content.json', + 'H5P.A-1.0/a.js', + 'H5PEditor.A-1.0/a.js', + 'H5PEditor.B-1.0/img/logo.png', + 'h5p.json' + ), + // `content` stays where it is, and a folder the entries lack is ignored. + ['H5PEditor.A-1.0', 'H5PEditor.B-1.0', 'content', 'H5P.Missing-1.0'] + ).map((entry) => entry.name) + expect(ordered).toEqual([ + 'h5p.json', + 'H5P.A-1.0/a.js', + 'content/content.json', + 'content/video.mp4', + 'H5PEditor.B-1.0/b.js', + 'H5PEditor.B-1.0/img/logo.png', + 'H5PEditor.A-1.0/a.js' + ]) + }) }) describe('targetFormOf', () => { diff --git a/packages/normalize/tests/package-types.ts b/packages/normalize/tests/package-types.ts index ae1af614..fe51b07e 100644 --- a/packages/normalize/tests/package-types.ts +++ b/packages/normalize/tests/package-types.ts @@ -19,10 +19,15 @@ export async function asACallerWould(): Promise { // @ts-expect-error the source is named `input` await normalizeArchive({ file: 'in.h5p' }) + // Library folders to write after everything else, the media included (#148). + const trailing: Report = await normalizeArchive({ input: 'in.h5p', output: 'out.h5p', last: ['H5PEditor.Wizard-1.2'] }) + // @ts-expect-error a list of folder names, not one name + await normalizeArchive({ input: 'in.h5p', last: 'H5PEditor.Wizard-1.2' }) + const checked: number = crc32(new Uint8Array(4), crc32(new Uint8Array(4))) // @ts-expect-error bytes, not text crc32('h5p.json') - void [moved, saved, orderEntries, checked] + void [moved, saved, orderEntries, checked, trailing] return `${report.revision} ${revision}` }