diff --git a/AGENTS.md b/AGENTS.md index a6aa19ce..46ca79d2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -159,7 +159,7 @@ Read it before changing the Service Worker's file server (`src/sw/mount.ts`, `pa - Every response needs a `Content-Type`. - `RangeHttpHandle.read` slices a `200` body as it flows, like `stream` always did. - The archive's size never comes from a plain `GET`. -- On a host that honours `Range`, the libraries are pulled whole before the frame boots. +- On a host that honours `Range`, the libraries `h5p.json` declares are pulled whole before the frame boots. - Media is held to a lower inline bar than everything else. - An inline entry is inflated once per burst. - An unknown `pkgId` is a 404 on every route. diff --git a/apps/demo/demo/content/glossary.h5p b/apps/demo/demo/content/glossary.h5p index 8412663e..0c7499ec 100644 Binary files a/apps/demo/demo/content/glossary.h5p and b/apps/demo/demo/content/glossary.h5p differ diff --git a/apps/demo/demo/content/quiz.h5p b/apps/demo/demo/content/quiz.h5p index 34a59fde..8df56351 100644 Binary files a/apps/demo/demo/content/quiz.h5p and b/apps/demo/demo/content/quiz.h5p differ diff --git a/apps/demo/demo/content/video.h5p b/apps/demo/demo/content/video.h5p index 08478283..928360bb 100644 Binary files a/apps/demo/demo/content/video.h5p and b/apps/demo/demo/content/video.h5p differ diff --git a/apps/demo/scripts/build-demo-content.mjs b/apps/demo/scripts/build-demo-content.mjs index 85070315..6882d491 100644 --- a/apps/demo/scripts/build-demo-content.mjs +++ b/apps/demo/scripts/build-demo-content.mjs @@ -24,7 +24,8 @@ import { download, hubBundle as cachedBundle } from './lib/hub.mjs' * dependencies with it at the exact major.minor, so a Moodle with no library installed refused * the demo's quiz, video and glossary for their editor libraries. `h5p.json` lists only what * plays, as the platforms' exports write it, and the player never reads the rest; they follow the - * libraries that play, and the normalizer keeps library folders in that order. + * libraries that play, and the normalizer writes them after the media (`last`), so a host that + * ignores `Range` boots before they arrive. * * The outputs are committed. This is a maintainer's tool with two network dependencies — the * hub, and whatever media a manifest points at — and a deploy should not have either. Hub @@ -193,7 +194,7 @@ async function buildPackage(name) { await mkdir(outDir, { recursive: true }) const output = join(outDir, `${name}.h5p`) - const report = await normalizeArchive({ input: assembled, output }) + const report = await normalizeArchive({ input: assembled, output, last: [...install] }) console.log( `[demo-content] ${name}.h5p ${formatBytes(report.bytesOut ?? 0)}: ${manifest.mainLibrary}, ` + diff --git a/design/author.md b/design/author.md index 855e3e7c..5c1fda4d 100644 --- a/design/author.md +++ b/design/author.md @@ -217,9 +217,10 @@ to 3,000. Eleven things the code does not say by itself: the selected parts over the pack's index at exact versions, through what each added library names in turn (`installClosure`), and `writePackage` copies them after the content, where nothing reads them as the package plays; `h5p.json` lists what plays, as the platforms' exports write it. - A package with media then goes through the normalizer, which sorts every library folder before - `content/` (`orderEntries`), so there the run precedes `content.json` and a host without `Range` - boots after it; one without media keeps the layout and boots before the run arrives. Measured on the 35 + A package with media then goes through the normalizer, told to write the run after the media + (`last`, #148), so with or without media a host without `Range` boots before the run arrives; + over one with `Range` the player's warm takes only what `h5p.json` declares, and so leaves the + run out too. Measured on the 35 packages of the Moodle check, with real files: a Question Set gains 123 kB, most types under 50 kB, and the three containers much more, because their editors preload every type they can hold: Interactive Video 2.4 → 5.1 MB, Course Presentation 1.2 → 4.8 MB (64 libraries, as diff --git a/design/demo.md b/design/demo.md index 848773ef..845f9b92 100644 --- a/design/demo.md +++ b/design/demo.md @@ -323,14 +323,16 @@ SPA-fallback trap described above cannot happen there. the hub's content types not yet installed; `how-it-works.h5p` and `markup.h5p` needed nothing, and `formulas.h5p` carries no library on purpose, so a site without Question Set refuses it, as it does an h5p.com export). The quiz grew 123 kB, the glossary 8 kB and the video from 2.5 to - 5.2 MB, an Interactive Video's editor preloading every type it can hold. The normalizer sorts - library folders before `content/`, so the added ones precede `content.json`: on a host without - `Range`, `/no-range/video.h5p` boots after 4.1 MB where it needed 1.5. The same rebuild moved the + 5.2 MB, an Interactive Video's editor preloading every type it can hold. The script has the + normalizer write the added ones after the media (`last`, #148), so `content.json` stays where it + was: on a host without `Range`, `/no-range/video.h5p` boots after 1.5 MB, where the first + rebuild, with them sorted before `content/`, needed 4.1. The same rebuild moved the quiz's Question Set, Blanks, Drag the Words, Video and Components to the hub's newer patches, which the plan had been refreshing from the pack (the quiz now refreshes only Components, 97 to 101). A bundle has older patches than the pack for some libraries, so the video's refresh also - fetches SingleChoiceSet 1.11 (88 to 89) from the site's pack, as it would for a platform's - export: 180 kB of a library nothing plays, which the plan cannot tell from one that does. Provenance: the libraries are MIT, the text is + fetched SingleChoiceSet 1.11 (88 to 89, 180 kB) from the site's pack, a library nothing in it + plays, carried for the editor; since #148 the plan reads and refreshes only what the content + reaches, and leaves it as it is. Provenance: the libraries are MIT, the text is ours under CC0, and the clip is Big Buck Bunny, © Blender Foundation, CC BY 3.0, credited in the package metadata and on the player page. The `/no-range/` route serves this content on the site, and this content or a fixture locally, which is what the browser tests need. diff --git a/design/libraries.md b/design/libraries.md index 6ee7d934..558df7fb 100644 --- a/design/libraries.md +++ b/design/libraries.md @@ -178,7 +178,17 @@ Which libraries a package plays with: the check that it has them, the sources in the bundle's version played, and the pack's was reported. The rule needs each own `library.json` for the patch, so on a host with `Range` it waits for the deep index after the warm, where they are cached (before it, names give - major and minor and nothing is refreshed). A refreshed folder is withheld from the package's own + major and minor and nothing is refreshed). Only the folders the content reaches are read, from + what it asks for at its top through each one's `preloadedDependencies` + (`PackageReader.ownLibraries`, 2026-11-07, #148): a library nothing plays, an editor library or an + older minor carried for an H5P site's upload check, keeps its name's version and is not + refreshed. The warm leaves those folders out, so reading them would be a request each, and + refreshing them downloaded parts nothing played, before the frame booted. A platform's export + carries every type its editor can hold: five from the local corpus fetched 12.7 MB of parts, + where what they play takes 1.9 (a Course Presentation 31 parts and 3.4 MB, where 3 parts and + 0.33 MB play). The demo's video fetched `H5P.SingleChoiceSet` 1.11, 180 kB, and Image Sequencing + the pack's `H5P.Audio` 1.5 for the 1.2 beside its own 1.5. + A refreshed folder is withheld from the package's own index whole (`superseded`, as `untrusted` withholds), so requests for it resolve to the part's folder by the compatible-version rule `get()` already applies — exact for a patch, the newer minor otherwise — and the content upgrade sees the newer version and runs its steps, as for a @@ -312,15 +322,15 @@ Which libraries a package plays with: the check that it has them, the sources in `index.js` lists what the player serves, as before, so a bundler emits only those parts, the installable app precaches 105 and not 146, and the VS Code extension copies the served ones with an index of them (`scripts/build.mjs`); `install.js` exports the whole pack, for a page - that writes packages with the author's browser build. What it costs the player: a package - carrying two minors of a library, Image Sequencing's `H5P.Audio` 1.2 beside 1.5, has the older - refreshed to the pack's 1.5 as any older minor is, one part downloaded that the package's own - 1.5 could have answered for; accepted, one small part in a rare case, over a rule in the plan - and a refresh reported that fetched nothing. Over a host with `Range` the warm pulls the run - with the rest, as it pulls a platform export's editor libraries, one request either way - (`range-requests.test.ts` on `installable.h5p`: the probe, the tail, the head and the warm, and - the deep index then reads the run's `library.json` files from the cache); on a host without - `Range` the frame boots before the run arrives for a package without media, which keeps the - author's layout, the run after the content. A package with media goes through the normalizer, - whose `orderEntries` sorts every library folder before `content/`, and boots after the run: the - demo's video has 4.1 MB before its content, where it had 1.5. + that writes packages with the author's browser build. What it costs the player, since #148 + nothing at boot: over a host with `Range` the warm takes only the folders `h5p.json` declares, + which leaves the run out as it leaves out a platform export's editor libraries (*the libraries + `h5p.json` declares are pulled whole* in serving.md), and the plan reads and refreshes only the + folders the content reaches (*The refresh rule* above), so it neither reads the run nor refreshes + an older minor in it. `range-requests.test.ts` checks that `installable.h5p`'s run is never + asked for, with the pack offered and without. Over a host without `Range` the frame boots + before the run arrives: the author writes it after the content, and a package with media goes + through the normalizer, which the author's Node build and the demo's script tell to write it + after the media (`last`). Before #148 the run sorted before `content.json` there, the warm + pulled it and the plan refreshed it: the demo's video, 5.2 MB with it, was ready at 4.5 s from + a host paced to 1 MB/s either way, where it is now 1.9 s with `Range` and 2.3 s without. diff --git a/design/normalize.md b/design/normalize.md index c3596a7d..ab3e7e35 100644 --- a/design/normalize.md +++ b/design/normalize.md @@ -7,9 +7,9 @@ publishes it. It is its own package, `packages/normalize`, published as `lib/normalize.mjs` holds the policy, `lib/mp4-faststart.mjs` the remux and `lib/zip-writer.mjs` the output side; `bin/h5p-normalize.mjs` is the command. Its documented API is `normalizeArchive`, with `formatBytes` and `archiveFingerprint` re-exported, so the other modules stay free to change; -the demo's content build and h5p-author's Node build are the callers in this repository. Its `prepublishOnly` runs its -typecheck and tests, and its `prepack` copies the root LICENSE in. Five things about -it that the code does not say by itself: +the demo's content build and h5p-author's Node build are the callers in this repository. Its +`prepublishOnly` runs its typecheck and tests, and its `prepack` copies the root LICENSE in. Five +things about it that the code does not say by itself: - **It writes the zip itself.** zip.js reads the input — descriptors, zip64, inflate, all handled — but its `ZipWriter`, asked not to write a data descriptor, buffers the whole entry @@ -30,14 +30,15 @@ it that the code does not say by itself: write after the media, `last` (2026-11-06, #148): libraries a package carries and nothing reads as it plays, which h5p-author and the demo's packages carry for an H5P site's upload check (#146) and which sorted before `content.json` like any library, so that a host that ignores - `Range` had them to wait for. Only the caller knows which they are, so the command has no flag - and the normalizer reads no `library.json`; `content` is never moved and a name the archive - lacks is ignored. Measured on the demo's video with them (Interactive Video, 5.2 MB, its content - ending at 4.1 MB in the library order and at 1.5 with `last`) from a host paced to 1 MB/s that - ignores `Range`: ready at 2.3 s with 2.3 MB sent, where it was 4.5 s with 4.4 MB. From a host - that honours `Range` the boot is the same, 4.5 s either way: the warm pulls every text run - before `ready`, and the trailing run is a span of its own, eight requests for seven. Writing - the run last does not shorten that; trimming the warm would, and is the player's. + `Range` had them to wait for. h5p-author's Node build and the demo's content script pass them. + Only the caller knows which they are, so the command has no flag and the normalizer reads no + `library.json`; `content` is never moved and a name the archive lacks is ignored. Measured on + the demo's video with them (Interactive Video, 5.2 MB, its content ending at 4.1 MB in the + library order and at 1.5 with `last`) from a host paced to 1 MB/s that ignores `Range`: ready + at 2.3 s with 2.3 MB sent, where it was 4.5 s with 4.4 MB. A host that honours `Range` gains + nothing from the order: the player's warm leaves out what `h5p.json` does not declare, + wherever it lies (*the libraries `h5p.json` declares are pulled whole* in serving.md), and the + same video is ready there at 1.9 s in either layout. - **It drops what the player would drop.** Directory entries, names that are not plain relative paths, duplicates after normalisation — read with `filenameValidation: 'tolerant'` for the same reason the player uses it. `plainRelativeName` is a copy of the player's `normalizeEntryName`, diff --git a/design/serving.md b/design/serving.md index 44d32908..6d657f2f 100644 --- a/design/serving.md +++ b/design/serving.md @@ -186,7 +186,7 @@ How a package's bytes reach the H5P runtime: the Service Worker's virtual file s applied to ranges too when the request accepts it; `tests/browser/compressing-host.test.ts` pins the browser behaviour the probe depends on, and `tests/unit/source-probe.test.ts` covers the fallback with CORS-filtered headers. -- **On a host that honours `Range`, the libraries are pulled whole before the frame boots.** The +- **On a host that honours `Range`, the libraries `h5p.json` declares are pulled whole before the frame boots.** The runtime reads them exhaustively at boot — every `library.json`, script and stylesheet — and an inline entry read on demand cost two ranged requests, the 30-byte local header and then the data (one since the block cache, or none when its block is already held). Measured against a real host (nginx in front of a store, `stream.dev.mkis.fwu.de`): a @@ -209,6 +209,27 @@ How a package's bytes reach the H5P runtime: the Service Worker's virtual file s a picked file and a downloaded archive are local reads. `tests/unit/warm.test.ts` builds a real archive with a video between two library folders and checks the split, the walk and the marker; `tests/browser/warm.test.ts` checks the phase, the cache and the skip on a second load. + + Only the library folders `h5p.json` declares are candidates, beside the root and `content/` + (`declaredFolders`, 2026-11-07, #148). An export lists there every library its content plays, + flattened, and none it carries only for an H5P site's upload check: the editor libraries, and an + older minor one of them names, which h5p-author's packages carry too since #146. Until then the + warm pulled them with the rest. Across the 88 real packages of the local corpus, 70 carry such + folders, and the warm came to 231 MB against 149 MB for what plays; a platform's Course + Presentation pulled 5.3 MB where its content needs 1.8. In all 88, `h5p.json` declared every + folder the dependency walk reaches, so nothing has to be read to decide; a folder left out is + served on demand if anything asks for it, as every file was before the warm, and an `h5p.json` + that declares nothing the archive carries keeps every folder in. Measured from a host paced to + 1 MB/s: the demo's video ready at 1.9 s where it took 4.5 (1.9 before #146 added its editors), + that Course Presentation at 2.2 s where it took 5.8, an Interactive Video export at 2.5 s where it + took 4.4. Leaving a folder out between two that play can split a run, a request more each: the + Interactive Video's load went from five requests to nine, a Column export's from eight to + thirteen, and with 100 ms added to every request and the link at 10 MB/s the two were still + ready sooner, 1.4 s for 1.6 and 1.7 s for 2.5. + The pack's plan reads only the folders the content reaches, for the same reason: what the warm + leaves out would otherwise be read one request at a time (*The refresh rule* in libraries.md). + `tests/browser/range-requests.test.ts` checks that `installable.h5p`'s run is never asked for, + with the pack offered and without. - **Media is held to a lower inline bar than everything else.** `INLINE_MAX_SIZE` is 16 MB, and an inline entry is copied whole before its first byte is served; a media element wants the head first and the rest by ranges. So `chooseStrategy` puts media over `MEDIA_INLINE_MAX_SIZE` diff --git a/docs/libraries.md b/docs/libraries.md index 1065e81d..0566449f 100644 --- a/docs/libraries.md +++ b/docs/libraries.md @@ -32,6 +32,8 @@ the fixes and the look of current H5P. - A library the pack does not have keeps the package's own copy, and runs against the pack's newer versions of the libraries it shares: H5P's rule is that a minor does not break what depends on it. +- A library the package carries that its content never plays, an editor library or an older + minor kept for an H5P site's upload check, stays as it is, and nothing is downloaded for it. Content written for an older minor is upgraded before it starts: the player runs the steps each library ships in its `upgrades.js` on the content's parameters, as the H5P plugins do: a Question diff --git a/packages/author/lib/build.mjs b/packages/author/lib/build.mjs index b1a336d6..ad9e0503 100644 --- a/packages/author/lib/build.mjs +++ b/packages/author/lib/build.mjs @@ -225,9 +225,8 @@ export async function composePackage(source, pack, media) { * What a package holds but its media, in the order a package streams in: its manifest, the * libraries it plays with, each part's own files copied as they are, the content, then the * libraries an H5P site's upload check wants, which nothing reads as the package plays, so the - * frame boots before they arrive on a host without `Range`, unless the normalizer, which a package - * with media goes through, sorts library folders before the content. The caller adds the media - * after it, last. + * frame boots before they arrive on a host without `Range`. The caller adds the media after it, + * last, and in Node has the normalizer keep those libraries after the media (`last`). * @param {import('@zip.js/zip.js').ZipWriter} writer * @param {{ manifest: Record, content: Record, selected: Map, install: Map }} built * @param {import('./pack.mjs').Pack} pack diff --git a/packages/author/lib/install.mjs b/packages/author/lib/install.mjs index dc0a43b5..a7a17343 100644 --- a/packages/author/lib/install.mjs +++ b/packages/author/lib/install.mjs @@ -6,8 +6,8 @@ * A fresh site has nothing, so a build carries the closure (#146): the editor libraries, and the * older minor a library names where the pack plays a newer one. The pack carries them as parts * the player never serves (`installOnly` in its index), and a build writes them after the - * content, where nothing reads them as the package plays; the normalizer, which a package with - * media goes through, sorts library folders before `content/`. + * content, where nothing reads them as the package plays, and in a package with media after the + * media too: the Node build names them to the normalizer as `last`. */ /** @param {{ machineName: string, majorVersion: number | string, minorVersion: number | string }} library */ diff --git a/packages/author/lib/node.mjs b/packages/author/lib/node.mjs index 784f552c..5fd9c24f 100644 --- a/packages/author/lib/node.mjs +++ b/packages/author/lib/node.mjs @@ -143,9 +143,11 @@ export async function buildPackage(input, output, options = {}) { } let final = staged if (media.files.length) { - // Media stored and last, an mp4's index first: the layout a package streams in. + // Media stored and last, an mp4's index first: the layout a package streams in. The + // libraries carried for an H5P site's upload check go after the media, where `writePackage` + // put them after the content: a host that ignores `Range` then boots before they arrive. final = join(temporary, 'normalized.h5p') - await normalizeArchive({ input: staged, output: final }) + await normalizeArchive({ input: staged, output: final, last: [...built.install.keys()] }) } // Unlike a hard link, an exclusive copy also works on FAT/exFAT destinations. await copyFile(final, output, constants.COPYFILE_EXCL).catch((error) => { diff --git a/packages/player/src/shared/constants.ts b/packages/player/src/shared/constants.ts index 2648dba0..014c8e6e 100644 --- a/packages/player/src/shared/constants.ts +++ b/packages/player/src/shared/constants.ts @@ -92,10 +92,12 @@ export const WARM_ENTRY = '__warm__' /** * Warming: on a host that honours `Range`, the runs of the archive that hold the runtime's own - * files — every library's scripts, styles and JSON — are pulled whole before the frame boots, - * instead of one or two ranged requests per file while it boots. Two requests of 4 MB against - * 156 of a few hundred bytes each, measured on one real package; on a host that answers a - * client's requests one at a time, that was the difference between 128 s and 9 s. + * files — the scripts, styles and JSON of every library `h5p.json` declares — are pulled whole + * before the frame boots, instead of one or two ranged requests per file while it boots. Two + * requests of 4 MB against 156 of a few hundred bytes each, measured on one real package; on a + * host that answers a client's requests one at a time, that was the difference between 128 s + * and 9 s. What it declares is what the content plays: an editor library carried for an H5P + * site's upload check is left to demand (#148). * * A span is a run of inline entries separated by gaps of at most `WARM_GAP`, so a large media * entry between two library folders splits it and is never pulled. An entry over diff --git a/packages/player/src/sw/pack-plan.ts b/packages/player/src/sw/pack-plan.ts index 00828833..a0e9e0fa 100644 --- a/packages/player/src/sw/pack-plan.ts +++ b/packages/player/src/sw/pack-plan.ts @@ -13,6 +13,9 @@ import { parseLibraryFolder } from '../shared/library-folders.js' * upgrade. Equal or older in the pack, or only another major there: the package's own. The * own folder is withheld and requests for it resolve to the part's, so a folder is always * served whole from one archive, and the frame upgrades the content to the version served. + * Only a library whose patch is known is refreshed, one whose `library.json` was read, and the + * reader reads those the content reaches: a library nothing plays, such as an editor library + * carried for an H5P site's upload check, stays as it is (#148). * - **Fill.** What the package needs and does not carry comes from the pack: what `h5p.json` * declares, the sub-content its parameters name, and what each library's `library.json` asks * for in turn. Off while a source listed before the pack is still to be tried. @@ -63,15 +66,21 @@ export interface ProvidedLibrary { * left unknown, which leaves the library as it is and its dependencies to the next look; a folder * whose name gives no version, an unversioned one, is the library at a version unknown, which is * how the runtime takes it too, falling back to the bare name. A `library.json` that does not - * parse, or a folder named for another library, is left out. `PackageReader.ownLibraries` and the - * SCORM writer both take what a package carries through here (#109). + * parse, or a folder named for another library, is left out. `readable`, when given, says which + * folders `read` is for; the others are taken by their names, as without it. + * `PackageReader.ownLibraries` reads only the folders the content reaches (#148), and the SCORM + * writer every one (#109). */ -export async function ownLibraries(folders: Iterable, read?: (folder: string) => Promise): Promise { +export async function ownLibraries( + folders: Iterable, + read?: (folder: string) => Promise, + readable?: (folder: string) => boolean +): Promise { const version = (part: unknown) => (typeof part === 'number' && Number.isInteger(part) && part >= 0 ? part : typeof part === 'string' && /^\d+$/.test(part) ? Number(part) : null) const libraries: OwnLibrary[] = [] for (const folder of folders) { const parsed = parseLibraryFolder(folder) - if (!read) { + if (!read || (readable && !readable(folder))) { libraries.push( parsed ? { folder, machineName: parsed.machineName, major: parsed.major, minor: parsed.minor, patch: null, dependencies: null } diff --git a/packages/player/src/sw/package-reader.ts b/packages/player/src/sw/package-reader.ts index 587f981e..a51868cd 100644 --- a/packages/player/src/sw/package-reader.ts +++ b/packages/player/src/sw/package-reader.ts @@ -358,13 +358,58 @@ export class PackageReader { return folders } + /** + * The folders of this archive that `h5p.json` declares in its `preloadedDependencies`, each the + * one the runtime would load for it (`reachableFolder`). A platform's export, and h5p-author's + * package, lists there every library the content plays, flattened, and none of those carried + * only for an H5P site's upload check. Empty when it declares nothing this archive carries. + */ + private declaredFolders(): Set { + const names = new Set(this.entries.keys()) + const available = indexLibraryFolders(names) + const folders = new Set() + for (const dependency of dependenciesIn(this.manifest)) { + const folder = reachableFolder(dependency, names, available, true) + if (folder) folders.add(folder) + } + return folders + } + /** * What the package carries of its libraries, for the pack's plan: the plan's `ownLibraries` over - * this archive's library folders, reading each folder's `library.json` through `read` when the - * caller can afford to — over HTTP each read is two ranged requests until the warm has cached it. + * this archive's library folders. With `read`, passed when the caller can afford one — over HTTP + * a `library.json` is a ranged request until the warm has cached it — the folders the content + * reaches are read: from what it asks for at its top (`contentDependencies`, the plan's roots), + * through each one's `preloadedDependencies`, as the runtime goes. Their versions, patch + * included, and their dependencies are what the plan refreshes and follows. The rest, which + * nothing plays — the editor libraries and older minors a package carries for an H5P site's + * upload check (#146) — are taken by their names, so the plan neither refreshes them nor reads + * them: the warm leaves them out (`warmSpans`), and each would be a request of its own. */ async ownLibraries(read?: (located: LocatedEntry) => Promise): Promise { - return carriedLibraries(this.ownLibraryFolders(), read && ((folder) => read({ reader: this, entry: this.entries.get(`${folder}/library.json`)! }))) + const folders = this.ownLibraryFolders() + if (!read) return carriedLibraries(folders) + const names = new Set(folders.map((folder) => `${folder}/library.json`)) + const available = indexLibraryFolders(names) + const libraries = new Map>() + const readFolder = (folder: string) => { + let library = libraries.get(folder) + if (!library) { + library = read({ reader: this, entry: this.entries.get(`${folder}/library.json`)! }) + libraries.set(folder, library) + } + return library + } + const { found } = await walkDependencies( + await this.contentDependencies(), + (dependency) => reachableFolder(dependency, names, available, this.downgrades), + // One that does not parse has no dependencies here; the plan leaves it out, below. + (folder) => readFolder(folder).catch(() => null) + ) + const reached = new Set(found.map(({ located }) => located)) + // Reached first: the plan takes the first folder that satisfies a dependency, so it takes one it read. + const ordered = [...folders.filter((folder) => reached.has(folder)), ...folders.filter((folder) => !reached.has(folder))] + return carriedLibraries(ordered, readFolder, (folder) => reached.has(folder)) } /** The library folders the attached archives provide, a part's or a bundle's. */ @@ -766,6 +811,13 @@ export class PackageReader { * nothing the boot reads — a cluster of images and no script, style or JSON — is left to * demand, and the total is capped at `WARM_MAX_BYTES`, taking spans in archive order. * + * Only the library folders `h5p.json` declares are candidates (`declaredFolders`), beside the + * root and `content/`: an export lists there every library its content plays, and nothing + * that an H5P site's upload check alone wants, the editor libraries most of all, which are + * often more than half of a package's library bytes (#148). A folder left out is served on + * demand if anything asks for it, as every file was before the warm. An `h5p.json` that + * declares nothing this archive carries leaves every folder in. + * * Only for an index read from the central directory: a forward entry has no offset to walk * from, and the archive it came from is local anyway. */ @@ -775,9 +827,15 @@ export class PackageReader { const nextOffset = new Map() for (let i = 0; i + 1 < offsets.length; i += 1) nextOffset.set(offsets[i], offsets[i + 1]) + const declared = this.declaredFolders() + const booted = (name: string) => { + const slash = name.indexOf('/') + return slash < 0 || declared.size === 0 || name.startsWith('content/') || declared.has(name.slice(0, slash)) + } const candidates = located .filter( (entry) => + booted(entry.name) && entry.strategy.kind === 'inline' && !entry.zip!.encrypted && (entry.method === STORED || entry.method === DEFLATE) && diff --git a/packages/player/tests/browser/range-requests.test.ts b/packages/player/tests/browser/range-requests.test.ts index 774e466e..fb4caebc 100644 --- a/packages/player/tests/browser/range-requests.test.ts +++ b/packages/player/tests/browser/range-requests.test.ts @@ -1,5 +1,6 @@ -import { describe, expect, it } from 'vitest' +import { beforeAll, describe, expect, it } from 'vitest' import '../../src/h5p-offline-player' +import { BLOCK_SIZE } from '../../src/shared/constants' import { FIXTURES, play } from './utils' /** @@ -16,6 +17,42 @@ async function rangesOf(fixture: string, attributes: Record = {} return (await fetch(`/counted/__log?log=${log}`)).json() } +/** + * Two bytes of `installable.h5p`'s run, one in each editor's script, that neither the head's + * block nor the tail's holds: only a request for the run itself asks for them. + */ +let probes: number[] = [] +let archiveSize = 0 + +/** Whether a logged `Range` header asks for the byte at `offset`; a suffix range is counted from the end. */ +function covers(range: string, offset: number): boolean { + const closed = /^bytes=(\d+)-(\d*)$/.exec(range) + if (closed) return Number(closed[1]) <= offset && (closed[2] === '' || offset <= Number(closed[2])) + const suffix = /^bytes=-(\d+)$/.exec(range) + return suffix !== null && offset >= archiveSize - Number(suffix[1]) +} + +beforeAll(async () => { + const archive = new Uint8Array(await (await fetch('/fixtures/installable.h5p')).arrayBuffer()) + archiveSize = archive.length + // A folder's first local header, which comes before the central directory names it: its name is 30 bytes in. + const headerOf = (folder: string) => { + const name = new TextEncoder().encode(folder) + search: for (let at = 0; at + name.length <= archive.length; at += 1) { + for (let i = 0; i < name.length; i += 1) if (archive[at + i] !== name[i]) continue search + return at - 30 + } + return -1 + } + const editor = headerOf('H5PEditor.OfflineTest-1.0/') + const widget = headerOf('H5PEditor.OfflineTestWidget-1.0/') + probes = [Math.max(editor, BLOCK_SIZE) + 4096, widget + 4096] + expect(editor).toBeGreaterThan(0) + expect(probes[0]).toBeLessThan(widget) + expect(probes[1]).toBeLessThan(archiveSize - BLOCK_SIZE) +}) + + describe('the requests a load costs', () => { it('reads a small archive’s index in one request', async () => { const ranges = await rangesOf('basic.h5p') @@ -24,14 +61,25 @@ describe('the requests a load costs', () => { expect(ranges[0]).toBe('bytes=0-0') }) - it('asks for nothing of its own for a run after the content that the boot never reads (#146)', async () => { + it('asks for none of a run after the content that the boot never reads (#146, #148)', async () => { // h5p-author's layout: after the content, the libraries an H5P site's upload check wants, - // 200 kB here past the index's first block. The probe, the tail, the head, and the warm, which - // pulls the run with the rest in its one request; the deep index then reads its library.json - // files from the cache, where each would otherwise be a request. + // 200 kB here, most of it past the index's first block. The probe, the tail, the head, and the + // warm, which stops where the run begins, since `h5p.json` does not declare those folders. + // What it pulls, `h5p.json`, the content type and the content, lies in the head's block, which + // the Jobs worker asks for again: its block cache is its own. const ranges = await rangesOf('installable.h5p') expect(ranges).toHaveLength(4) - expect(new Set(ranges).size).toBe(ranges.length) + expect(ranges.filter((range) => range === `bytes=0-${BLOCK_SIZE - 1}`)).toHaveLength(2) + expect(ranges.filter((range) => probes.some((probe) => covers(range, probe)))).toEqual([]) + }) + + it('asks for none of it with the pack offered either: the plan reads only what the content reaches (#148)', async () => { + // The pack's newer version of the content type replaces the package's own, which the plan + // learns from the library.json files it reads after the warm: the content type's from the + // cache, and none of the run's, each of which would be a request. + const ranges = await rangesOf('installable.h5p', { libraries: 'pack', 'pack-base': '/fixtures/pack/' }) + expect(ranges).toHaveLength(4) + expect(ranges.filter((range) => probes.some((probe) => covers(range, probe)))).toEqual([]) }) it('asks for nothing again once a bundle is attached and the reader is built anew', async () => { diff --git a/packages/player/tests/unit/package-reader.test.ts b/packages/player/tests/unit/package-reader.test.ts index 8bde6073..3c46e501 100644 --- a/packages/player/tests/unit/package-reader.test.ts +++ b/packages/player/tests/unit/package-reader.test.ts @@ -91,6 +91,63 @@ describe('PackageReader.foldersWithUpgrades', () => { }) }) +describe('PackageReader.ownLibraries', () => { + const library = (machineName: string, minorVersion: number, needs: Array<[string, number, number]> = []) => + JSON.stringify({ + machineName, + majorVersion: 1, + minorVersion, + patchVersion: 3, + preloadedDependencies: needs.map(([name, majorVersion, minor]) => ({ machineName: name, majorVersion, minorVersion: minor })) + }) + + it('reads the library.json of the folders the content reaches, and takes the rest by their names (#148)', async () => { + // H5P.A plays and needs H5P.Dep, served by a newer minor; the content runs H5P.Sub as + // sub-content. The editors are carried for an H5P site's upload check, and nothing plays them. + const content = await zipOf({ + 'h5p.json': manifest('H5P.A', [['H5P.A', 1, 0]]), + 'H5P.A-1.0/library.json': library('H5P.A', 0, [['H5P.Dep', 1, 0]]), + 'H5PEditor.A-1.0/library.json': library('H5PEditor.A', 0, [['H5PEditor.W', 1, 0]]), + 'H5P.Dep-1.2/library.json': library('H5P.Dep', 2), + 'H5PEditor.W-1.0/library.json': library('H5PEditor.W', 0), + 'H5P.Sub-1.0/library.json': library('H5P.Sub', 0), + 'content/content.json': JSON.stringify({ part: { library: 'H5P.Sub 1.0', params: {} } }) + }) + const reader = await PackageReader.open('own', memoryHandle(content).handle) + const read: string[] = [] + const own = await reader.ownLibraries(({ entry }) => { + read.push(entry.name) + return new Response(reader.inflate(entry)).json() + }) + + expect(read.sort()).toEqual(['H5P.A-1.0/library.json', 'H5P.Dep-1.2/library.json', 'H5P.Sub-1.0/library.json']) + // What was read first, with its patch and dependencies; the editors by their names alone, + // which the plan neither refreshes nor follows. + expect(own.map(({ folder, patch, dependencies }) => [folder, patch, dependencies?.length ?? null])).toEqual([ + ['H5P.A-1.0', 3, 1], + ['H5P.Dep-1.2', 3, 0], + ['H5P.Sub-1.0', 3, 0], + ['H5PEditor.A-1.0', null, null], + ['H5PEditor.W-1.0', null, null] + ]) + }) + + it('reads nothing without a read, taking every folder by its name', async () => { + const content = await zipOf({ + 'h5p.json': manifest('H5P.A', [['H5P.A', 1, 0]]), + 'H5P.A-1.0/library.json': library('H5P.A', 0), + 'H5PEditor.A-1.0/library.json': library('H5PEditor.A', 0), + 'content/content.json': '{}' + }) + const reader = await PackageReader.open('own', memoryHandle(content).handle) + + expect((await reader.ownLibraries()).map(({ folder, patch }) => [folder, patch])).toEqual([ + ['H5P.A-1.0', null], + ['H5PEditor.A-1.0', null] + ]) + }) +}) + describe('PackageReader.title', () => { const open = async (h5pJson: string) => PackageReader.open('t', memoryHandle(await zipOf({ 'h5p.json': h5pJson, 'content/content.json': '{}' })).handle, { diff --git a/packages/player/tests/unit/warm.test.ts b/packages/player/tests/unit/warm.test.ts index 1b527009..845c06c5 100644 --- a/packages/player/tests/unit/warm.test.ts +++ b/packages/player/tests/unit/warm.test.ts @@ -50,6 +50,60 @@ describe('warmSpans', () => { expect(spans[1].start - spans[0].end).toBeGreaterThanOrEqual(video.length) expect(spans[1].end).toBeLessThanOrEqual(handle.size) }) + + it('leaves out the library folders h5p.json does not declare, which nothing plays (#148)', async () => { + // An editor library between two that play, large enough to split the run, and after the + // content another, as h5p-author writes the ones an H5P site's upload check wants. + const editor = new Uint8Array(WARM_GAP + 1024) + for (let i = 0; i < editor.length; i += 1) editor[i] = (i * 7919) & 0xff + const library = (machineName: string) => JSON.stringify({ machineName, majorVersion: 1, minorVersion: 0 }) + const bytes = await zipOf( + { + 'h5p.json': manifest('H5P.A', [ + ['H5P.A', 1, 0], + ['H5P.B', 1, 0] + ]), + 'H5P.A-1.0/library.json': library('H5P.A'), + 'H5P.A-1.0/a.js': script, + 'H5PEditor.A-1.0/library.json': library('H5PEditor.A'), + 'H5PEditor.A-1.0/a.js': editor, + 'H5P.B-1.0/library.json': library('H5P.B'), + 'H5P.B-1.0/b.js': script, + 'content/content.json': '{"params":{}}', + 'H5PEditor.B-1.0/library.json': library('H5PEditor.B'), + 'H5PEditor.B-1.0/b.js': script + }, + { stored: ['H5PEditor.A-1.0/a.js'] } + ) + const reader = await PackageReader.open('pkg', memoryHandle(bytes).handle, { requireLibraries: false }) + const spans = reader.warmSpans() + + expect(spans.map((span) => span.entries.map((entry) => entry.name))).toEqual([ + ['h5p.json', 'H5P.A-1.0/library.json', 'H5P.A-1.0/a.js'], + ['H5P.B-1.0/library.json', 'H5P.B-1.0/b.js', 'content/content.json'] + ]) + // Each span stops where an editor begins: none of their bytes are pulled. + const offset = (name: string) => reader.entries.get(name)!.zip!.offset + expect(spans[0].end).toBe(offset('H5PEditor.A-1.0/library.json')) + expect(spans[1].end).toBe(offset('H5PEditor.B-1.0/library.json')) + }) + + it('keeps every folder when h5p.json declares nothing the archive carries', async () => { + const bytes = await zipOf({ + 'h5p.json': manifest('H5P.A', []), + 'H5P.A-1.0/library.json': JSON.stringify({ machineName: 'H5P.A', majorVersion: 1, minorVersion: 0 }), + 'H5P.A-1.0/a.js': script, + 'content/content.json': '{"params":{}}' + }) + const reader = await PackageReader.open('pkg', memoryHandle(bytes).handle, { requireLibraries: false }) + + expect(reader.warmSpans().flatMap((span) => span.entries.map((entry) => entry.name))).toEqual([ + 'h5p.json', + 'H5P.A-1.0/library.json', + 'H5P.A-1.0/a.js', + 'content/content.json' + ]) + }) }) describe('warmPackage', () => {