diff --git a/AGENTS.md b/AGENTS.md index 2a77be5a..5a06b368 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -39,9 +39,9 @@ packages/player/ @missing-elements/h5p-offline-player — the published h5p-offline-player.ts the custom element — the only public entry point plan.ts `./plan` -> dist/h5p-plan.js: the player's library rules (`planPack`, `ownLibraries`, `librariesNamedIn`, `addonPatterns`, `textMatches`, `dependenciesIn`, `reachableFolder`, - `walkDependencies` and the folder helpers) for the packages that write what it plays, - h5p-author and embed; not covered by docs/compatibility.md, so they pin the player - exactly (#106, #109) + `walkDependencies` and the folder helpers, `packLibrariesIn`) for the packages that + write what it plays, h5p-author and embed; not covered by docs/compatibility.md, so + they pin the player exactly (#106, #109, #108) jobs-worker-handle.ts starts the Jobs worker from the first script the page's CSP allows, holding jobs until it runs user-data-store.ts the learner's saved state for `resume`, in a database of the page's own shadow.css the element's own box, imported `?inline` so Vite minifies it into the bundle @@ -161,7 +161,10 @@ packages/libraries/ @missing-elements/h5p-libraries — the hub's runtime packages/normalize/ @missing-elements/h5p-normalize — the package rewriter, published as a command bin/h5p-normalize.mjs the command lib/ normalize.mjs (the policy), mp4-faststart.mjs (the remux), zip-writer.mjs, - crc32.mjs, format.mjs, fingerprint.mjs (the player's `indexFingerprint` for Node) + crc32.mjs, format.mjs, fingerprint.mjs (the player's `indexFingerprint` for Node); + crc32-table.mjs, the CRC-32 with no Node import, which `./crc32` gives anything + but Node, crc32.mjs zlib's to Node (#108): embed's SCORM writer and h5p-author's + page build write their stored entries with it tests/ node tests; tests/helpers/mp4.ts builds structurally honest mp4s; package-types.ts the package by its name as a TypeScript caller sees it types/ generated, not in git: the declarations, from the JSDoc (`pnpm types`; @@ -183,9 +186,16 @@ packages/verify/ @missing-elements/h5p-verify — plays a package headl packages/author/ @missing-elements/h5p-author — a playable .h5p of any content type in the pack from a short source, published as a command (see *The author package*) bin/h5p-author.mjs the command: `build` and `schema` - lib/build.mjs the source read and checked, the libraries loaded, the content generated, - media copied, the content checked as saved, the parts planned with the - player's `planPack`, the archive written + lib/author.mjs, core.mjs, browser.mjs the three entries: `.`, Node's, the core and Node's build; + `./core`, what needs neither Node nor a page; `./browser`, the core and a + page's build (#108) + lib/build.mjs what both builds do: the source read and checked, the libraries loaded, + the content generated, media resolved, the content checked as saved, the + parts planned with the player's `planPack`, the archive's manifest, + libraries and content written into a zip.js writer + lib/node.mjs what only Node does, and the one module that imports `node:`: the pack read + from disk, a source's files beside it, downloads in a temporary folder, + the package written to a path and normalized (#108) lib/check.mjs `checkContent`: content in the form H5P saves it checked, the layouts' `after` run, for a tool that edits a saved package and for the build, which runs it last on what it writes (#107); no Node import of its own @@ -201,18 +211,23 @@ packages/author/ @missing-elements/h5p-author — a playable .h5p of an Presentation's layout, Interactive Video's placement, Branching Scenario's paths, Game Map's stages, Find the Hotspot's areas lib/text.mjs, translate.mjs, media.mjs, metadata.mjs, pack.mjs, schema.mjs, quiz.mjs text as H5P - stores it; the libraries' translations; files and web addresses; the - licences and roles H5P's metadata takes; the pinned pack read by part; - the field list a source is written from; the first skill's quiz format, - still built + stores it; the libraries' translations; files and web addresses, reached + through the environment's `MediaAccess`; the licences and roles H5P's + metadata takes; the pack read by part, each read whole and checked by + `crypto.subtle`; the field list a source is written from; the first + skill's quiz format, still built scripts/ sync-skill.mjs writes the skill's type files' generated sections from the pack (`pnpm --filter @missing-elements/h5p-author sync:skill`); - check-pack-published.mjs, the publish guard for the pinned pack and player + check-pack-published.mjs, the publish guard for the pinned pack and player, + and the normalizer's `./crc32` tests/ node tests: the generator, text and translations, layouts and schema, the loader, builds with media from disk and a local server, the command, the - quiz format, the check of saved content, the demo's packages among it; - fixtures/ the quiz format's two examples, which apps/skills plays too; - package-types.ts as for the normalizer + quiz format, the check of saved content, the demo's packages among it, and + the page's build (page.test.ts, against Node's); browser/ the page's build in + Chromium, its packages played by the verifier in Node through a Vitest + command (`playUntrusted`, vitest.config.ts); fixtures/ the quiz format's + two examples, which apps/skills plays too; package-types.ts as for the + normalizer, the three entries types/ generated, not in git: the declarations packages/cmi5/ @missing-elements/h5p-cmi5 — cmi5 for the element, published as a library @@ -242,7 +257,8 @@ packages/embed/ @missing-elements/h5p-embed — the embed page as a st scorm/ the SCORM package's launch page (index.html, its title, language and build filled in by the writer) and scorm.js, its bridge to the LMS's API bin/h5p-embed.mjs the command - scripts/ check-player-published.mjs, the publish guard for the player it pins exactly (#109) + scripts/ check-published.mjs, the publish guard for the player it pins exactly (#109) + and the normalizer's `./crc32` (#108) tests/ build.test.ts, no browser; site.test.ts, a written site on 127.0.0.1 framed by a page on localhost, as deployed; scorm.test.ts, the zip in Node and the launch page under a stand-in for an LMS's player page in @@ -1998,7 +2014,7 @@ already holds text to the content check's rules, from the pack an untrusted chec `apps/skills` plays every example untrusted too. Measured on a five-question quiz, at four characters a token: `content.json` was about 3,470 tokens to write by hand, after reading 3,900 to 9,500 of a type's semantics; the source is 1,000 to 1,800 tokens, written from a type file of 200 -to 3,000. Nine things the code does not say by itself: +to 3,000. Ten things the code does not say by itself: - **One generator, three ways to run it.** The third, `saved`, checks content as H5P stores it (*Saved content is checked as the build checks what it writes*, below). `ContentGenerator` was the sweep's @@ -2068,7 +2084,7 @@ to 3,000. Nine things the code does not say by itself: needs no project set up; `--libraries` names another `parts/` folder. Each part is checked against the index's SHA-256 before it is read. `prepublishOnly` refuses while that version is not on npm with this checkout's index (`scripts/check-pack-published.mjs`, which checks the player it - pins too, next bullet), as the player's does. + pins too, next bullet, and the normalizer's `./crc32`, *A page builds too*), as the player's does. - **The player's library rules, not copies of them** (2026-11-02, #106, the owner's rule: reuse what another package has, and extend it where needed). Which libraries content names, an addon's patterns and what they match, and which parts of the pack a package needs are the player's: the @@ -2122,6 +2138,41 @@ to 3,000. Nine things the code does not say by itself: optional select at its empty choice (`goto.type: ""`). A group that is sub-content now gets an id in a build too, as the editor's `initSubContent` gives one: Summary's statements and Single Choice Set's choices had none, and Summary names a statement's statements by it. +- **A page builds too** (2026-11-02, #108). embed-my/creator builds content in a page and in a Tauri + window, on the generator it copied from the sweep on 2026-10-18, without the form's checks, the + problem paths and the translated defaults; and a paste mode for any content type needs the whole + build there. So the package has three entries: `.` for Node, `./core` for what needs neither Node + nor a page, and `./browser`, the core and a page's `openPack` and `buildPackage`. They are + subpaths and not the `browser` condition, which TypeScript ignores unless a project sets + `customConditions`: a page importing `.` would get the page's code typed with Node's declarations, + and the two `buildPackage`s take different arguments. One build: everything but where the pack's + bytes, a source's files and the package are is `build.mjs`'s and runs in both, and `node.mjs` is + the only module that imports `node:`, which `page.test.ts` holds the entries' imports to. Three + things follow: + - **A part is read whole and hashed with `crypto.subtle`, in Node too.** The reader by range Node + had went: the largest part, MathDisplay's, is 2.8 MB, and the hash read every byte anyway. A + page cannot read by range in any case: SHA-256 needs every byte and `crypto.subtle.digest` + takes no stream, ranges fetched after the check are other responses, whose bytes nobody + checked, and jsDelivr cuts a range from its compressed copy. + - **What a page names the pack by.** The `pack` export is taken as the page's bundler emitted it. + The address of a copy of `parts/` has its `index.json` held to the player's shape by + `packLibrariesIn`, which `./plan` exports for this. That is http(s) only, as the player reads + a copy, so a Tauri window on macOS, whose own files are `tauri://`, gives the export. + - **Media are given as `File`s, by the names the source gives them, and stored with their sizes and + CRC in their local headers**, passed through as embed's SCORM writer has stored its entries since + #92, so that the player reads each one's size off its header on a host without `Range`. The CRC + is the normalizer's, which a page imports through `./crc32`, so embed's copy went. An mp4 is not + moved to start before it has arrived whole: that is the normalizer's, which runs in Node. + + A page needs a secure context for `crypto.subtle` and `crypto.randomUUID`, which the generator + now takes from `globalThis.crypto`, and fetches media named by web address under CORS, which most + image hosts refuse; the build says which, where it can. The page's build, run in Node, writes + H5P.Blanks file for file as Node's build does (`page.test.ts`). The Chromium test + (`tests/browser/page.test.ts`) builds H5P.Blanks and a Question Set with a picture drawn on a + canvas, 0.9 and 1.3 MB, and hands the bytes to Node through a Vitest command, where the verifier + plays each untrusted: a pass, no warning, nothing changed by the content check. Of what #108 + listed for the core, `translateDefaults`, `quizSource` and `quizProblems` stay internal: + `loadLibraries` gives the translated defaults, and `buildPackage` takes the quiz format itself. - **The skill's type files are partly generated.** `skills/h5p-author/references/types/.md` holds hand-written notes, an example, and the type's fields between two markers, written by `schemaOf(…, { markdown: true })`: names, kinds, defaults, options, list sizes, a markup field's @@ -2345,7 +2396,7 @@ one row each; an LMS added here gets a row there. from the player's `./plan` with `refresh` off and nothing provided, over `ownLibraries`, `dependenciesIn` and `librariesNamedIn` from the same module. Until #109 (2026-11-02) those were copies kept in step by hand, which nothing checked; since then embed pins the player exactly, as - h5p-author does, and `scripts/check-player-published.mjs` refuses to publish until the pinned + h5p-author does, and `scripts/check-published.mjs` refuses to publish until the pinned version is on npm with `./plan`. It writes those parts into `libraries/`, with an `index.json` that lists only them, and the page names the folder with `libraries="pack"`, `pack-base` and `refresh-libraries="off"`. The element @@ -2498,9 +2549,10 @@ one row each; an LMS added here gets a row there. reader, on Rustici's Java engine, though its code is not public — while Moodle (PHP) and Frappe (Python) read the central directory and never noticed. The package, the fonts and the pack's parts are stored, so each goes in passed through with `compressionMethod: 0`, its size and its - CRC (`fileCrc32`, a second read of the file; `lib/crc32.mjs` is a copy of the normalizer's, - native from Node 22.2 and a table before, since the floor is 22 and a named import of a missing - builtin export fails the whole module at link time — keep the two in step), which zip.js writes + CRC (`fileCrc32`, a second read of the file, by the normalizer's `crc32` from its `./crc32`, + zlib's from Node 22.2 and a table before, since the floor is 22 and a named import of a missing + builtin export fails the whole module at link time; embed kept a copy of it until #108, when + h5p-author's page build needed it too and the table became a module a page imports), which zip.js writes as a complete local header and still streams: writing a 433 MB package peaks between 170 and 230 MB resident, run to run, as the writer before did (170 to 193 in two runs), nowhere near the package held whole. zip.js computes nothing for a passed-through entry, so the test holds the diff --git a/README.md b/README.md index fb96f063..982ef662 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ something else. plays a package headless and says whether it works, [`h5p-normalize`](https://github.com/missing-elements/h5p-offline-player/blob/main/docs/streaming-video.md) rewrites one so its video streams, [`h5p-embed`](https://github.com/missing-elements/h5p-offline-player/tree/main/packages/embed#readme) writes the embed site, and [`h5p-author`](https://github.com/missing-elements/h5p-offline-player/tree/main/packages/author#readme) - builds a package of any content type from a short JSON source. The [agent skills](#agent-skills) + builds a package of any content type from a short JSON source, in Node or in a page. The [agent skills](#agent-skills) cover those workflows and authoring. - **Is tested where it runs.** CI runs the browser tests in Chromium, Firefox and WebKit on every change, and plays every content type in the library pack, untrusted and trusted, on each change diff --git a/docs/compatibility.md b/docs/compatibility.md index 6546c5b9..3bd95df4 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -111,7 +111,8 @@ with the specification is a conformance fix. ### The normalizer, `@missing-elements/h5p-normalize` The command's options, and `normalizeArchive()`, `formatBytes()` and `archiveFingerprint()`, with -their types. The content of a package it rewrites never changes. How it lays out the zip may change in a minor, as +their types, and `crc32()` from `@missing-elements/h5p-normalize/crc32`, which embed and h5p-author +take through a range. The content of a package it rewrites never changes. How it lays out the zip may change in a minor, as long as the player streams the result at least as well. ### The verifier, `@missing-elements/h5p-verify` diff --git a/packages/author/README.md b/packages/author/README.md index c3293905..990a2052 100644 --- a/packages/author/README.md +++ b/packages/author/README.md @@ -176,6 +176,51 @@ it plays. Content saved by the platforms can hold what their editor let an autho does not, such as a required text left empty; a tool that changes one part keeps the problems under the paths it changed. +## In a page + +`@missing-elements/h5p-author/browser` is the same build in a page, or in a Tauri window: the +source checked, filled in, laid out and checked as above, every problem listed at once. Three +things differ. The libraries come from a pack the page can reach. A source's files are `File`s or +`Blob`s the page was given. And the package comes back as a `Blob`. + +```js +import { pack } from '@missing-elements/h5p-libraries' +import { buildPackage, openPack, SourceError } from '@missing-elements/h5p-author/browser' + +const libraries = await openPack(pack) // once, for the page's builds +const { blob, notes } = await buildPackage(source, { libraries, files: { 'volga.png': input.files[0] } }) +``` + +- `openPack(pack)` takes either of two things: + - the `pack` export, its parts emitted by the page's bundler; + - the address of a copy of the pack's `parts/` folder, as a site serves it to ``'s + `pack-base`. Its `index.json` is read from there and held to the shape the player holds it to. + + Each part is fetched whole when a build first needs it, and checked against the index's SHA-256, + as in Node. `close()` the pack when the page is done with it. +- `buildPackage(source, { libraries, files, fetch })` throws a `SourceError` as in Node, and returns + `{ blob, manifest, content, libraries, translations, notes, media }`. Its options: + - `libraries`: a pack `openPack` opened, or what to open one from for this build alone. + - `files`: an object or a `Map` of the files a source names, by the names it gives them: + `{ "path": "volga.png" }` takes `files['volga.png']`. + - `fetch`: for media named by web address. +- Media go in last and stored, each with its size and CRC in the header in front of it, as the player + reads them on a host without `Range`. An mp4 goes in as it was given. Moving its index to the + front, so that it starts before the whole file has arrived, is the normalizer's job, which runs + in Node: `npx @missing-elements/h5p-normalize@latest course.h5p`. + +A page has two limits that Node does not: + +- It needs a secure context, https, `localhost` or a Tauri window, for `crypto.subtle` and + `crypto.randomUUID`. Elsewhere `openPack` and `buildPackage` refuse, and say so. +- It fetches a file named by web address itself, and gets another site's only where that site + allows it (CORS). Most image hosts do not, so give the file itself instead, or a `fetch` that may + fetch it, such as Tauri's HTTP plugin's. + +`@missing-elements/h5p-author/core` is the part of the package that needs neither Node nor a page: +the generator, `loadLibraries` and `packageLibraries`, `checkContent` and the rules for text, as +the package's main export has them. A tool beside a build in either imports it from there. + ## What it does not do It checks structure, not pedagogy: whether the answers are right, the questions fair, or the diff --git a/packages/author/bin/h5p-author.mjs b/packages/author/bin/h5p-author.mjs index 3083129f..763ef590 100755 --- a/packages/author/bin/h5p-author.mjs +++ b/packages/author/bin/h5p-author.mjs @@ -2,7 +2,7 @@ import { readFile } from 'node:fs/promises' import { dirname, resolve } from 'node:path' import { buildPackage, schemaOf, SourceError } from '../lib/author.mjs' -import { packVersion } from '../lib/pack.mjs' +import { packVersion } from '../lib/node.mjs' const USAGE = `Usage: h5p-author build [--libraries ] diff --git a/packages/author/lib/author.mjs b/packages/author/lib/author.mjs index 1a893585..229ff612 100644 --- a/packages/author/lib/author.mjs +++ b/packages/author/lib/author.mjs @@ -5,35 +5,20 @@ * package pins, checked against its index. A tool beside the build takes the libraries as the build * loads them (`openPack`, `loadLibraries`, and `packageLibraries` for a package's own), its rules * for text (`markupText`, `plainText`, `allowedTags`), and the check it runs last on what it writes, - * for saved content (`checkContent`). + * for saved content (`checkContent`). This is the entry for Node: `./core` is what of it needs no + * Node, and `./browser` builds in a page. */ -import { openPack } from './pack.mjs' +import { openPack } from './node.mjs' import { schemaLines, schemaMarkdown } from './schema.mjs' import { LAYOUTS } from './types.mjs' -export { buildPackage, SourceError } from './build.mjs' -export { checkContent } from './check.mjs' -export { ContentGenerator, librariesIn, libraryString } from './generator.mjs' -export { loadLibraries, packageLibraries } from './libraries.mjs' -export { openPack } from './pack.mjs' -export { GENERATED_END, GENERATED_START } from './schema.mjs' -export { allowedTags, markupText, plainText } from './text.mjs' +export * from './core.mjs' +export { buildPackage, openPack } from './node.mjs' /** - * @typedef {import('./build.mjs').Source} Source - * @typedef {import('./build.mjs').BuildResult} BuildResult - * @typedef {import('./build.mjs').BuildOptions} BuildOptions - * @typedef {import('./check.mjs').CheckedContent} CheckedContent - * @typedef {import('./generator.mjs').GeneratorOptions} GeneratorOptions - * @typedef {import('./generator.mjs').PackEntry} PackEntry - * @typedef {import('./libraries.mjs').LibraryRecord} LibraryRecord - * @typedef {import('./libraries.mjs').LibrarySource} LibrarySource - * @typedef {import('./libraries.mjs').LoadedLibraries} LoadedLibraries - * @typedef {import('./libraries.mjs').PackageFiles} PackageFiles - * @typedef {import('./translate.mjs').TranslationReport} TranslationReport - * @typedef {import('./pack.mjs').IndexEntry} IndexEntry - * @typedef {import('./pack.mjs').PartLibrary} PartLibrary + * @typedef {import('./node.mjs').BuildResult} BuildResult + * @typedef {import('./node.mjs').BuildOptions} BuildOptions */ /** diff --git a/packages/author/lib/browser.mjs b/packages/author/lib/browser.mjs new file mode 100644 index 00000000..b755ad0b --- /dev/null +++ b/packages/author/lib/browser.mjs @@ -0,0 +1,178 @@ +import { BlobReader, BlobWriter, ZipWriter } from '@zip.js/zip.js' +import { crc32 } from '@missing-elements/h5p-normalize/crc32' +import { packLibrariesIn } from '@missing-elements/h5p-offline-player/plan' +import { composePackage, readSource, writePackage } from './build.mjs' +import { MediaStore } from './media.mjs' +import { Pack } from './pack.mjs' + +export * from './core.mjs' + +/** + * @missing-elements/h5p-author/browser: the build in a page, or a Tauri window. The same build as + * Node's, the source checked, generated, laid out and checked as saved content, but for three + * things: the pack is one the page can reach, each part fetched whole and checked against the + * index's SHA-256; a source's files are `File`s or `Blob`s the page was given, by name; and the + * package comes back as a `Blob`, its media stored with their sizes and CRC in their local headers, + * as embed's SCORM writer stores its entries since #92, so that the player reads each one's size + * off its header on a host without `Range`. An mp4 goes in as it was given: moving its index to + * the front is the normalizer's, which runs in Node. It needs a secure context, https, localhost or + * a Tauri window, for `crypto.subtle` and `crypto.randomUUID`. + */ + +/** + * @typedef {{ libraries: import('./pack.mjs').IndexEntry[] }} PackIndex + * the `pack` export of `@missing-elements/h5p-libraries` as a bundler emits it: the index, each + * library with its part's URL + * @typedef {{ + * blob: Blob, manifest: Record, content: Record, libraries: string[], + * translations: import('./translate.mjs').TranslationReport[], notes: import('./build.mjs').Problem[], media: string[] + * }} BuildResult the package, and what is worth telling about it + * @typedef {{ libraries: Pack | PackIndex | string | URL, files?: Map | Record, fetch?: typeof fetch }} BuildOptions + * `libraries`, a pack `openPack` opened, which a page keeps for its builds, or what to open one + * from; `files`, the files a source names, by the names it gives them; `fetch`, for media named by + * web address + */ + +/** What a page's build takes from the browser, which it has in a secure context only. */ +function secure() { + if (!globalThis.crypto?.subtle || typeof globalThis.crypto.randomUUID !== 'function') { + throw new Error('A page builds packages in a secure context only, https, localhost or a Tauri window: it needs crypto.subtle and crypto.randomUUID, which a page has nowhere else') + } +} + +/** + * The pack a page's build reads: the `pack` export of `@missing-elements/h5p-libraries` as a + * bundler emits it, or the address of a copy of its `parts/` folder, served as a site serves it to + * ``'s `pack-base`, whose `index.json` is read from there and held to its shape by the + * player's own rule (`packLibrariesIn`). A part is fetched whole when a build first needs it, and + * checked against the index's SHA-256 before it is read. Keep it open for the page's builds, and + * close it when they are done. + * @param {PackIndex | string | URL} pack + * @param {{ fetch?: typeof fetch }} [options] `fetch`, for the index and the parts + */ +export async function openPack(pack, { fetch: fetchImpl = globalThis.fetch } = {}) { + secure() + /** @type {import('./pack.mjs').IndexEntry[]} */ + let libraries + if (typeof pack === 'string' || pack instanceof URL) { + const written = String(pack) + const base = new URL(written.endsWith('/') ? written : `${written}/`, globalThis.location?.href) + if (base.protocol !== 'https:' && base.protocol !== 'http:') { + throw new Error(`${base.href} is not an http(s) address: a copy of the pack's parts/ folder is read over http(s), as the player reads one; give the pack export instead`) + } + const url = new URL('index.json', base).href + let index + try { + const response = await fetchImpl(url) + if (!response.ok) throw new Error(`it answered ${response.status}`) + index = await response.json() + } catch (error) { + throw new Error(`${url} is not a library pack's index: ${error instanceof Error ? error.message : error}`, { cause: error }) + } + libraries = packLibrariesIn((Array.isArray(index?.libraries) ? index.libraries : []).map((/** @type {any} */ entry) => ( + entry && typeof entry === 'object' && typeof entry.folder === 'string' ? { ...entry, url: new URL(`${entry.folder}.h5p`, base).href } : null + ))) + if (!libraries.length) throw new Error(`${url} is not a library pack's index: it lists no library as the pack's index does`) + } else if (Array.isArray(pack?.libraries) && pack.libraries.every((entry) => typeof entry?.url === 'string')) { + libraries = pack.libraries.map((entry) => ({ ...entry, url: new URL(entry.url, globalThis.location?.href).href })) + } else { + throw new TypeError('Name the library pack: the pack export of @missing-elements/h5p-libraries, or the address of a copy of its parts/ folder') + } + return new Pack(libraries, async (entry) => { + let response + try { + response = await fetchImpl(entry.url) + } catch (error) { + throw new Error(`Cannot fetch the library part ${entry.folder} from ${entry.url}`, { cause: error }) + } + if (!response.ok) throw new Error(`Cannot fetch the library part ${entry.folder}: ${entry.url} answered ${response.status}`) + return new Uint8Array(await response.arrayBuffer()) + }, 'serve the parts of the pack this index belongs to') +} + +/** + * A source's files in a page: the `File`s or `Blob`s it was given, by the names the source gives + * them; downloads kept as `Blob`s. + * @param {Map | Record} files + * @returns {import('./media.mjs').MediaAccess} + */ +function pageMedia(files) { + const named = files instanceof Map ? files : new Map(Object.entries(files)) + return { + what: 'the name of a file given', + name: (source) => source.slice(source.lastIndexOf('/') + 1), + async local(source) { + const file = named.get(source) + if (file instanceof Blob) return { file } + if (file !== undefined) return { missing: 'is not a File or a Blob' } + const names = [...named.keys()] + return { missing: `no such file among those given (${names.length ? `${names.slice(0, 10).join(', ')}${names.length > 10 ? ', …' : ''}` : 'none'})` } + }, + keep: (body) => new Response(body).blob(), + async open(file) { + return { + read: async (offset, length) => new Uint8Array(await file.slice(offset, offset + length).arrayBuffer()), + close: async () => {} + } + } + } +} + +/** + * A `Blob`'s CRC-32, read as a stream. + * @param {Blob} blob + */ +async function crcOf(blob) { + const reader = blob.stream().getReader() + let crc = 0 + for (;;) { + const { done, value } = await reader.read() + if (done) return crc + crc = crc32(value, crc) + } +} + +/** + * The page's own `fetch` for media named by web address, saying what a failure most likely is: a + * page reads another site's file only where that site allows it (CORS), and most do not. + * @type {typeof fetch} + */ +const pageFetch = (input, init) => globalThis.fetch(input, init).catch((error) => { + if (init?.signal?.aborted || !(error instanceof TypeError)) throw error + throw new Error(`${error.message}; a page reads a file from another site only where that site allows it (CORS), so give the file itself instead`, { cause: error }) +}) + +/** + * Builds a package from a source in a page, and returns it as a `Blob`. + * @param {unknown} input the source: `{ type, title, language, license, authors, params }`, or a quiz in the published quiz format + * @param {BuildOptions} options + * @returns {Promise} + */ +export async function buildPackage(input, options) { + secure() + const { libraries, files = {}, fetch: fetchImpl = pageFetch } = options ?? {} + if (!libraries) throw new TypeError('Name the library pack: a pack openPack opened, the pack export of @missing-elements/h5p-libraries, or the address of a copy of its parts/ folder') + const source = readSource(input) + const pack = libraries instanceof Pack ? libraries : await openPack(libraries) + try { + const media = new MediaStore(pageMedia(files), { fetch: fetchImpl }) + const built = await composePackage(source, pack, media) + const writer = new ZipWriter(new BlobWriter('application/zip'), { useWebWorkers: false }) + await writePackage(writer, built, pack) + for (const item of media.files) { + // Stored, its size and CRC in its local header: written into a stream, zip.js would put them in a descriptor after it. + await writer.add(`content/${item.name}`, new BlobReader(item.from), { passThrough: true, compressionMethod: 0, uncompressedSize: item.from.size, crc32: await crcOf(item.from) }) + } + return { + blob: await writer.close(), + manifest: built.manifest, + content: built.content, + libraries: [...built.selected.keys()], + translations: built.translations, + notes: built.notes, + media: media.files.map((item) => item.name) + } + } finally { + if (pack !== libraries) await pack.close() + } +} diff --git a/packages/author/lib/build.mjs b/packages/author/lib/build.mjs index a90199e0..8880fe2a 100644 --- a/packages/author/lib/build.mjs +++ b/packages/author/lib/build.mjs @@ -1,16 +1,9 @@ -import { constants, createReadStream } from 'node:fs' -import { copyFile, lstat, mkdtemp, open, rm, stat } from 'node:fs/promises' -import { dirname, join, resolve } from 'node:path' -import { Readable, Writable } from 'node:stream' -import { TextReader, ZipWriter } from '@zip.js/zip.js' -import { normalizeArchive } from '@missing-elements/h5p-normalize' +import { TextReader } from '@zip.js/zip.js' import { librariesNamedIn, planPack } from '@missing-elements/h5p-offline-player/plan' import { checkContent, layoutContext, valueAt } from './check.mjs' import { ContentGenerator, libraryString } from './generator.mjs' import { loadLibraries } from './libraries.mjs' -import { MediaStore } from './media.mjs' import { CC_VERSIONS, LICENSES, ROLES } from './metadata.mjs' -import { openPack } from './pack.mjs' import { quizProblems, quizSource } from './quiz.mjs' import { SourceError } from './source-error.mjs' import { escapeText } from './text.mjs' @@ -23,7 +16,8 @@ export { SourceError } * its media copied and described, checked as saved content is and laid out, and written with the * libraries it needs, each part checked against the pack's index. Nothing is written unless all of * it is right: every problem the source has is listed at once, each by its path, so a fix takes one - * round. + * round. All of it but where the files are and where the package goes is the same in Node + * (`node.mjs`) and in a page (`browser.mjs`), and is here. */ /** @@ -34,13 +28,6 @@ export { SourceError } * }} Source a package's source: the content type, its metadata, and the parameters that differ from the defaults * @typedef {{ path: string, message: string }} Problem * @typedef {import('./translate.mjs').TranslationReport} TranslationReport - * @typedef {{ - * output: string, manifest: Record, content: Record, - * libraries: string[], translations: TranslationReport[], notes: Problem[], media: string[] - * }} BuildResult what was written, and what is worth telling about it - * @typedef {{ libraries?: string, base?: string, fetch?: typeof fetch }} BuildOptions - * `libraries`, a library pack's parts folder in place of the pinned pack; `base`, the folder a - * source's file paths are read from; `fetch`, for media named by web address */ /** @param {unknown} value @returns {value is Record} */ @@ -140,147 +127,110 @@ async function closure(pack, main, content) { } /** - * The refusal of an output that exists, with the file system's code, so a caller tells it from - * a source's problems. - * @param {string} output - */ -function exists(output) { - return Object.assign(new Error(`${output} exists, and a build never writes over a file: name a new one`), { code: 'EEXIST' }) -} - -/** - * Builds a package from a source and writes it to `output`, which must not exist yet. - * @param {unknown} input the source: `{ type, title, language, license, authors, params }`, or a quiz in the published quiz format - * @param {string} output - * @param {BuildOptions} [options] - * @returns {Promise} + * A source made what a package holds, its media resolved through `media`: the manifest, the content, + * the libraries' parts in the order the package lists them, and what is worth telling. Nothing is + * written: the caller writes the package (`writePackage`) and its media. + * @template F + * @param {Source} source as `readSource` returned it + * @param {import('./pack.mjs').Pack} pack + * @param {import('./media.mjs').MediaStore} media + * @returns {Promise<{ manifest: Record, content: Record, selected: Map, translations: TranslationReport[], notes: Problem[] }>} + * @internal */ -export async function buildPackage(input, output, options = {}) { - output = resolve(output) - // Before any work, and again as the package lands, for a file that appears in between. - if (await lstat(output).then(() => true, () => false)) throw exists(output) - if (!(await stat(dirname(output)).catch(() => null))?.isDirectory()) throw new Error(`${dirname(output)} is not a folder to write the package in`) - const source = readSource(input) - const pack = await openPack(options.libraries) - let temporary - try { - /** @type {Problem[]} */ - const problems = [] - /** @type {Problem[]} */ - const notes = [] - const report = (/** @type {string} */ path, /** @type {string} */ message) => problems.push({ path, message }) - const note = (/** @type {string} */ path, /** @type {string} */ message) => notes.push({ path, message }) - const { entries, main: mainLibrary, translations } = await loadLibraries(pack, source) - const main = mainLibrary.library.machineName - const authors = source.authors.map((author) => ({ name: escapeText(author.name), role: author.role })) - const generator = new ContentGenerator(entries, { - metadata(entry, written, path) { - const title = written?.title - if (title !== undefined && (typeof title !== 'string' || !title.trim())) report(`${path}.metadata.title`, 'expected a title') - else if (typeof title === 'string' && title.length > 255) report(`${path}.metadata.title`, 'takes at most 255 characters') - return { - contentType: escapeText(String(entry.library.title ?? entry.library.machineName)), - license: source.license, - ...(source.licenseVersion ? { licenseVersion: source.licenseVersion } : {}), - authors: authors.map((author) => ({ ...author })), - ...(typeof title === 'string' && title.trim() ? { title: escapeText(title) } : {}) - } +export async function composePackage(source, pack, media) { + /** @type {Problem[]} */ + const problems = [] + /** @type {Problem[]} */ + const notes = [] + const report = (/** @type {string} */ path, /** @type {string} */ message) => problems.push({ path, message }) + const note = (/** @type {string} */ path, /** @type {string} */ message) => notes.push({ path, message }) + const { entries, main: mainLibrary, translations } = await loadLibraries(pack, source) + const main = mainLibrary.library.machineName + const authors = source.authors.map((author) => ({ name: escapeText(author.name), role: author.role })) + const generator = new ContentGenerator(entries, { + metadata(entry, written, path) { + const title = written?.title + if (title !== undefined && (typeof title !== 'string' || !title.trim())) report(`${path}.metadata.title`, 'expected a title') + else if (typeof title === 'string' && title.length > 255) report(`${path}.metadata.title`, 'takes at most 255 characters') + return { + contentType: escapeText(String(entry.library.title ?? entry.library.machineName)), + license: source.license, + ...(source.licenseVersion ? { licenseVersion: source.licenseVersion } : {}), + authors: authors.map((author) => ({ ...author })), + ...(typeof title === 'string' && title.trim() ? { title: escapeText(title) } : {}) } - }) - if (!mainLibrary.semantics) report('type', `${source.type} has no semantics: it is not a content type an author fills in`) - else if (!mainLibrary.library.runnable) report('type', `${mainLibrary.library.machineName} is a part other content holds, not a content type on its own: put it in one, such as H5P.Column`) - - // Layouts that turn what a source writes into what H5P stores, before it is generated. - const layout = layoutContext(entries, report, note) - const written = structuredClone(source.params) - for (const place of placesOf(main, written)) LAYOUTS[place.library]?.before?.(place.params, place.path, layout(place.library)) - - const generated = generator.generate(mainLibrary, written) - problems.push(...generator.problems) - notes.push(...generator.notes) - - // Media: copied, described, put in place of what was written. - temporary = await mkdtemp(join(dirname(output), '.h5p-author-')) - const media = new MediaStore({ base: options.base ?? process.cwd(), temporary, fetch: options.fetch }) - for (const { field, value, path } of generator.media) { - const resolved = await media.resolve(field, value, path, report) - for (const key of Object.keys(value)) delete value[key] - Object.assign(value, resolved) } + }) + if (!mainLibrary.semantics) report('type', `${source.type} has no semantics: it is not a content type an author fills in`) + else if (!mainLibrary.library.runnable) report('type', `${mainLibrary.library.machineName} is a part other content holds, not a content type on its own: put it in one, such as H5P.Column`) + + // Layouts that turn what a source writes into what H5P stores, before it is generated. + const layout = layoutContext(entries, report, note) + const written = structuredClone(source.params) + for (const place of placesOf(main, written)) LAYOUTS[place.library]?.before?.(place.params, place.path, layout(place.library)) + + const generated = generator.generate(mainLibrary, written) + problems.push(...generator.problems) + notes.push(...generator.notes) + + // Media: copied, described, put in place of what was written. + for (const { field, value, path } of generator.media) { + const resolved = await media.resolve(field, value, path, report) + for (const key of Object.keys(value)) delete value[key] + Object.assign(value, resolved) + } - // Checked last as saved content is, as a tool that changes a package checks it: the layouts set - // what the editor's widgets set and check what they would not let pass. Where the source has a - // problem already, the check finds the same one again, or the gap it left: a value missing - // where the generator refused what was written inside it. Only what it finds elsewhere is added. - const checked = checkContent({ entries, main: mainLibrary, translations }, { type: libraryString(mainLibrary.library), params: generated }) - const fromSource = [...problems] - const known = (/** @type {Problem} */ problem) => fromSource.some(({ path }) => path === problem.path || inside(problem.path, path) || - (inside(path, problem.path) && valueAt(checked.params, problem.path) === undefined)) - for (const problem of checked.problems) if (!known(problem)) problems.push(problem) - for (const added of checked.notes) if (!notes.some(({ path, message }) => path === added.path && message === added.message)) notes.push(added) - if (problems.length) throw new SourceError(problems) - const content = checked.params - - const { selected, addons } = await closure(pack, mainLibrary.library, content) - if (addons.length) notes.push({ path: 'params', message: `its text matches ${addons.join(', ')}, which the package carries, as a site with it installed would add it` }) - - // As H5P's exports write it, unescaped: the platforms check it when they load content. - const manifest = { - title: source.title, - language: source.language, - defaultLanguage: source.language, - license: source.license, - ...(source.licenseVersion ? { licenseVersion: source.licenseVersion } : {}), - authors: source.authors.map(({ name, role }) => ({ name, role })), - ...(source.source ? { source: source.source } : {}), - ...(source.yearFrom !== undefined ? { yearFrom: source.yearFrom } : {}), - ...(source.yearTo !== undefined ? { yearTo: source.yearTo } : {}), - mainLibrary: mainLibrary.library.machineName, - embedTypes: ['div'], - preloadedDependencies: [...selected.values()].map((part) => ({ - machineName: part.library.machineName, majorVersion: String(part.library.majorVersion), minorVersion: String(part.library.minorVersion) - })) - } + // Checked last as saved content is, as a tool that changes a package checks it: the layouts set + // what the editor's widgets set and check what they would not let pass. Where the source has a + // problem already, the check finds the same one again, or the gap it left: a value missing + // where the generator refused what was written inside it. Only what it finds elsewhere is added. + const checked = checkContent({ entries, main: mainLibrary, translations }, { type: libraryString(mainLibrary.library), params: generated }) + const fromSource = [...problems] + const known = (/** @type {Problem} */ problem) => fromSource.some(({ path }) => path === problem.path || inside(problem.path, path) || + (inside(path, problem.path) && valueAt(checked.params, problem.path) === undefined)) + for (const problem of checked.problems) if (!known(problem)) problems.push(problem) + for (const added of checked.notes) if (!notes.some(({ path, message }) => path === added.path && message === added.message)) notes.push(added) + if (problems.length) throw new SourceError(problems) + const content = checked.params + + const { selected, addons } = await closure(pack, mainLibrary.library, content) + if (addons.length) notes.push({ path: 'params', message: `its text matches ${addons.join(', ')}, which the package carries, as a site with it installed would add it` }) + + // As H5P's exports write it, unescaped: the platforms check it when they load content. + const manifest = { + title: source.title, + language: source.language, + defaultLanguage: source.language, + license: source.license, + ...(source.licenseVersion ? { licenseVersion: source.licenseVersion } : {}), + authors: source.authors.map(({ name, role }) => ({ name, role })), + ...(source.source ? { source: source.source } : {}), + ...(source.yearFrom !== undefined ? { yearFrom: source.yearFrom } : {}), + ...(source.yearTo !== undefined ? { yearTo: source.yearTo } : {}), + mainLibrary: mainLibrary.library.machineName, + embedTypes: ['div'], + preloadedDependencies: [...selected.values()].map((part) => ({ + machineName: part.library.machineName, majorVersion: String(part.library.majorVersion), minorVersion: String(part.library.minorVersion) + })) + } + return { manifest, content, selected, translations, notes } +} - const staged = join(temporary, 'package.h5p') - const file = await open(staged, 'wx') - try { - const writer = new ZipWriter(Writable.toWeb(file.createWriteStream()), { useWebWorkers: false }) - await writer.add('h5p.json', new TextReader(JSON.stringify(manifest))) - for (const part of selected.values()) { - await writer.appendZip(pack.archiveOf(part), { filter: (entry) => !entry.directory && entry.filename.startsWith(`${part.folder}/`) }) - } - await writer.add('content/content.json', new TextReader(JSON.stringify(content))) - for (const item of media.files) { - await writer.add(`content/${item.name}`, /** @type {ReadableStream} */ (Readable.toWeb(createReadStream(item.from))), { level: 0 }) - } - await writer.close() - } finally { - await file.close() - } - let final = staged - if (media.files.length) { - // Media stored and last, an mp4's index first: the layout a package streams in. - final = join(temporary, 'normalized.h5p') - await normalizeArchive({ input: staged, output: final }) - } - // Unlike a hard link, an exclusive copy also works on FAT/exFAT destinations. - await copyFile(final, output, constants.COPYFILE_EXCL).catch((error) => { - throw error?.code === 'EEXIST' ? exists(output) : error - }) - return { - output, - manifest, - content, - libraries: [...selected.keys()], - translations, - notes, - media: media.files.map((item) => item.name) - } - } finally { - await pack.close() - if (temporary) await rm(temporary, { recursive: true, force: true }) +/** + * What a package holds but its media, in the order a package streams in: its manifest, the + * libraries, each part's own files copied as they are, then the content. The caller adds the media + * after it, last. + * @param {import('@zip.js/zip.js').ZipWriter} writer + * @param {{ manifest: Record, content: Record, selected: Map }} built + * @param {import('./pack.mjs').Pack} pack + * @internal + */ +export async function writePackage(writer, { manifest, content, selected }, pack) { + await writer.add('h5p.json', new TextReader(JSON.stringify(manifest))) + for (const part of selected.values()) { + await writer.appendZip(pack.archiveOf(part), { filter: (entry) => !entry.directory && entry.filename.startsWith(`${part.folder}/`) }) } + await writer.add('content/content.json', new TextReader(JSON.stringify(content))) } /** diff --git a/packages/author/lib/core.mjs b/packages/author/lib/core.mjs new file mode 100644 index 00000000..15f49037 --- /dev/null +++ b/packages/author/lib/core.mjs @@ -0,0 +1,28 @@ +/** + * @missing-elements/h5p-author/core: what the package does that needs neither Node nor a page, for + * a tool that runs beside a build in either: the generator, the libraries loaded as a build loads + * them from any source of them, the check of saved content, and the rules for text. The package's + * entry for Node (`.`) and the one for a page (`./browser`) carry all of it, each with its own + * `buildPackage` and `openPack`. + */ + +export { checkContent } from './check.mjs' +export { ContentGenerator, librariesIn, libraryString } from './generator.mjs' +export { loadLibraries, packageLibraries } from './libraries.mjs' +export { GENERATED_END, GENERATED_START } from './schema.mjs' +export { SourceError } from './source-error.mjs' +export { allowedTags, markupText, plainText } from './text.mjs' + +/** + * @typedef {import('./build.mjs').Source} Source + * @typedef {import('./check.mjs').CheckedContent} CheckedContent + * @typedef {import('./generator.mjs').GeneratorOptions} GeneratorOptions + * @typedef {import('./generator.mjs').PackEntry} PackEntry + * @typedef {import('./libraries.mjs').LibraryRecord} LibraryRecord + * @typedef {import('./libraries.mjs').LibrarySource} LibrarySource + * @typedef {import('./libraries.mjs').LoadedLibraries} LoadedLibraries + * @typedef {import('./libraries.mjs').PackageFiles} PackageFiles + * @typedef {import('./translate.mjs').TranslationReport} TranslationReport + * @typedef {import('./pack.mjs').IndexEntry} IndexEntry + * @typedef {import('./pack.mjs').PartLibrary} PartLibrary + */ diff --git a/packages/author/lib/generator.mjs b/packages/author/lib/generator.mjs index 7cecb006..341fa6f0 100644 --- a/packages/author/lib/generator.mjs +++ b/packages/author/lib/generator.mjs @@ -1,4 +1,3 @@ -import { randomUUID } from 'node:crypto' import { checkMetadata } from './metadata.mjs' import { markupText, plainText } from './text.mjs' @@ -232,7 +231,7 @@ export class ContentGenerator { /** A sub-content id: random, or the caller's, numbered, where it wants the same bytes every run. */ nextId() { this.ids += 1 - return this.options.id ? this.options.id(this.ids) : randomUUID() + return this.options.id ? this.options.id(this.ids) : newId() } /** @@ -715,6 +714,16 @@ export class ContentGenerator { } } +/** + * A new id, random, as H5P's editor gives each part one: crypto's, which Node has, and a page in a + * secure context only, https, localhost or a Tauri window. + * @internal + */ +export function newId() { + if (typeof globalThis.crypto?.randomUUID !== 'function') throw new Error('Content takes ids from crypto.randomUUID, which a page has in a secure context only: https, localhost, or a Tauri window') + return globalThis.crypto.randomUUID() +} + /** Every `library` string in the parameters: the sub-content a package plays. */ export function librariesIn(/** @type {any} */ params, found = new Set()) { if (Array.isArray(params)) for (const item of params) librariesIn(item, found) diff --git a/packages/author/lib/media.mjs b/packages/author/lib/media.mjs index d420a8b1..5e4b2d10 100644 --- a/packages/author/lib/media.mjs +++ b/packages/author/lib/media.mjs @@ -1,16 +1,14 @@ -import { createWriteStream } from 'node:fs' -import { open, stat } from 'node:fs/promises' -import { basename, extname, join, resolve } from 'node:path' -import { Readable, Transform } from 'node:stream' -import { pipeline } from 'node:stream/promises' import { escapeText } from './text.mjs' /** * The files content shows: images, video, audio and other files, named in a source by a path - * beside it or a web address, copied into the package's `content/` folder with what H5P records - * of each — its type, an image's size, its copyright. A video on YouTube, Vimeo, Panopto or - * Echo360 stays there and is played from there, as H5P's editor embeds it (h5peditor-av.js, its - * `providers`), at an address H5P.Video plays. + * beside it, by a name among the files a page was given, or by a web address, copied into the + * package's `content/` folder with what H5P records of each — its type, an image's size, its + * copyright. A video on YouTube, Vimeo, Panopto or Echo360 stays there and is played from there, as + * H5P's editor embeds it (h5peditor-av.js, its `providers`), at an address H5P.Video plays. Where + * the files are, and where a download is kept, is the environment's (`MediaAccess`): `node.mjs` + * reads them from disk and keeps downloads in a temporary folder, `browser.mjs` takes `File`s and + * keeps downloads as `Blob`s. */ /** The types H5P accepts for content (`H5PCore::$defaultContentWhitelist`), by what a field takes. */ @@ -89,10 +87,45 @@ const MAX_DOWNLOAD = 1024 ** 3 const IDLE_MS = 60_000 /** - * @typedef {{ name: string, from: string }} StagedFile a file to add to the package: its name under `content/`, and where it is on disk + * @template F + * @typedef {{ name: string, from: F }} StagedFile a file to add to the package: its name under `content/`, and where its bytes are * @typedef {{ path: string, message: string }} Problem */ +/** + * Where a build's files are, and where a download is kept, which differ between Node and a page. + * @template F a kept file: a path on disk, or a `Blob` + * @typedef {{ + * what: string, + * name(source: string): string, + * local(source: string): Promise<{ file: F } | { missing: string }>, + * keep(body: ReadableStream, extension: string): Promise, + * open(file: F): Promise<{ read(offset: number, length: number): Promise, close(): Promise }> + * }} MediaAccess + * `what`, what a source's path names, for problems; `name`, the file's own name it names; + * `local`, that file, or why there is none; `keep`, a download's bytes kept; `open`, a kept + * file's bytes read from an offset, fewer at its end + */ + +/** + * A file name's extension, lowercased, as Node's `extname` takes it: none for a name that only + * starts with a dot. + * @param {string} name + */ +function extensionOf(name) { + const dot = name.lastIndexOf('.') + return dot > 0 ? name.slice(dot + 1).toLowerCase() : '' +} + +/** + * A path's last segment, past any slash it ends with, as Node's `basename` takes it. + * @param {string} path + */ +function lastSegment(path) { + const trimmed = path.replace(/\/+$/, '') + return trimmed.slice(trimmed.lastIndexOf('/') + 1) +} + /** * An image's width and height, read from its header: PNG, JPEG, GIF and BMP. A JPEG's are in its * frame header, which any number of other segments may come before, a colour profile or XMP among @@ -130,24 +163,23 @@ export async function imageSize(read) { return null } +/** @template F */ export class MediaStore { /** - * @param {{ base: string, temporary: string, fetch?: typeof fetch, idle?: number }} options - * `base`, the folder a source's relative paths are read from; `temporary`, where downloads land; - * `idle`, the milliseconds a download may go without a byte + * @param {MediaAccess} access where the files are, and where a download is kept + * @param {{ fetch?: typeof fetch, idle?: number }} [options] + * `fetch`, for media named by web address; `idle`, the milliseconds a download may go without a byte */ - constructor({ base, temporary, fetch: fetchImpl = globalThis.fetch, idle = IDLE_MS }) { - this.base = base - this.temporary = temporary + constructor(access, { fetch: fetchImpl = globalThis.fetch, idle = IDLE_MS } = {}) { + this.access = access this.fetch = fetchImpl this.idle = idle - /** @type {StagedFile[]} */ + /** @type {StagedFile[]} */ this.files = [] /** @type {Map} by source, so one file named twice is copied once */ this.known = new Map() /** @type {Set} */ this.names = new Set() - this.downloads = 0 } /** @@ -177,7 +209,7 @@ export class MediaStore { } } if (/^[a-z][a-z0-9+.-]*:/i.test(source) && !web) { - report(`${at}.path`, 'expected a file beside the source or an http(s) address') + report(`${at}.path`, `expected ${this.access.what} or an http(s) address`) return { path: source, copyright } } const types = /** @type {Record} */ (TYPES[kind]) @@ -191,17 +223,13 @@ export class MediaStore { /** @type {{ name: string, mime: string, width?: number, height?: number }} */ const described = { name, mime } if (kind === 'image') { - const handle = await open(staged.from, 'r') + const file = await this.access.open(staged.from) try { - const size = await imageSize(async (offset, length) => { - const bytes = new Uint8Array(length) - const { bytesRead } = await handle.read(bytes, 0, length, offset) - return bytes.subarray(0, bytesRead) - }) + const size = await imageSize((offset, length) => file.read(offset, length)) if (!size) report(`${at}.path`, `${source} is not a ${extension.toUpperCase()} image whose size can be read`) else Object.assign(described, size) } finally { - await handle.close() + await file.close() } } this.files.push({ name, from: staged.from }) @@ -248,25 +276,23 @@ export class MediaStore { * @param {(path: string, message: string) => void} report */ async #local(source, types, at, report) { - const from = resolve(this.base, source) - const extension = extname(from).slice(1).toLowerCase() + const name = this.access.name(source) + const extension = extensionOf(name) if (!(extension in types)) { report(`${at}.path`, `${source}: this field takes ${Object.keys(types).map((each) => `.${each}`).join(', ')}`) return null } - try { - const info = await stat(from) - if (!info.isFile()) throw new Error('not a file') - } catch { - report(`${at}.path`, `${source}: no such file beside the source (${from})`) + const found = await this.access.local(source) + if ('missing' in found) { + report(`${at}.path`, `${source}: ${found.missing}`) return null } - return { from, extension, stem: basename(from, extname(from)) } + return { from: found.file, extension, stem: name.slice(0, name.length - extension.length - 1) } } /** - * A web address downloaded to the temporary folder, its type taken from its name or, failing - * that, from what the server says it is. + * A web address downloaded and kept, its type taken from its name or, failing that, from what + * the server says it is. * @param {string} source * @param {string} kind * @param {Record} types @@ -281,14 +307,20 @@ export class MediaStore { report(`${at}.path`, `${source} is not a web address`) return null } - // Started again by every byte that arrives. const controller = new AbortController() const failed = (/** @type {unknown} */ error) => { const cause = controller.signal.aborted ? controller.signal.reason : error report(`${at}.path`, `${source} could not be downloaded: ${cause instanceof Error ? cause.message : cause}`) return null } - const timer = setTimeout(() => controller.abort(new Error(`nothing arrived for ${this.idle / 1000} s`)), this.idle) + // Started again by every byte that arrives: a page's timers have no `refresh`. + /** @type {ReturnType | undefined} */ + let timer + const wait = () => { + clearTimeout(timer) + timer = setTimeout(() => controller.abort(new Error(`nothing arrived for ${this.idle / 1000} s`)), this.idle) + } + wait() try { let response try { @@ -296,13 +328,13 @@ export class MediaStore { } catch (error) { return failed(error) } - timer.refresh() + wait() if (!response.ok || !response.body) { await response.body?.cancel() report(`${at}.path`, `${source} answered ${response.status}${kind === 'video' ? '; for a video on YouTube, Vimeo, Panopto or Echo360 give its address there' : ''}`) return null } - const named = extname(new URL(response.url || url.href).pathname).slice(1).toLowerCase() + const named = extensionOf(lastSegment(new URL(response.url || url.href).pathname)) const served = (response.headers.get('content-type') ?? '').split(';')[0].trim().toLowerCase() const extension = named in types ? named : Object.keys(types).find((each) => types[each] === served) if (!extension) { @@ -310,22 +342,23 @@ export class MediaStore { report(`${at}.path`, `${source} is ${served || 'of an unknown type'}: this field takes ${Object.keys(types).map((each) => `.${each}`).join(', ')}`) return null } - this.downloads += 1 - const from = join(this.temporary, `download-${this.downloads}.${extension}`) let received = 0 - const limit = new Transform({ - transform(chunk, _encoding, callback) { - timer.refresh() + const limited = response.body.pipeThrough(new TransformStream({ + transform(chunk, stream) { + wait() received += chunk.length - callback(received > MAX_DOWNLOAD ? new Error(`larger than ${MAX_DOWNLOAD / 1024 ** 3} GB`) : null, chunk) + if (received > MAX_DOWNLOAD) stream.error(new Error(`larger than ${MAX_DOWNLOAD / 1024 ** 3} GB`)) + else stream.enqueue(chunk) } - }) + })) + let from try { - await pipeline(Readable.fromWeb(/** @type {import('node:stream/web').ReadableStream} */ (response.body)), limit, createWriteStream(from)) + from = await this.access.keep(limited, extension) } catch (error) { return failed(error) } - const stem = basename(decodeURIComponent(new URL(response.url || url.href).pathname), `.${named}`) || kind + const last = lastSegment(decodeURIComponent(new URL(response.url || url.href).pathname)) + const stem = (named && last.endsWith(`.${named}`) && last !== `.${named}` ? last.slice(0, -named.length - 1) : last) || kind return { from, extension, stem } } finally { clearTimeout(timer) diff --git a/packages/author/lib/node.mjs b/packages/author/lib/node.mjs new file mode 100644 index 00000000..dbf3da30 --- /dev/null +++ b/packages/author/lib/node.mjs @@ -0,0 +1,172 @@ +import { constants, createReadStream, createWriteStream } from 'node:fs' +import { copyFile, lstat, mkdtemp, open, readFile, rm, stat } from 'node:fs/promises' +import { basename, dirname, join, resolve } from 'node:path' +import { Readable, Writable } from 'node:stream' +import { fileURLToPath, pathToFileURL } from 'node:url' +import { ZipWriter } from '@zip.js/zip.js' +import { normalizeArchive } from '@missing-elements/h5p-normalize' +import { composePackage, readSource, writePackage } from './build.mjs' +import { MediaStore } from './media.mjs' +import { Pack } from './pack.mjs' + +/** + * What only Node does of a build: the pack read from disk, a source's files read beside it and + * downloads kept in a temporary folder, and the package written to a path, laid out to stream by + * the normalizer, an mp4's index first. Everything else a build does is the page's build's too + * (`browser.mjs`). + */ + +/** + * @typedef {import('./build.mjs').Problem} Problem + * @typedef {import('./translate.mjs').TranslationReport} TranslationReport + * @typedef {{ + * output: string, manifest: Record, content: Record, + * libraries: string[], translations: TranslationReport[], notes: Problem[], media: string[] + * }} BuildResult what was written, and what is worth telling about it + * @typedef {{ libraries?: string, base?: string, fetch?: typeof fetch }} BuildOptions + * `libraries`, a library pack's parts folder in place of the pinned pack; `base`, the folder a + * source's file paths are read from; `fetch`, for media named by web address + */ + +/** + * The pack a build reads: the one this package pins, or the `parts/` folder of another copy. + * @param {string} [folder] a folder holding `index.json` and the parts it names + */ +export async function openPack(folder) { + /** @type {import('./pack.mjs').IndexEntry[]} */ + let libraries + let origin + if (folder) { + const directory = resolve(folder) + let index + try { + index = JSON.parse(await readFile(join(directory, 'index.json'), 'utf8')) + } catch (error) { + throw new Error(`${directory} is not a library pack's parts folder: it has no readable index.json`, { cause: error }) + } + if (!Array.isArray(index?.libraries)) throw new Error(`${join(directory, 'index.json')} lists no libraries`) + libraries = index.libraries.map((/** @type {import('./pack.mjs').IndexEntry} */ entry) => ({ ...entry, url: pathToFileURL(join(directory, `${entry.folder}.h5p`)).href })) + origin = directory + } else { + const { pack } = await import('@missing-elements/h5p-libraries') + libraries = /** @type {import('./pack.mjs').IndexEntry[]} */ (pack.libraries) + origin = '@missing-elements/h5p-libraries' + } + return new Pack(libraries, async (entry) => { + try { + return new Uint8Array(await readFile(fileURLToPath(entry.url))) + } catch (error) { + throw new Error(`Cannot read the library part ${entry.folder} from ${origin}`, { cause: error }) + } + }, 'reinstall the pack') +} + +/** The pinned pack's version, for messages and reports. */ +export async function packVersion() { + const url = import.meta.resolve('@missing-elements/h5p-libraries/package.json') + return JSON.parse(await readFile(fileURLToPath(url), 'utf8')).version +} + +/** + * A source's files on disk: beside the source, by their paths from `base`; downloads kept in + * `temporary`, numbered. + * @param {{ base: string, temporary: string }} folders + * @returns {import('./media.mjs').MediaAccess} + * @internal + */ +export function nodeMedia({ base, temporary }) { + let downloads = 0 + return { + what: 'a file beside the source', + name: (source) => basename(resolve(base, source)), + async local(source) { + const from = resolve(base, source) + try { + if ((await stat(from)).isFile()) return { file: from } + } catch {} + return { missing: `no such file beside the source (${from})` } + }, + async keep(body, extension) { + downloads += 1 + const from = join(temporary, `download-${downloads}.${extension}`) + await body.pipeTo(/** @type {WritableStream} */ (Writable.toWeb(createWriteStream(from)))) + return from + }, + async open(file) { + const handle = await open(file, 'r') + return { + async read(offset, length) { + const bytes = new Uint8Array(length) + const { bytesRead } = await handle.read(bytes, 0, length, offset) + return bytes.subarray(0, bytesRead) + }, + close: () => handle.close() + } + } + } +} + +/** + * The refusal of an output that exists, with the file system's code, so a caller tells it from + * a source's problems. + * @param {string} output + */ +function exists(output) { + return Object.assign(new Error(`${output} exists, and a build never writes over a file: name a new one`), { code: 'EEXIST' }) +} + +/** + * Builds a package from a source and writes it to `output`, which must not exist yet. + * @param {unknown} input the source: `{ type, title, language, license, authors, params }`, or a quiz in the published quiz format + * @param {string} output + * @param {BuildOptions} [options] + * @returns {Promise} + */ +export async function buildPackage(input, output, options = {}) { + output = resolve(output) + // Before any work, and again as the package lands, for a file that appears in between. + if (await lstat(output).then(() => true, () => false)) throw exists(output) + if (!(await stat(dirname(output)).catch(() => null))?.isDirectory()) throw new Error(`${dirname(output)} is not a folder to write the package in`) + const source = readSource(input) + const pack = await openPack(options.libraries) + let temporary + try { + temporary = await mkdtemp(join(dirname(output), '.h5p-author-')) + const media = new MediaStore(nodeMedia({ base: options.base ?? process.cwd(), temporary }), { fetch: options.fetch }) + const built = await composePackage(source, pack, media) + const staged = join(temporary, 'package.h5p') + const file = await open(staged, 'wx') + try { + const writer = new ZipWriter(Writable.toWeb(file.createWriteStream()), { useWebWorkers: false }) + await writePackage(writer, built, pack) + for (const item of media.files) { + await writer.add(`content/${item.name}`, /** @type {ReadableStream} */ (Readable.toWeb(createReadStream(item.from))), { level: 0 }) + } + await writer.close() + } finally { + await file.close() + } + let final = staged + if (media.files.length) { + // Media stored and last, an mp4's index first: the layout a package streams in. + final = join(temporary, 'normalized.h5p') + await normalizeArchive({ input: staged, output: final }) + } + // Unlike a hard link, an exclusive copy also works on FAT/exFAT destinations. + await copyFile(final, output, constants.COPYFILE_EXCL).catch((error) => { + throw error?.code === 'EEXIST' ? exists(output) : error + }) + return { + output, + manifest: built.manifest, + content: built.content, + libraries: [...built.selected.keys()], + translations: built.translations, + notes: built.notes, + media: media.files.map((item) => item.name) + } + } finally { + await pack.close() + if (temporary) await rm(temporary, { recursive: true, force: true }) + } +} diff --git a/packages/author/lib/pack.mjs b/packages/author/lib/pack.mjs index 446e8f9c..f2497a1b 100644 --- a/packages/author/lib/pack.mjs +++ b/packages/author/lib/pack.mjs @@ -1,17 +1,13 @@ -import { createHash } from 'node:crypto' -import { createReadStream } from 'node:fs' -import { open, readFile } from 'node:fs/promises' -import { join, resolve } from 'node:path' -import { Readable } from 'node:stream' -import { fileURLToPath, pathToFileURL } from 'node:url' -import { Reader, TextWriter, ZipReader } from '@zip.js/zip.js' +import { TextWriter, Uint8ArrayReader, ZipReader } from '@zip.js/zip.js' /** - * The library pack a build takes its libraries from: by default the one this package was released - * with, `@missing-elements/h5p-libraries` at the version it pins; or a copy of its `parts/` folder. - * Its index says each library's version, dependencies and part; a part is opened only when a - * build needs that library, and checked against the index's SHA-256 first, so a damaged or - * altered part is never read, let alone copied into a package. + * The library pack a build takes its libraries from: by default, in Node, the one this package was + * released with, `@missing-elements/h5p-libraries` at the version it pins; a copy of its `parts/` + * folder; or, in a page, the parts the page can reach. Its index says each library's version, + * dependencies and part; a part is opened only when a build needs that library, read whole and + * checked against the index's SHA-256 first, so a damaged or altered part is never read, let alone + * copied into a package. How a part's bytes are reached is the environment's: `node.mjs` reads them + * from disk, `browser.mjs` fetches them. The largest part, MathDisplay's, is 2.8 MB. */ /** @@ -25,76 +21,35 @@ import { Reader, TextWriter, ZipReader } from '@zip.js/zip.js' /** * A library whose part has been opened: what the generator reads (`folder`, `library`, * `semantics`), its record in the index, and the names of its translation files, which - * `translation` reads. The part's file stays inside the pack, which copies from it for a build. + * `translation` reads. The part's bytes stay inside the pack, which copies from them for a build. * @typedef {{ folder: string, library: any, semantics: any[] | null, index: IndexEntry, translations: string[] }} PartLibrary */ -/** - * A part read from disk by range: the largest, MathDisplay's, need not be held whole. - * @extends {Reader} - */ -class PartFile extends Reader { - /** @param {string} path */ - constructor(path) { - super(path) - this.path = path - /** @type {import('node:fs/promises').FileHandle | null} */ - this.handle = null - } - - async init() { - await super.init?.() - this.handle = await open(this.path, 'r') - this.size = (await this.handle.stat()).size - } - - /** - * A span streamed from disk, which zip.js copies an entry through. - * @param {{ offset?: number, size?: number }} [span] - * @returns {ReadableStream} - */ - createReadable({ offset = 0, size = this.size - offset } = {}) { - if (!size) return new ReadableStream({ start(controller) { controller.close() } }) - return /** @type {ReadableStream} */ (Readable.toWeb(createReadStream(this.path, { start: offset, end: offset + size - 1 }))) - } - - /** @param {number} offset @param {number} length */ - async readUint8Array(offset, length) { - if (!this.handle) throw new Error(`Library part not open: ${this.path}`) - const bytes = new Uint8Array(length) - let read = 0 - while (read < length) { - const { bytesRead } = await this.handle.read(bytes, read, length - read, offset + read) - if (!bytesRead) throw new Error(`Short read from library part: ${this.path}`) - read += bytesRead - } - return bytes - } - - async close() { - await this.handle?.close() - this.handle = null - } -} +/** @param {ArrayBuffer} buffer */ +const hex = (buffer) => Array.from(new Uint8Array(buffer), (byte) => byte.toString(16).padStart(2, '0')).join('') export class Pack { /** @type {Map>} */ #opened = new Map() /** - * Each opened part's file, its reader, and its translation files by name. - * @type {Map, translations: Map }>} + * Each opened part's bytes, its reader, and its translation files by name. + * @type {Map, translations: Map }>} */ #archives = new Map() - /** @type {string} where the pack came from, for messages */ - #origin + /** @type {(entry: IndexEntry) => Promise>} */ + #read + /** @type {string} */ + #damaged /** * @param {IndexEntry[]} libraries - * @param {string} origin where the pack came from, for messages + * @param {(entry: IndexEntry) => Promise>} read a part's bytes, from wherever the pack is + * @param {string} damaged what to do about a part that does not match the index, for the message */ - constructor(libraries, origin) { + constructor(libraries, read, damaged) { this.libraries = libraries - this.#origin = origin + this.#read = read + this.#damaged = damaged } /** @@ -132,20 +87,13 @@ export class Pack { * @returns {Promise} */ async #open(entry) { - const path = fileURLToPath(entry.url) - const hash = createHash('sha256') - try { - for await (const chunk of createReadStream(path)) hash.update(chunk) - } catch (error) { - throw new Error(`Cannot read the library part ${entry.folder} from ${this.#origin}`, { cause: error }) - } - if (hash.digest('hex') !== entry.sha256) throw new Error(`The library part ${entry.folder} does not match the pack's index (SHA-256): reinstall the pack.`) - const file = new PartFile(path) - const reader = new ZipReader(file, { checkSignature: true }) + const bytes = await this.#read(entry) + if (hex(await crypto.subtle.digest('SHA-256', bytes)) !== entry.sha256) throw new Error(`The library part ${entry.folder} does not match the pack's index (SHA-256): ${this.#damaged}.`) + const reader = new ZipReader(new Uint8ArrayReader(bytes), { checkSignature: true }) // Kept before anything is read, so that `close` closes a part that then fails to open. /** @type {Map} */ const translations = new Map() - this.#archives.set(entry.folder, { file, reader, translations }) + this.#archives.set(entry.folder, { bytes, reader, translations }) const entries = (await reader.getEntries()).filter((item) => !item.directory && item.filename.startsWith(`${entry.folder}/`)) /** @param {string} name */ const json = async (name) => { @@ -189,48 +137,14 @@ export class Pack { archiveOf(part) { const archive = this.#archives.get(part.folder) if (!archive) throw new Error(`The library part ${part.folder} is not open`) - return archive.file + return new Uint8ArrayReader(archive.bytes) } /** Every opened part closed, each on its own, so that one that fails to close leaves no other open. */ async close() { await Promise.allSettled(this.#opened.values()) - await Promise.allSettled([...this.#archives.values()].map(async ({ file, reader }) => { - try { - await reader.close() - } finally { - // A part that failed to open has its file to close, whatever its reader makes of closing. - await file.close() - } - })) + await Promise.allSettled([...this.#archives.values()].map(({ reader }) => reader.close())) this.#archives.clear() this.#opened.clear() } } - -/** - * The pack a build reads: the one this package pins, or the `parts/` folder of another copy. - * @param {string} [folder] a folder holding `index.json` and the parts it names - */ -export async function openPack(folder) { - if (folder) { - const directory = resolve(folder) - let index - try { - index = JSON.parse(await readFile(join(directory, 'index.json'), 'utf8')) - } catch (error) { - throw new Error(`${directory} is not a library pack's parts folder: it has no readable index.json`, { cause: error }) - } - if (!Array.isArray(index?.libraries)) throw new Error(`${join(directory, 'index.json')} lists no libraries`) - const libraries = index.libraries.map((/** @type {IndexEntry} */ entry) => ({ ...entry, url: pathToFileURL(join(directory, `${entry.folder}.h5p`)).href })) - return new Pack(libraries, directory) - } - const { pack } = await import('@missing-elements/h5p-libraries') - return new Pack(/** @type {IndexEntry[]} */ (pack.libraries), '@missing-elements/h5p-libraries') -} - -/** The pinned pack's version, for messages and reports. */ -export async function packVersion() { - const url = import.meta.resolve('@missing-elements/h5p-libraries/package.json') - return JSON.parse(await readFile(fileURLToPath(url), 'utf8')).version -} diff --git a/packages/author/lib/types.mjs b/packages/author/lib/types.mjs index e9ba6f69..a95998b6 100644 --- a/packages/author/lib/types.mjs +++ b/packages/author/lib/types.mjs @@ -1,4 +1,4 @@ -import { randomUUID } from 'node:crypto' +import { newId } from './generator.mjs' /** * What a content type needs beyond its semantics. H5P's editor has a widget of its own for some @@ -477,7 +477,7 @@ const gameMap = { const value = element.telemetry?.[key] if (value !== undefined && !(typeof value === 'string' && value.trim() !== '' && Number(value) >= 0 && Number(value) <= 100)) context.report(`${at}.telemetry.${key}`, 'expected a number from 0 to 100, percent of the map') } - if (element.id === undefined) element.id = randomUUID() + if (element.id === undefined) element.id = newId() if (element.type === undefined) element.type = element.specialStageType ? 'special-stage' : 'stage' if (!Array.isArray(element.neighbors)) element.neighbors = [] element.neighbors.forEach((/** @type {unknown} */ neighbor, /** @type {number} */ j) => { diff --git a/packages/author/package.json b/packages/author/package.json index d50106e3..c33d06b2 100644 --- a/packages/author/package.json +++ b/packages/author/package.json @@ -24,6 +24,14 @@ "types": "./types/author.d.mts", "default": "./lib/author.mjs" }, + "./core": { + "types": "./types/core.d.mts", + "default": "./lib/core.mjs" + }, + "./browser": { + "types": "./types/browser.d.mts", + "default": "./lib/browser.mjs" + }, "./package.json": "./package.json" }, "files": [ @@ -42,8 +50,8 @@ "agent-skill" ], "scripts": { - "test": "pnpm --filter @missing-elements/h5p-offline-player build:plan && vitest run", - "test:unit": "pnpm --filter @missing-elements/h5p-offline-player build:plan && vitest run", + "test": "pnpm --filter @missing-elements/h5p-offline-player build && vitest run", + "test:unit": "pnpm --filter @missing-elements/h5p-offline-player build:plan && vitest run --project node", "typecheck": "pnpm types && tsc -p tsconfig.json", "sync:skill": "node scripts/sync-skill.mjs", "prepack": "node -e \"require('node:fs').copyFileSync('../../LICENSE', 'LICENSE')\" && pnpm types", @@ -57,7 +65,11 @@ "@zip.js/zip.js": "^2.24.0" }, "devDependencies": { + "@missing-elements/h5p-verify": "workspace:*", "@types/node": "^26.6.5", + "@vitest/browser": "^5.0.5", + "@vitest/browser-playwright": "^5.0.5", + "playwright": "^1.64.1", "typescript": "^7.0.2", "vitest": "^5.0.5" }, diff --git a/packages/author/scripts/check-pack-published.mjs b/packages/author/scripts/check-pack-published.mjs index cfe50372..f0869ea6 100644 --- a/packages/author/scripts/check-pack-published.mjs +++ b/packages/author/scripts/check-pack-published.mjs @@ -5,7 +5,9 @@ * install, and one built against other parts would check them against an index npm does not * serve. The player, whose `./plan` the build imports: a release made before the player's would * not install either, and one pinned to a player published before `./plan` existed would install - * and fail on its first import, so the version on npm has to export it. + * and fail on its first import, so the version on npm has to export it. And the normalizer, a + * range from the version packages/normalize has, whose `./crc32` the page's build imports (#108): + * that version has to be on npm with it, or a page's build fails on its first import. */ import { readFile } from 'node:fs/promises' import { resolve } from 'node:path' @@ -40,3 +42,15 @@ if (!(await published.json()).exports?.['./plan']) { process.exit(1) } console.log(`[player] @missing-elements/h5p-offline-player@${player} is published, and exports ./plan`) + +const normalize = await versionOf('normalize') +const normalizer = await fetch(`https://registry.npmjs.org/@missing-elements%2fh5p-normalize/${normalize}`) +if (!normalizer.ok) { + console.error(`[normalize] @missing-elements/h5p-normalize@${normalize} is not on npm (${normalizer.status}). Publish it from packages/normalize first: the author package depends on it, for its ./crc32.`) + process.exit(1) +} +if (!(await normalizer.json()).exports?.['./crc32']) { + console.error(`[normalize] @missing-elements/h5p-normalize@${normalize} on npm does not export ./crc32, which the page's build imports: bump the normalizer's version, publish it, and publish the author package against that.`) + process.exit(1) +} +console.log(`[normalize] @missing-elements/h5p-normalize@${normalize} is published, and exports ./crc32`) diff --git a/packages/author/tests/browser/page.test.ts b/packages/author/tests/browser/page.test.ts new file mode 100644 index 00000000..ec490d6b --- /dev/null +++ b/packages/author/tests/browser/page.test.ts @@ -0,0 +1,80 @@ +import { pack } from '@missing-elements/h5p-libraries' +import { describe, expect, it } from 'vitest' +import { commands } from 'vitest/browser' +import { buildPackage, openPack } from '../../lib/browser.mjs' + +// The page's build in Chromium, as a page runs it: the pack a bundler emits, a picture the page +// was given as a `File`, the package back as a `Blob`. Its bytes go to Node, where the verifier +// plays them untrusted, as a site that did not make them plays them (`playUntrusted`, in +// vitest.config.ts). tests/page.test.ts runs the same build in Node, against the build there. + +declare module 'vitest/browser' { + interface BrowserCommands { + playUntrusted: (name: string, base64: string) => Promise<{ + verdict: 'pass' | 'fail', reasons: string[], warnings: string[], errors: unknown[], state: string, untrusted: boolean, + contentCheck: { notes: string[], changes: unknown[] | null } | null + }> + } +} + +const META = { title: 'Rivers', language: 'en', license: 'CC BY', authors: [{ name: 'Ann Author', role: 'Author' }] } +const BLANKS = { ...META, type: 'H5P.Blanks', params: { text: 'Fill in the names of the rivers.', questions: ['The *Danube* reaches the Black Sea.', 'Paris lies on the *Seine*.'] } } +const QUIZ = { + ...META, + type: 'H5P.QuestionSet', + params: { + passPercentage: 50, + questions: [ + { + library: 'H5P.MultiChoice', + metadata: { title: 'The longest river' }, + params: { + media: { type: { library: 'H5P.Image', params: { file: { path: 'volga.png' }, alt: 'The Volga at Yaroslavl' } } }, + question: 'Which river is the longest in Europe?', + answers: [{ text: 'Volga', correct: true }, { text: 'Danube', correct: false }] + } + }, + { library: 'H5P.TrueFalse', metadata: { title: 'The Rhine' }, params: { question: 'The Rhine flows into the North Sea.', correct: 'true' } } + ] + } +} + +/** A picture as a page is given one: a file the person chose, here drawn. */ +async function picture(width: number, height: number, name: string) { + const canvas = new OffscreenCanvas(width, height) + const context = canvas.getContext('2d')! + context.fillStyle = '#2a6f97' + context.fillRect(0, 0, width, height) + return new File([await canvas.convertToBlob({ type: 'image/png' })], name, { type: 'image/png' }) +} + +/** A `Blob` as base64, which a command's arguments carry to Node. */ +function base64(blob: Blob) { + return new Promise((resolve, reject) => { + const reader = new FileReader() + reader.onload = () => resolve(String(reader.result).slice(String(reader.result).indexOf(',') + 1)) + reader.onerror = () => reject(reader.error) + reader.readAsDataURL(blob) + }) +} + +describe('the build in a page', () => { + it('builds H5P.Blanks and a Question Set with a picture, which play untrusted with nothing changed by the content check', async () => { + expect(isSecureContext).toBe(true) + const libraries = await openPack(pack) + try { + const blanks = await buildPackage(BLANKS, { libraries }) + const quiz = await buildPackage(QUIZ, { libraries, files: { 'volga.png': await picture(480, 270, 'IMG_0412.png') } }) + expect(quiz.media).toEqual(['images/volga.png']) + expect((quiz.content as any).questions[0].params.media.type.params.file).toMatchObject({ path: 'images/volga.png', mime: 'image/png', width: 480, height: 270 }) + for (const [name, built] of [['blanks.h5p', blanks], ['quiz.h5p', quiz]] as const) { + expect(built.notes, name).toEqual([]) + const report = await commands.playUntrusted(name, await base64(built.blob)) + expect(report, name).toMatchObject({ verdict: 'pass', reasons: [], warnings: [], errors: [], state: 'ready', untrusted: true }) + expect(report.contentCheck, name).toEqual({ notes: [], changes: [] }) + } + } finally { + await libraries.close() + } + }) +}) diff --git a/packages/author/tests/build.test.ts b/packages/author/tests/build.test.ts index caf58652..d7a2551a 100644 --- a/packages/author/tests/build.test.ts +++ b/packages/author/tests/build.test.ts @@ -11,6 +11,7 @@ import { afterAll, beforeAll, describe, expect, it } from 'vitest' import { buildPackage, librariesIn, SourceError } from '../lib/author.mjs' import { readSource } from '../lib/build.mjs' import { MediaStore } from '../lib/media.mjs' +import { nodeMedia } from '../lib/node.mjs' import { quizProblems } from '../lib/quiz.mjs' import { jpeg, mp4, png } from './helpers/media' @@ -478,7 +479,7 @@ describe('media', () => { it('gives a download up once nothing has arrived for a while, however long a slow one takes', async () => { const reported: string[] = [] const report = (path: string, message: string) => reported.push(`${path}: ${message}`) - const store = new MediaStore({ base: dir, temporary: dir, idle: 300 }) + const store = new MediaStore(nodeMedia({ base: dir, temporary: dir }), { idle: 300 }) expect(await store.resolve({ type: 'image' }, { path: `${origin}/trickle.png` }, 'params.slow', report)).toMatchObject({ path: 'images/trickle.png', width: 40, height: 30 }) expect(await store.resolve({ type: 'image' }, { path: `${origin}/stalled.png` }, 'params.silent', report)).toEqual({ path: `${origin}/stalled.png`, copyright: { license: 'U' } }) expect(reported).toEqual([`params.silent.path: ${origin}/stalled.png could not be downloaded: nothing arrived for 0.3 s`]) diff --git a/packages/author/tests/package-types.ts b/packages/author/tests/package-types.ts index 65506b2f..a8e7f7e3 100644 --- a/packages/author/tests/package-types.ts +++ b/packages/author/tests/package-types.ts @@ -6,6 +6,10 @@ import { } from '@missing-elements/h5p-author' // @ts-expect-error The source reader is internal: a helper the tests import from its module. import { readSource } from '@missing-elements/h5p-author' +import { buildPackage as buildInPage, openPack as openInPage, type BuildResult as PageBuildResult } from '@missing-elements/h5p-author/browser' +import { checkContent as checkBeside, ContentGenerator as Generator, loadLibraries as loadBeside, type IndexEntry } from '@missing-elements/h5p-author/core' +// @ts-expect-error The core builds nothing: a build is Node's or a page's. +import { buildPackage as noBuild } from '@missing-elements/h5p-author/core' const source: Source = { type: 'H5P.TrueFalse', title: 'T', language: 'en', license: 'U', authors: [{ name: 'A', role: 'Author' }], params: {} } export async function check(): Promise<[BuildResult, string, string]> { @@ -61,3 +65,20 @@ export async function edit(names: string[], read: (name: string) => Promise { + const libraries = await openInPage(pack) + try { + const built: PageBuildResult = await buildInPage(source, { libraries, files: { 'volga.png': picture } }) + // @ts-expect-error A page's build writes no file: it takes no output path. + await buildInPage(source, 'out.h5p') + // The core takes the page's pack as it takes Node's: a tool beside the page's build checks what it changes. + const content = { type: 'H5P.Blanks 1.14', language: 'en', params: built.content } + const checked = checkBeside(await loadBeside(libraries, content), content) + void [checked.problems, new Generator(new Map()), noBuild, built.notes.map(({ path }) => path)] + return built.blob + } finally { + await libraries.close() + } +} diff --git a/packages/author/tests/page.test.ts b/packages/author/tests/page.test.ts new file mode 100644 index 00000000..5e145e66 --- /dev/null +++ b/packages/author/tests/page.test.ts @@ -0,0 +1,223 @@ +import { createHash } from 'node:crypto' +import { mkdtemp, readdir, readFile, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' +import { fileURLToPath, pathToFileURL } from 'node:url' +import { crc32 } from 'node:zlib' +import { Uint8ArrayReader, Uint8ArrayWriter, ZipReader, type FileEntry } from '@zip.js/zip.js' +import { pack } from '@missing-elements/h5p-libraries' +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' +import { buildPackage as buildInNode } from '../lib/author.mjs' +import { buildPackage, openPack, SourceError } from '../lib/browser.mjs' +import { png } from './helpers/media' + +// The page's build, run in Node with the pack's parts read from disk as a page fetches them from +// its server. tests/browser/ runs it in Chromium and plays what it writes. +const parts = resolve(import.meta.dirname, '../../libraries/parts') +const META = { title: 'Rivers', language: 'en', license: 'CC BY', authors: [{ name: 'Ann Author', role: 'Author' }] } +const BLANKS = { ...META, type: 'H5P.Blanks', params: { questions: ['The *Danube* reaches the Black Sea.'] } } +const image = (path: string) => ({ library: 'H5P.Image', params: { file: { path }, alt: 'The Volga at Yaroslavl' } }) +const QUIZ = { + ...META, + type: 'H5P.QuestionSet', + params: { + questions: [{ + library: 'H5P.MultiChoice', + metadata: { title: 'The longest river' }, + params: { + media: { type: image('volga.png') }, + question: 'Which river is the longest in Europe?', + answers: [{ text: 'Volga', correct: true }, { text: 'Danube', correct: false }] + } + }] + } +} +let dir: string +let opened: Awaited> +let fetched: string[] + +/** A part, as the page's server would serve it. */ +async function fromDisk(input: RequestInfo | URL) { + fetched.push(String(input)) + return new Response(new Uint8Array(await readFile(fileURLToPath(String(input))))) +} + +beforeAll(async () => { + dir = await mkdtemp(join(tmpdir(), 'h5p-author-page-')) + fetched = [] + opened = await openPack(pack, { fetch: fromDisk }) +}) +afterAll(async () => { + await opened?.close() + if (dir) await rm(dir, { recursive: true, force: true }) +}) + +/** Every file of a package, and its entries. */ +async function archive(bytes: Uint8Array) { + const reader = new ZipReader(new Uint8ArrayReader(bytes), { checkSignature: true }) + try { + const entries = (await reader.getEntries()).filter((entry) => !entry.directory) as FileEntry[] + const files = new Map() + for (const entry of entries) files.set(entry.filename, await entry.getData(new Uint8ArrayWriter())) + return { files, entries } + } finally { + await reader.close() + } +} + +/** An entry's local header, as the player reads it on a host without `Range`. */ +function localHeader(bytes: Uint8Array, offset: number) { + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength) + return { + signature: view.getUint32(offset, true), flags: view.getUint16(offset + 6, true), method: view.getUint16(offset + 8, true), + crc32: view.getUint32(offset + 14, true), compressedSize: view.getUint32(offset + 18, true), size: view.getUint32(offset + 22, true) + } +} + +/** The problems a page's build finds, as `path: message`. */ +async function problems(source: unknown, options: Parameters[1]) { + const error = await buildPackage(source, options).then(() => null, (error: unknown) => error) + expect(error).toBeInstanceOf(SourceError) + return (error as SourceError).problems.map(({ path, message }) => `${path}: ${message}`) +} + +/** The modules an entry loads, by following its relative imports. */ +async function graph(entry: string, seen = new Map()) { + if (seen.has(entry)) return seen + const text = await readFile(entry, 'utf8') + const specifiers = [...text.matchAll(/^(?:import|export)\b[^'"]*?from\s+'([^']+)'/gm)].map(([, specifier]) => specifier) + seen.set(entry, specifiers) + for (const specifier of specifiers) if (specifier.startsWith('.')) await graph(resolve(entry, '..', specifier), seen) + return seen +} + +describe('the build in a page', () => { + it('writes what the build in Node writes: the manifest, the content, the libraries and their files', async () => { + const page = await buildPackage(BLANKS, { libraries: opened }) + const node = await buildInNode(BLANKS, join(dir, 'blanks.h5p')) + expect(page.blob.type).toBe('application/zip') + expect([page.manifest, page.content, page.libraries, page.notes, page.media]).toEqual([node.manifest, node.content, node.libraries, node.notes, node.media]) + // Each file by its name and hash, in order: comparing the bytes themselves with toEqual takes seconds. + const hashes = ({ files }: { files: Map }) => [...files].map(([name, bytes]) => [name, createHash('sha256').update(bytes).digest('hex')]) + const fromPage = await archive(new Uint8Array(await page.blob.arrayBuffer())) + const fromNode = await archive(new Uint8Array(await readFile(node.output))) + expect(hashes(fromPage)).toEqual(hashes(fromNode)) + }) + + it('stores media last, with their size and CRC in their local headers, and describes them as H5P records them', async () => { + const picture = png(480, 270) + const result = await buildPackage(QUIZ, { libraries: opened, files: { 'volga.png': new File([new Uint8Array(picture)], 'IMG_0412.png') } }) + expect(result.media).toEqual(['images/volga.png']) + expect(result.notes).toEqual([]) + const question = (result.content as any).questions[0] + expect(question.subContentId).toMatch(/^[0-9a-f-]{36}$/) + expect(question.params.media.type.params.file).toEqual({ path: 'images/volga.png', mime: 'image/png', width: 480, height: 270, copyright: { license: 'U' } }) + const bytes = new Uint8Array(await result.blob.arrayBuffer()) + const { files, entries } = await archive(bytes) + expect(entries.at(-1)!.filename).toBe('content/images/volga.png') + expect(files.get('content/images/volga.png')).toEqual(new Uint8Array(picture)) + // No data descriptor: the header says where the data ends. + expect(localHeader(bytes, entries.at(-1)!.offset)).toEqual({ signature: 0x04034b50, flags: 0, method: 0, crc32: crc32(picture), compressedSize: picture.length, size: picture.length }) + }) + + it('takes the files from a Map too, and names one the source names that the page did not give', async () => { + const given = new Map([['volga.png', new Blob([new Uint8Array(png(8, 8))])]]) + expect((await buildPackage(QUIZ, { libraries: opened, files: given })).media).toEqual(['images/volga.png']) + const elsewhere = { ...QUIZ, params: { questions: [{ ...QUIZ.params.questions[0], params: { ...QUIZ.params.questions[0].params, media: { type: image('rhine.png') } } }] } } + expect(await problems(elsewhere, { libraries: opened, files: given })).toEqual([ + 'params.questions[0].params.media.type.params.file.path: rhine.png: no such file among those given (volga.png)' + ]) + expect(await problems(QUIZ, { libraries: opened, files: { 'volga.png': 'a picture' as unknown as Blob } })).toEqual([ + 'params.questions[0].params.media.type.params.file.path: volga.png: is not a File or a Blob' + ]) + }) + + it('fetches each part once for a pack it was given, which stays open; one it was named it opens and closes', async () => { + const before = fetched.length + await buildPackage(BLANKS, { libraries: opened }) + expect(fetched.length).toBe(before) + const library = (await opened.open(opened.newest('H5P.Blanks')!)).library + expect(library.machineName).toBe('H5P.Blanks') + // Named by the pack export, the build fetches with the page's fetch, which in Node reads no file: URL. + await expect(buildPackage(BLANKS, { libraries: pack })).rejects.toThrow(/^Cannot fetch the library part H5P\.Blanks-1\.\d+ from file:/) + }) + + it('refuses a part whose bytes are not the ones the index names', async () => { + const damaged = await openPack(pack, { + fetch: async (input) => { + const response = await fromDisk(input) + if (!String(input).includes('/H5P.Blanks-')) return response + const bytes = new Uint8Array(await response.arrayBuffer()) + bytes[bytes.length >> 1] ^= 1 + return new Response(bytes) + } + }) + try { + await expect(buildPackage(BLANKS, { libraries: damaged })).rejects.toThrow(/^The library part H5P\.Blanks-1\.\d+ does not match the pack's index \(SHA-256\): serve the parts of the pack this index belongs to\.$/) + } finally { + await damaged.close() + } + }) + + it('reads a copy of the parts folder by its address, its index held to the player\'s shape', async () => { + const index = JSON.parse(await readFile(join(parts, 'index.json'), 'utf8')) + const site = 'https://pack.example/parts/' + const serving = (body: unknown) => async (input: RequestInfo | URL) => String(input) === `${site}index.json` + ? Response.json(body) + : fromDisk(String(input).replace(site, pathToFileURL(`${parts}/`).href)) + const [first, second] = index.libraries + // A folder that is not its library's own name, and a hash that is not hex, as the player drops them. + const tampered = { libraries: [...index.libraries, { ...first, folder: '../escape' }, { ...second, sha256: 'not a hash' }] } + const copy = await openPack('https://pack.example/parts', { fetch: serving(tampered) }) + try { + expect(copy.libraries).toHaveLength(index.libraries.length) + expect(copy.libraries[0]).toMatchObject({ folder: first.folder, url: `${site}${first.folder}.h5p` }) + const built = await buildPackage(BLANKS, { libraries: copy }) + expect(built.libraries).toEqual((await buildPackage(BLANKS, { libraries: opened })).libraries) + } finally { + await copy.close() + } + await expect(openPack(site, { fetch: serving({ libraries: [{ folder: '../escape' }] }) })).rejects.toThrow(`${site}index.json is not a library pack's index: it lists no library as the pack's index does`) + await expect(openPack(site, { fetch: async () => new Response('Not Found', { status: 404 }) })).rejects.toThrow(`${site}index.json is not a library pack's index: it answered 404`) + await expect(openPack(pathToFileURL(`${parts}/`))).rejects.toThrow('is not an http(s) address: a copy of the pack\'s parts/ folder is read over http(s)') + // @ts-expect-error neither a pack nor its address + await expect(openPack({ libraries: [{ folder: 'H5P.Blanks-1.14' }] })).rejects.toThrow('Name the library pack') + }) + + it('says what a failed download most likely is, in a page', async () => { + vi.stubGlobal('fetch', async () => { throw new TypeError('Failed to fetch') }) + try { + const remote = { ...QUIZ, params: { questions: [{ ...QUIZ.params.questions[0], params: { ...QUIZ.params.questions[0].params, media: { type: image('https://images.example/volga.png') } } }] } } + expect(await problems(remote, { libraries: opened })).toEqual([ + 'params.questions[0].params.media.type.params.file.path: https://images.example/volga.png could not be downloaded: Failed to fetch; a page reads a file from another site only where that site allows it (CORS), so give the file itself instead' + ]) + } finally { + vi.unstubAllGlobals() + } + }) + + it('needs a secure context, and says so', async () => { + vi.stubGlobal('crypto', { getRandomValues: (array: Uint8Array) => array }) + try { + const refused = 'A page builds packages in a secure context only, https, localhost or a Tauri window' + await expect(openPack(pack)).rejects.toThrow(refused) + await expect(buildPackage(BLANKS, { libraries: opened })).rejects.toThrow(refused) + } finally { + vi.unstubAllGlobals() + } + }) + + it('imports nothing of Node, nor the normalizer, which is Node\'s, through ./core or ./browser', async () => { + const lib = resolve(import.meta.dirname, '../lib') + for (const entry of ['core.mjs', 'browser.mjs']) { + const modules = await graph(join(lib, entry)) + const outside = [...modules.values()].flat().filter((specifier) => !specifier.startsWith('.')) + expect(outside.filter((specifier) => specifier.startsWith('node:') || specifier === '@missing-elements/h5p-normalize'), entry).toEqual([]) + expect([...modules.keys()].map((path) => path.slice(lib.length + 1))).not.toContain('node.mjs') + } + // node.mjs is the one module that does, and the Node entry is the only one that loads it. + const imports = await Promise.all((await readdir(lib)).map(async (name) => [name, (await graph(join(lib, name), new Map())).get(join(lib, name))!] as const)) + expect(imports.filter(([, specifiers]) => specifiers.some((specifier) => specifier.startsWith('node:'))).map(([name]) => name)).toEqual(['node.mjs']) + expect(imports.filter(([, specifiers]) => specifiers.includes('./node.mjs')).map(([name]) => name)).toEqual(['author.mjs']) + }) +}) diff --git a/packages/author/tsconfig.types.json b/packages/author/tsconfig.types.json index ec74a5b8..1cb50803 100644 --- a/packages/author/tsconfig.types.json +++ b/packages/author/tsconfig.types.json @@ -8,5 +8,5 @@ "rootDir": "lib", "outDir": "types" }, - "include": ["lib/author.mjs"] + "include": ["lib/author.mjs", "lib/core.mjs", "lib/browser.mjs"] } diff --git a/packages/author/vitest.config.ts b/packages/author/vitest.config.ts new file mode 100644 index 00000000..41fdf0ea --- /dev/null +++ b/packages/author/vitest.config.ts @@ -0,0 +1,59 @@ +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' +import { verifyPackage } from '@missing-elements/h5p-verify' +import { playwright } from '@vitest/browser-playwright' +import { configDefaults, defineConfig } from 'vitest/config' +import type { BrowserCommand } from 'vitest/node' + +// The pack the build pins, which the verifier plays an untrusted package against, as a site with it does. +const parts = resolve(import.meta.dirname, '../libraries/parts') + +/** + * A package the page's build wrote in Chromium, handed to Node as base64 and played by the verifier + * untrusted, as a site that did not make it plays it: the report's verdict, what it warns of, and + * what the content check changed. + */ +const playUntrusted: BrowserCommand<[name: string, base64: string]> = async (_context, name, base64) => { + const dir = await mkdtemp(join(tmpdir(), 'h5p-author-page-')) + try { + const file = join(dir, name) + await writeFile(file, Buffer.from(base64, 'base64')) + const { verdict, reasons, warnings, errors, state, untrusted, contentCheck } = await verifyPackage({ file, untrusted: true, libraries: parts, settle: 500 }) + return { verdict, reasons, warnings, errors, state, untrusted, contentCheck } + } finally { + await rm(dir, { recursive: true, force: true }) + } +} + +export default defineConfig({ + test: { + projects: [ + { + test: { + name: 'node', + environment: 'node', + include: ['tests/**/*.test.ts'], + exclude: [...configDefaults.exclude, 'tests/browser/**'] + } + }, + { + // Served from its own folder, so that each part's URL, a `new URL` beside the pack's index, + // is a file there rather than in Vite's bundle of dependencies. + optimizeDeps: { exclude: ['@missing-elements/h5p-libraries'] }, + test: { + name: 'browser', + include: ['tests/browser/**/*.test.ts'], + testTimeout: 180_000, + browser: { + enabled: true, + headless: true, + provider: playwright(), + instances: [{ browser: 'chromium' }], + commands: { playUntrusted } + } + } + } + ] + } +}) diff --git a/packages/embed/lib/crc32.mjs b/packages/embed/lib/crc32.mjs deleted file mode 100644 index bf1eeeb9..00000000 --- a/packages/embed/lib/crc32.mjs +++ /dev/null @@ -1,36 +0,0 @@ -import zlib from 'node:zlib' - -/** @type {Uint32Array | undefined} */ -let table - -/** - * @param {Uint8Array} bytes - * @param {number} previous - */ -function tableCrc32(bytes, previous) { - if (!table) { - table = new Uint32Array(256) - for (let n = 0; n < 256; n++) { - let c = n - for (let k = 0; k < 8; k++) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1 - table[n] = c >>> 0 - } - } - let crc = ~previous >>> 0 - for (let i = 0; i < bytes.length; i++) crc = table[(crc ^ bytes[i]) & 0xff] ^ (crc >>> 8) - return ~crc >>> 0 -} - -/** - * CRC-32 of `bytes`, continuing from `previous`. Native from Node 22.2 on; a table before that. - * A namespace import rather than a named one, because a named import of a builtin's missing - * export fails at link time and would take the whole script down on an older Node. - * - * A copy of the normalizer's `lib/crc32.mjs`; keep the two in step. - * - * @type {(bytes: Uint8Array, previous?: number) => number} - */ -export const crc32 = - typeof zlib.crc32 === 'function' - ? (bytes, previous = 0) => zlib.crc32(bytes, previous) - : (bytes, previous = 0) => tableCrc32(bytes, previous) diff --git a/packages/embed/lib/scorm.mjs b/packages/embed/lib/scorm.mjs index a2050f73..5486a73d 100644 --- a/packages/embed/lib/scorm.mjs +++ b/packages/embed/lib/scorm.mjs @@ -5,9 +5,9 @@ import { basename, dirname, extname, join, resolve, sep } from 'node:path' import { Writable } from 'node:stream' import { fileURLToPath, pathToFileURL } from 'node:url' import { BlobReader, TextReader, TextWriter, ZipReader, ZipWriter, configure } from '@zip.js/zip.js' +import { crc32 } from '@missing-elements/h5p-normalize/crc32' import { dependenciesIn, indexLibraryFolders, librariesNamedIn, ownLibraries, planPack, reachableFolder, walkDependencies } from '@missing-elements/h5p-offline-player/plan' import { EmbedError, PLAYER_FILES, installed } from './build.mjs' -import { crc32 } from './crc32.mjs' /** * Writes a SCORM 1.2 package that plays one `.h5p` in an LMS and reports its result there: the diff --git a/packages/embed/package.json b/packages/embed/package.json index e1f56a21..5d596365 100644 --- a/packages/embed/package.json +++ b/packages/embed/package.json @@ -61,11 +61,12 @@ "test": "pnpm --filter @missing-elements/h5p-offline-player build && vitest run", "typecheck": "pnpm types && tsc -p tsconfig.json", "prepack": "node -e \"require('node:fs').copyFileSync('../../LICENSE', 'LICENSE')\" && pnpm types", - "prepublishOnly": "node -e \"if (!/^pnpm\\//.test(process.env.npm_config_user_agent || '')) { console.error('Publish with pnpm publish: npm would ship the workspace: dependency ranges as they are, and the package would not install.'); process.exit(1) }\" && node scripts/check-player-published.mjs && pnpm typecheck && pnpm test", + "prepublishOnly": "node -e \"if (!/^pnpm\\//.test(process.env.npm_config_user_agent || '')) { console.error('Publish with pnpm publish: npm would ship the workspace: dependency ranges as they are, and the package would not install.'); process.exit(1) }\" && node scripts/check-published.mjs && pnpm typecheck && pnpm test", "types": "pnpm --filter @missing-elements/h5p-offline-player exec tsc -b && node -e \"require('node:fs').rmSync('types', { recursive: true, force: true })\" && tsc -p tsconfig.types.json" }, "dependencies": { "@missing-elements/h5p-libraries": "workspace:^", + "@missing-elements/h5p-normalize": "workspace:^", "@missing-elements/h5p-offline-player": "workspace:*", "@missing-elements/h5p-runtime": "workspace:^", "@zip.js/zip.js": "^2.24.0" diff --git a/packages/embed/scripts/check-player-published.mjs b/packages/embed/scripts/check-player-published.mjs deleted file mode 100644 index cbef3385..00000000 --- a/packages/embed/scripts/check-player-published.mjs +++ /dev/null @@ -1,21 +0,0 @@ -/** - * Refuses to publish the embed package while the player it pins is not on npm, or is there without - * `./plan`, which the SCORM writer imports. The dependency is `workspace:*` since #109, published - * as the exact version packages/player has, because `./plan` is not covered by - * docs/compatibility.md: a release made before the player's would not install, and one pinned to a - * player published before `./plan` existed would install and fail on its first import. - */ -import { readFile } from 'node:fs/promises' -import { resolve } from 'node:path' - -const { version } = JSON.parse(await readFile(resolve(import.meta.dirname, '..', '..', 'player', 'package.json'), 'utf8')) -const response = await fetch(`https://registry.npmjs.org/@missing-elements%2fh5p-offline-player/${version}`) -if (!response.ok) { - console.error(`[player] @missing-elements/h5p-offline-player@${version} is not on npm (${response.status}). Publish it from packages/player first: the embed package pins it, for its ./plan.`) - process.exit(1) -} -if (!(await response.json()).exports?.['./plan']) { - console.error(`[player] @missing-elements/h5p-offline-player@${version} on npm does not export ./plan, which the SCORM writer imports: bump the player's version, publish it, and publish the embed package against that.`) - process.exit(1) -} -console.log(`[player] @missing-elements/h5p-offline-player@${version} is published, and exports ./plan`) diff --git a/packages/embed/scripts/check-published.mjs b/packages/embed/scripts/check-published.mjs new file mode 100644 index 00000000..739ae261 --- /dev/null +++ b/packages/embed/scripts/check-published.mjs @@ -0,0 +1,34 @@ +/** + * Refuses to publish the embed package while what it imports from two packages of this repository is + * not on npm: the player's `./plan`, which the SCORM writer plans with, and the normalizer's + * `./crc32`, which it writes the stored entries' CRC with. The player is pinned exactly, `workspace:*` + * since #109, because `./plan` is not covered by docs/compatibility.md: a release made before the + * player's would not install, and one pinned to a player published before `./plan` existed would + * install and fail on its first import. The normalizer is `workspace:^`, published as a range from + * the version packages/normalize has, so that version has to be on npm with `./crc32` (#108). + */ +import { readFile } from 'node:fs/promises' +import { resolve } from 'node:path' + +/** + * @param {string} folder the package's folder under packages/ + * @param {string} name + * @param {string} subpath the export the embed package imports + * @param {string} why what imports it + */ +async function published(folder, name, subpath, why) { + const { version } = JSON.parse(await readFile(resolve(import.meta.dirname, '..', '..', folder, 'package.json'), 'utf8')) + const response = await fetch(`https://registry.npmjs.org/${name.replaceAll('/', '%2f')}/${version}`) + if (!response.ok) { + console.error(`[${folder}] ${name}@${version} is not on npm (${response.status}). Publish it from packages/${folder} first: the embed package depends on it, for its ${subpath}.`) + process.exit(1) + } + if (!(await response.json()).exports?.[subpath]) { + console.error(`[${folder}] ${name}@${version} on npm does not export ${subpath}, which ${why} imports: bump its version, publish it, and publish the embed package against that.`) + process.exit(1) + } + console.log(`[${folder}] ${name}@${version} is published, and exports ${subpath}`) +} + +await published('player', '@missing-elements/h5p-offline-player', './plan', 'the SCORM writer') +await published('normalize', '@missing-elements/h5p-normalize', './crc32', 'the SCORM writer') diff --git a/packages/normalize/README.md b/packages/normalize/README.md index 85924c53..5f946c20 100644 --- a/packages/normalize/README.md +++ b/packages/normalize/README.md @@ -48,7 +48,9 @@ const report = await normalizeArchive({ TypeScript declarations ship with the package: `Report` and `EntryResult` describe what it returns. -Node only, not a browser. Why deflated video cannot stream: the demo's +Node only, not a browser. `crc32(bytes, previous)` from `@missing-elements/h5p-normalize/crc32` +is the CRC-32 it writes entries with, for code that writes zips beside it, in a page too: zlib's in +Node, a table elsewhere. Why deflated video cannot stream: the demo's [normalize page](https://github.com/missing-elements/h5p-offline-player/blob/main/apps/demo/demo/normalize.html). ## Licence diff --git a/packages/normalize/lib/crc32-table.mjs b/packages/normalize/lib/crc32-table.mjs new file mode 100644 index 00000000..ddf9c3ee --- /dev/null +++ b/packages/normalize/lib/crc32-table.mjs @@ -0,0 +1,25 @@ +/** @type {Uint32Array | undefined} */ +let table + +/** + * CRC-32 of `bytes`, continuing from `previous`, by a table: no Node import, so a page can import + * it, as `@missing-elements/h5p-normalize/crc32` resolves there. Node takes `lib/crc32.mjs`, which + * runs zlib's where Node has it. + * + * @param {Uint8Array} bytes + * @param {number} [previous] + * @returns {number} + */ +export function crc32(bytes, previous = 0) { + if (!table) { + table = new Uint32Array(256) + for (let n = 0; n < 256; n++) { + let c = n + for (let k = 0; k < 8; k++) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1 + table[n] = c >>> 0 + } + } + let crc = ~previous >>> 0 + for (let i = 0; i < bytes.length; i++) crc = table[(crc ^ bytes[i]) & 0xff] ^ (crc >>> 8) + return ~crc >>> 0 +} diff --git a/packages/normalize/lib/crc32.mjs b/packages/normalize/lib/crc32.mjs index 3ca4671b..71059201 100644 --- a/packages/normalize/lib/crc32.mjs +++ b/packages/normalize/lib/crc32.mjs @@ -1,30 +1,11 @@ import zlib from 'node:zlib' - -/** @type {Uint32Array | undefined} */ -let table - -/** - * @param {Uint8Array} bytes - * @param {number} previous - */ -function tableCrc32(bytes, previous) { - if (!table) { - table = new Uint32Array(256) - for (let n = 0; n < 256; n++) { - let c = n - for (let k = 0; k < 8; k++) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1 - table[n] = c >>> 0 - } - } - let crc = ~previous >>> 0 - for (let i = 0; i < bytes.length; i++) crc = table[(crc ^ bytes[i]) & 0xff] ^ (crc >>> 8) - return ~crc >>> 0 -} +import { crc32 as tableCrc32 } from './crc32-table.mjs' /** * CRC-32 of `bytes`, continuing from `previous`. Native from Node 22.2 on; a table before that. * A namespace import rather than a named one, because a named import of a builtin's missing - * export fails at link time and would take the whole script down on an older Node. + * export fails at link time and would take the whole script down on an older Node. Exported as + * `@missing-elements/h5p-normalize/crc32` for Node; a page resolves that to `lib/crc32-table.mjs`. * * @type {(bytes: Uint8Array, previous?: number) => number} */ diff --git a/packages/normalize/package.json b/packages/normalize/package.json index 2dd2c784..f9f8bcae 100644 --- a/packages/normalize/package.json +++ b/packages/normalize/package.json @@ -24,6 +24,11 @@ "types": "./types/normalize.d.mts", "default": "./lib/normalize.mjs" }, + "./crc32": { + "types": "./types/crc32.d.mts", + "node": "./lib/crc32.mjs", + "default": "./lib/crc32-table.mjs" + }, "./package.json": "./package.json" }, "files": [ diff --git a/packages/normalize/tests/crc32.test.ts b/packages/normalize/tests/crc32.test.ts new file mode 100644 index 00000000..6b841613 --- /dev/null +++ b/packages/normalize/tests/crc32.test.ts @@ -0,0 +1,27 @@ +import { execFile } from 'node:child_process' +import { readFile } from 'node:fs/promises' +import { resolve } from 'node:path' +import { promisify } from 'node:util' +import { describe, expect, it } from 'vitest' +import { crc32 as native } from '../lib/crc32.mjs' +import { crc32 as table } from '../lib/crc32-table.mjs' + +const bytes = new Uint8Array(100_000).map((_, index) => (index * 7919) & 0xff) + +describe('crc32', () => { + it('by the table a page takes, is what zlib gives, continued across chunks too', () => { + // The check value every CRC-32 implementation is held to. + expect(table(new TextEncoder().encode('123456789'))).toBe(0xcbf43926) + expect(table(new Uint8Array())).toBe(0) + expect(table(bytes)).toBe(native(bytes)) + expect(table(bytes.subarray(40_000), table(bytes.subarray(0, 40_000)))).toBe(native(bytes)) + expect(native(bytes.subarray(40_000), native(bytes.subarray(0, 40_000)))).toBe(native(bytes)) + }) + + it('is zlib\'s for Node by the package\'s name, and the table for anything else', async () => { + const { stdout } = await promisify(execFile)(process.execPath, ['--input-type=module', '-e', "console.log(import.meta.resolve('@missing-elements/h5p-normalize/crc32'))"], { cwd: resolve(import.meta.dirname, '..') }) + expect(stdout.trim()).toMatch(/\/lib\/crc32\.mjs$/) + const { exports } = JSON.parse(await readFile(resolve(import.meta.dirname, '../package.json'), 'utf8')) + expect(exports['./crc32'].default).toBe('./lib/crc32-table.mjs') + }) +}) diff --git a/packages/normalize/tests/package-types.ts b/packages/normalize/tests/package-types.ts index 855007d4..ae1af614 100644 --- a/packages/normalize/tests/package-types.ts +++ b/packages/normalize/tests/package-types.ts @@ -6,6 +6,8 @@ import { archiveFingerprint, formatBytes, normalizeArchive, type EntryResult, type Report } from '@missing-elements/h5p-normalize' // @ts-expect-error a helper the tests use, not part of the package's API (#60) import { orderEntries } from '@missing-elements/h5p-normalize' +// For the packages of this repository that write zips, h5p-author's page build among them. +import { crc32 } from '@missing-elements/h5p-normalize/crc32' export async function asACallerWould(): Promise { const seen: EntryResult[] = [] @@ -17,6 +19,10 @@ export async function asACallerWould(): Promise { // @ts-expect-error the source is named `input` await normalizeArchive({ file: 'in.h5p' }) - void [moved, saved, orderEntries] + const checked: number = crc32(new Uint8Array(4), crc32(new Uint8Array(4))) + // @ts-expect-error bytes, not text + crc32('h5p.json') + + void [moved, saved, orderEntries, checked] return `${report.revision} ${revision}` } diff --git a/packages/normalize/tsconfig.types.json b/packages/normalize/tsconfig.types.json index 88fee24d..454d11b7 100644 --- a/packages/normalize/tsconfig.types.json +++ b/packages/normalize/tsconfig.types.json @@ -8,5 +8,5 @@ "rootDir": "lib", "outDir": "types" }, - "include": ["lib/normalize.mjs"] + "include": ["lib/normalize.mjs", "lib/crc32.mjs"] } diff --git a/packages/player/src/plan.ts b/packages/player/src/plan.ts index 19534282..014e9f9f 100644 --- a/packages/player/src/plan.ts +++ b/packages/player/src/plan.ts @@ -3,10 +3,10 @@ * with, for the packages that write what it plays, so that they run the player's own rules rather * than copies of them: which libraries content names, an addon's patterns and what they match, * which folder serves a library and the walk through what the libraries need in turn, what a - * package carries of its libraries, and which parts of the library pack it needs. Pure and free of - * zip.js, for Node and a page alike. Not named in docs/compatibility.md, so it may change in any - * release: a package that imports it depends on an exact version of the player (h5p-author since - * #106, embed since #109). + * package carries of its libraries, which parts of the library pack it needs, and a pack's index + * held to its shape. Pure and free of zip.js, for Node and a page alike. Not named in + * docs/compatibility.md, so it may change in any release: a package that imports it depends on an + * exact version of the player (h5p-author since #106, embed since #109). */ export { addonPatterns, dependenciesIn, librariesNamedIn, textMatches, type LibraryDependency, type PackageManifest } from './shared/content-libraries.js' @@ -20,5 +20,6 @@ export { walkDependencies, type LibraryFolder } from './shared/library-folders.js' +export { packLibrariesIn } from './shared/pack.js' export { ownLibraries, planPack, type OwnLibrary, type PlanInput, type ProvidedLibrary } from './sw/pack-plan.js' export type { PackDependency, PackLibrary, PackPart, PackPlan, RefreshedLibrary } from './shared/protocol.js' diff --git a/packages/player/tests/package-types.ts b/packages/player/tests/package-types.ts index 1ed4f373..4b6dd990 100644 --- a/packages/player/tests/package-types.ts +++ b/packages/player/tests/package-types.ts @@ -8,7 +8,7 @@ */ import { H5PPlayerElement, type PackageMetadata, type PlayerErrorDetail, type ReadyDetail } from '@missing-elements/h5p-offline-player' import { mountH5P } from '@missing-elements/h5p-offline-player/sw' -import { librariesNamedIn, ownLibraries, planPack, reachableFolder, walkDependencies, type PackPlan } from '@missing-elements/h5p-offline-player/plan' +import { librariesNamedIn, ownLibraries, packLibrariesIn, planPack, reachableFolder, walkDependencies, type PackPlan } from '@missing-elements/h5p-offline-player/plan' export function asACallerWould(worker: ServiceWorkerGlobalScope): string[] { const player: H5PPlayerElement | null = document.querySelector('h5p-player') @@ -24,6 +24,8 @@ export function asACallerWould(worker: ServiceWorkerGlobalScope): string[] { const folder: string | undefined = reachableFolder(roots[0], names, new Map()) const walked: Promise<{ missing: { machineName: string }[] }> = walkDependencies(roots, (dependency) => reachableFolder(dependency, names, new Map()), async (located: string) => ({ located })) const own: Promise<{ folder: string; major: number | null }[]> = ownLibraries(['H5P.Blanks-1.14']) + // A pack's index read from the network, held to its shape, as h5p-author's page build reads one. + const index: { folder: string; url: string; sha256: string }[] = packLibrariesIn(JSON.parse('[]')) // @ts-expect-error not an error code const code: PlayerErrorDetail['code'] = 'offline' @@ -34,6 +36,6 @@ export function asACallerWould(worker: ServiceWorkerGlobalScope): string[] { // @ts-expect-error the title is a string const title: PackageMetadata['title'] = 1 - void [player, code, folders, source, title, plan, parts, folder, walked, own] + void [player, code, folders, source, title, plan, parts, folder, walked, own, index] return player?.libraryBundle?.refreshed?.map(({ library, to }) => `${library} ${to}`) ?? [] } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b1819892..d4a56e7b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -100,9 +100,21 @@ importers: specifier: ^2.24.0 version: 2.24.0 devDependencies: + '@missing-elements/h5p-verify': + specifier: workspace:* + version: link:../verify '@types/node': specifier: ^26.6.5 version: 26.6.5 + '@vitest/browser': + specifier: ^5.0.5 + version: 5.0.5(vite@8.3.4(@types/node@26.6.5)(esbuild@0.28.2))(vitest@5.0.5) + '@vitest/browser-playwright': + specifier: ^5.0.5 + version: 5.0.5(playwright@1.64.1)(vite@8.3.4(@types/node@26.6.5)(esbuild@0.28.2))(vitest@5.0.5) + playwright: + specifier: ^1.64.1 + version: 1.64.1 typescript: specifier: ^7.0.2 version: 7.0.2 @@ -134,6 +146,9 @@ importers: '@missing-elements/h5p-libraries': specifier: workspace:^ version: link:../libraries + '@missing-elements/h5p-normalize': + specifier: workspace:^ + version: link:../normalize '@missing-elements/h5p-offline-player': specifier: workspace:* version: link:../player