From 107cb4baf80cb2dd53c169665935abedf6d9b14a Mon Sep 17 00:00:00 2001 From: Aleksandr Shevchuk Date: Mon, 9 Nov 2026 21:28:46 +0200 Subject: [PATCH 1/3] feat(author): a library bundle per content type, for a build where npm is unreachable `h5p-author bundle ` writes the parts a build of one content type can read: what the type can hold, as the build's own rules plan it (planPack, installClosure), with the pack's whole index and its licence list, and no addon. `bundle --all ` writes one per content type. Where npx cannot install, as in ChatGPT's sandbox, the skill builds from a type's bundle and the command line as one file, served by embed-my.github.io (embed-my/embed-my.github.io#14), and falls back to the build page's link. Closes #166 --- design/author.md | 23 +++- packages/author/README.md | 26 +++- packages/author/bin/h5p-author.mjs | 25 +++- packages/author/lib/author.mjs | 17 ++- packages/author/lib/bundle.mjs | 116 ++++++++++++++++ packages/author/lib/core.mjs | 7 +- packages/author/lib/generator.mjs | 2 +- packages/author/lib/node.mjs | 128 +++++++++++++++++- packages/author/tests/bundle.test.ts | 179 +++++++++++++++++++++++++ packages/author/tests/package-types.ts | 22 ++- skills/h5p-author/SKILL.md | 38 ++++-- 11 files changed, 550 insertions(+), 33 deletions(-) create mode 100644 packages/author/lib/bundle.mjs create mode 100644 packages/author/tests/bundle.test.ts diff --git a/design/author.md b/design/author.md index 12e0e09b..e7feb1f0 100644 --- a/design/author.md +++ b/design/author.md @@ -15,7 +15,7 @@ already holds text to the content check's rules, from the pack an untrusted chec `apps/skills` plays every example untrusted too. Measured on a five-question quiz, at four characters a token: `content.json` was about 3,470 tokens to write by hand, after reading 3,900 to 9,500 of a type's semantics; the source is 1,000 to 1,800 tokens, written from a type file of 200 -to 3,000. Twelve things the code does not say by itself: +to 3,000. Thirteen things the code does not say by itself: - **One generator, three ways to run it.** The third, `saved`, checks content as H5P stores it (*Saved content is checked as the build checks what it writes*, below). `ContentGenerator` was the sweep's @@ -277,6 +277,27 @@ to 3,000. Twelve things the code does not say by itself: an authoring tool's public address without "H5P" in it, and that origin already serves the player and the pack. It builds from `@missing-elements/h5p-libraries/install` at the version the author package pins, so the page plays with the pack it built from. +- **A bundle per content type, where the pack cannot be installed** (2026-11-09, #166). The page + leaves the last steps to the user, but ChatGPT's sandbox runs Node and fetches a file its + browsing tool has opened: given the libraries as files, a chat builds and fixes its source + itself. The skill chooses the content type first, so the unit is the type. `bundleLibraries` is + the build's own rules over the most a source of the type can name: the type and every library + its library fields offer or give as their default, theirs in turn; `planPack` over all of them + as roots, as `closure` in `build.mjs` plans one package; `installClosure` over what that plans. + Both only ever add for more roots, so one source's libraries are within its type's bundle. + `writeBundle` zips them as `parts/`, with `libraries.txt`, without which the parts are not handed + on. The bundle carries the pack's whole index, not its own entries: a build from it plans as it + would from the pack, so it writes the same package (the skill's 32 examples were built both ways + and matched, but for the ids a build draws) or stops naming the part it lacks, where a cut-down + index would build a source with math without its addon and say nothing. No addon is in a + bundle: H5P.MathDisplay, 2.8 MB, joins wherever text holds math, and would more than double most + bundles, so a source that calls for it fetches that one part. A bundle rather than the parts, + since ChatGPT's download tool costs two calls a file, opened and then fetched: a Question Set's + 33 parts would be 66. With pack 1.1.0 a question type's is 0.6 to 1.5 MB, a Question Set's + 2.6 MB, a container's 4.3 to 8.4 MB, and the 55 content types' 99 MB in all, which is why + embed-my.github.io serves them (embed-my/embed-my.github.io#14) rather than the libraries + package on jsDelivr, whose cap is 50 MB. The build tool beside them is the command line as one + file, esbuild's: 558 kB, and it builds from an unzipped bundle with nothing installed. Not built: editing an existing package, which a tool beside it does with `checkContent` as its check; layout for Drag and Drop and Crossword; and any judgement of whether content is right, which diff --git a/packages/author/README.md b/packages/author/README.md index a21a39c0..33209181 100644 --- a/packages/author/README.md +++ b/packages/author/README.md @@ -108,6 +108,25 @@ H5P.TrueFalse 1.8: True/False Question `--all` adds the interface texts; `--markdown` prints the section the skill's type files carry. An unknown name lists the pack's content types. +## A content type's bundle + +```bash +npx @missing-elements/h5p-author@latest bundle H5P.QuestionSet H5P.QuestionSet.zip +``` + +``` +Bundled H5P.QuestionSet 1.21 in /work/H5P.QuestionSet.zip: 33 parts from the library pack 1.1.0, 2.6 MB. Unzip it beside a source and build with --libraries parts. +``` + +The libraries a build of one content type reads, for a build where the pack cannot be installed, +such as a chat's sandbox that reaches no npm registry: everything the type can hold, with the +editor libraries an H5P site's upload check asks for. The zip holds `parts/`, the pack's whole +index with those parts, and `libraries.txt`, the licences they are used under. A build from it +writes the package the pack would. An addon is not in it, so a source whose text calls for one, +H5P.MathDisplay for math, stops naming the part to add to `parts/` from the pack's. With pack +1.1.0 a question type's bundle is about 1 MB, a Question Set's 2.6 MB, a container's up to +8.4 MB. `--all ` writes one per content type, `.zip`: 55, 99 MB in all. + ## Options - `--libraries ` or `--libraries=`: another copy of the pack's `parts/` folder, in @@ -128,8 +147,11 @@ const { output, manifest, content, libraries, install, translations, notes, medi plays with, `install` those carried for an H5P site's upload check. `missingForInstall(libraries)` is that check, over a package's `library.json` files: what a site with no library installed would refuse it for, empty for a build. `schemaOf(machineName, { markdown, all, -libraries })` returns the field list. `ContentGenerator` is the engine underneath, also run by the -repository's sweep of every content type. +libraries })` returns the field list. `writeBundle(type, output, { libraries })` and +`writeBundles(folder, { libraries })` write bundles, each `{ output, type, parts, size }`; +`bundleLibraries(pack, type)` returns a bundle's parts, and `contentTypes(pack)` the pack's content +types. `ContentGenerator` is the engine underneath, also run by the repository's sweep of every +content type. A tool beside the build takes the libraries as the build loads them, the generator, and the build's rules for text: diff --git a/packages/author/bin/h5p-author.mjs b/packages/author/bin/h5p-author.mjs index bb1dad59..5f5cdf2b 100755 --- a/packages/author/bin/h5p-author.mjs +++ b/packages/author/bin/h5p-author.mjs @@ -2,18 +2,23 @@ import { readFile } from 'node:fs/promises' import { dirname, resolve } from 'node:path' import { parseArgs } from 'node:util' -import { buildPackage, schemaOf, SourceError } from '../lib/author.mjs' +import { buildPackage, schemaOf, SourceError, writeBundle, writeBundles } from '../lib/author.mjs' import { packVersion } from '../lib/node.mjs' const USAGE = `Usage: h5p-author build [--libraries ] h5p-author schema [--markdown] [--all] [--libraries ] + h5p-author bundle [--libraries ] + h5p-author bundle --all [--libraries ] build Makes a playable .h5p from a source: { "type", "title", "language", "license", "authors", "params" }, or a quiz in the h5p-author skill's quiz format. Writes only when the whole source is right, and never over an existing file. schema Prints a content type's fields as a source writes them: --markdown as a skill's type file carries them, --all with the interface texts. +bundle Writes the libraries a build of one content type reads, as a zip of a parts/ + folder for build --libraries parts, where the pack cannot be installed; --all, + one per content type, named by it. -h, --help show this -v, --version print this package's version and the pack's @@ -22,6 +27,9 @@ The libraries come from the pack this release pins (@missing-elements/h5p-librar --libraries names another copy's parts/ folder. Check every package with npx @missing-elements/h5p-verify@latest before handing it over.` +/** A size in kilobytes below a megabyte, in megabytes above. @param {number} bytes */ +const sized = (bytes) => (bytes < 1_000_000 ? `${Math.max(1, Math.round(bytes / 1000))} kB` : `${(bytes / 1_000_000).toFixed(1)} MB`) + /** * The command line as the verifier's and embed's read theirs: `--libraries dir` or * `--libraries=dir`, and `-h`, `-v`. Which flags a command takes is checked after. @@ -85,6 +93,21 @@ try { } for (const { path, message } of result.notes) console.warn(`Note: ${path} ${message}`) console.log('Not verified: run npx @missing-elements/h5p-verify@latest on it and look at its screenshot before handing it over.') + } else if (command === 'bundle') { + for (const flag of Object.keys(flags)) if (flag !== 'all' && flag !== 'libraries') throw new Error(`bundle does not take --${flag}`) + const libraries = /** @type {string | undefined} */ (flags.libraries) + const from = libraries ? `the pack in ${libraries}` : `the library pack ${await packVersion()}` + if (flags.all) { + if (rest.length !== 1) throw new Error('bundle --all takes the folder to write them in: h5p-author bundle --all bundles') + const results = await writeBundles(rest[0], { libraries }) + for (const { type, parts, size } of results) console.log(`${type}: ${parts.length} parts, ${sized(size)}`) + const total = results.reduce((sum, { size }) => sum + size, 0) + console.log(`Bundled ${results.length} content types in ${resolve(rest[0])} from ${from}, ${sized(total)} in all.`) + } else { + if (rest.length !== 2) throw new Error('bundle takes a content type and an output: h5p-author bundle H5P.QuestionSet H5P.QuestionSet.zip') + const { output, type, parts, size } = await writeBundle(rest[0], rest[1], { libraries }) + console.log(`Bundled ${type} in ${output}: ${parts.length} parts from ${from}, ${sized(size)}. Unzip it beside a source and build with --libraries parts.`) + } } else { throw new Error(`unknown command "${command}"\n\n${USAGE}`) } diff --git a/packages/author/lib/author.mjs b/packages/author/lib/author.mjs index 229ff612..5d579d10 100644 --- a/packages/author/lib/author.mjs +++ b/packages/author/lib/author.mjs @@ -5,20 +5,23 @@ * package pins, checked against its index. A tool beside the build takes the libraries as the build * loads them (`openPack`, `loadLibraries`, and `packageLibraries` for a package's own), its rules * for text (`markupText`, `plainText`, `allowedTags`), and the check it runs last on what it writes, - * for saved content (`checkContent`). This is the entry for Node: `./core` is what of it needs no - * Node, and `./browser` builds in a page. + * for saved content (`checkContent`). A content type's bundle (`writeBundle`) carries the parts a + * build of it reads, for a build where the pack cannot be installed. This is the entry for Node: + * `./core` is what of it needs no Node, and `./browser` builds in a page. */ +import { contentTypes } from './bundle.mjs' import { openPack } from './node.mjs' import { schemaLines, schemaMarkdown } from './schema.mjs' import { LAYOUTS } from './types.mjs' export * from './core.mjs' -export { buildPackage, openPack } from './node.mjs' +export { buildPackage, openPack, writeBundle, writeBundles } from './node.mjs' /** * @typedef {import('./node.mjs').BuildResult} BuildResult * @typedef {import('./node.mjs').BuildOptions} BuildOptions + * @typedef {import('./node.mjs').BundleResult} BundleResult */ /** @@ -34,12 +37,8 @@ export async function schemaOf(machineName, options = {}) { try { const entry = pack.newest(machineName) if (!entry) { - const runnable = [] - for (const each of pack.libraries) { - const part = await pack.open(each) - if (part.library.runnable) runnable.push(`${each.machineName} (${part.library.title})`) - } - throw new Error(`The library pack has no ${machineName}. Its content types: ${[...new Set(runnable)].sort().join(', ')}`) + const types = (await contentTypes(pack)).map((part) => `${part.library.machineName} (${part.library.title})`) + throw new Error(`The library pack has no ${machineName}. Its content types: ${types.join(', ')}`) } const part = await pack.open(entry) if (!part.semantics) throw new Error(`${machineName} has no semantics: it is not a content type an author fills in`) diff --git a/packages/author/lib/bundle.mjs b/packages/author/lib/bundle.mjs new file mode 100644 index 00000000..c2825e16 --- /dev/null +++ b/packages/author/lib/bundle.mjs @@ -0,0 +1,116 @@ +import { librariesNamedIn, planPack } from '@missing-elements/h5p-offline-player/plan' +import { libraryOptions } from './generator.mjs' +import { installClosure } from './install.mjs' + +/** + * A content type's bundle: the parts of the pack a build of that type can read, whatever its source + * holds, for a build where the pack cannot be installed and one file per type can be fetched, as in + * ChatGPT's sandbox (#166). It is the build's own rules applied to the most a source of the type can + * name: the content type and every library its library fields offer or give as their default, those + * libraries' in turn; what they play with, as the player plans it (`planPack`); and what an H5P + * site's upload check wants with them (`installClosure`). Both rules only ever add for more roots, + * so whatever one source names is within the bundle. + * + * An addon is not in it. A build adds one where a source's text matches its pattern, H5P.MathDisplay + * (2.8 MB) where the text holds math, which would more than double most bundles. A bundle carries + * the pack's whole index, so a build from it plans as it would from the pack, and either writes the + * same package or stops at the part it lacks, by name. + */ + +/** + * @typedef {import('./pack.mjs').Pack} Pack + * @typedef {import('./pack.mjs').IndexEntry} IndexEntry + * @typedef {import('./pack.mjs').PartLibrary} PartLibrary + */ + +/** + * The pack's content types: each runnable library at the newest version the pack serves, the one a + * source's `type` builds. + * @param {Pack} pack + * @returns {Promise} by machine name + */ +export async function contentTypes(pack) { + /** @type {Map} */ + const found = new Map() + for (const { machineName } of pack.libraries) { + if (found.has(machineName)) continue + const entry = pack.newest(machineName) + if (!entry || entry.installOnly) continue + const part = await pack.open(entry) + if (part.library.runnable) found.set(machineName, part) + } + return [...found.values()].sort((a, b) => a.folder.localeCompare(b.folder)) +} + +/** + * Every library field of semantics, in groups and lists at any depth. + * @param {unknown} semantics + * @returns {any[]} + */ +function libraryFields(semantics) { + /** @type {any[]} */ + const found = [] + /** @param {any} field */ + const walk = (field) => { + if (!field || typeof field !== 'object') return + if (field.type === 'library') found.push(field) + if (Array.isArray(field.fields)) field.fields.forEach(walk) + walk(field.field) + } + if (Array.isArray(semantics)) semantics.forEach(walk) + return found +} + +/** + * The parts of a content type's bundle: the content type's first, then what it plays with, then + * what an H5P site's upload check wants, each once. + * @param {Pack} pack + * @param {string} type a content type's machine name, such as `H5P.QuestionSet`, as a source names it + * @returns {Promise} + */ +export async function bundleLibraries(pack, type) { + const main = pack.newest(String(type).trim()) + if (!main || main.installOnly) { + const known = (await contentTypes(pack)).map((part) => part.library.machineName) + throw new Error(`The library pack has no content type ${type}. Its content types: ${known.join(', ')}`) + } + if (!(await pack.open(main)).library.runnable) { + throw new Error(`${main.machineName} is not a content type on its own but a library content types use: bundle the content type you build`) + } + // What content of the type can hold: the libraries its library fields offer, and theirs in turn. + /** @type {Map} */ + const content = new Map() + const queue = [main] + while (queue.length) { + const entry = /** @type {IndexEntry} */ (queue.shift()) + if (content.has(entry.folder)) continue + content.set(entry.folder, entry) + for (const field of libraryFields((await pack.open(entry)).semantics)) { + const offered = [ + ...libraryOptions(field), + ...librariesNamedIn(field.default).map(({ machineName, majorVersion }) => ({ machineName, major: Number(majorVersion) })) + ] + // Every major the pack has of the name, not only the one offered: a source names a library + // without its version, and `loadLibraries` then opens each major the pack has of it. + for (const { machineName } of offered) { + for (const major of new Set(pack.libraries.filter((entry) => entry.machineName === machineName).map((entry) => entry.majorVersion))) { + const found = pack.newest(machineName, major) + if (found && !found.installOnly) queue.push(found) + } + } + } + } + const roots = [...content.values()].map(({ machineName, majorVersion, minorVersion }) => ({ machineName, majorVersion, minorVersion })) + // As the build plans a package (`closure` in build.mjs), with no parameters: an addon is added + // only for text that matches its pattern, and these are left out. + const planned = planPack({ own: [], provided: [], roots, params: undefined, pack: pack.libraries, refresh: false, fill: true })?.parts ?? [] + /** @type {Map} */ + const selected = new Map() + for (const { folder } of planned) { + const entry = pack.libraries.find((each) => each.folder === folder) + if (entry) selected.set(folder, await pack.open(entry)) + } + const install = await installClosure(pack, selected) + const parts = [main, ...[...selected.values(), ...install.values()].map((part) => part.index)] + return [...new Map(parts.map((entry) => [entry.folder, entry])).values()] +} diff --git a/packages/author/lib/core.mjs b/packages/author/lib/core.mjs index 6898badf..9045b652 100644 --- a/packages/author/lib/core.mjs +++ b/packages/author/lib/core.mjs @@ -1,11 +1,12 @@ /** * @missing-elements/h5p-author/core: what the package does that needs neither Node nor a page, for * a tool that runs beside a build in either: the generator, the libraries loaded as a build loads - * them from any source of them, the check of saved content, and the rules for text. The package's - * entry for Node (`.`) and the one for a page (`./browser`) carry all of it, each with its own - * `buildPackage` and `openPack`. + * them from any source of them, the parts a content type's build can read, the check of saved + * content, and the rules for text. The package's entry for Node (`.`) and the one for a page + * (`./browser`) carry all of it, each with its own `buildPackage` and `openPack`. */ +export { bundleLibraries, contentTypes } from './bundle.mjs' export { checkContent } from './check.mjs' export { ContentGenerator, librariesIn, libraryString } from './generator.mjs' export { missingForInstall } from './install.mjs' diff --git a/packages/author/lib/generator.mjs b/packages/author/lib/generator.mjs index b60932ac..26b77dba 100644 --- a/packages/author/lib/generator.mjs +++ b/packages/author/lib/generator.mjs @@ -753,7 +753,7 @@ export function libraryString(/** @type {any} */ library) { } /** @param {any} field @returns {{ machineName: string, major: number, minor: number }[]} */ -function libraryOptions(field) { +export function libraryOptions(field) { return (field?.options ?? []) .map((/** @type {any} */ option) => /^(\S+) (\d+)\.(\d+)$/.exec(typeof option === 'string' ? option : option?.name ?? '')) .filter(Boolean) diff --git a/packages/author/lib/node.mjs b/packages/author/lib/node.mjs index 2134e4ce..8e777049 100644 --- a/packages/author/lib/node.mjs +++ b/packages/author/lib/node.mjs @@ -1,11 +1,12 @@ import { constants, createReadStream, createWriteStream } from 'node:fs' -import { copyFile, lstat, mkdtemp, open, readFile, rm, stat } from 'node:fs/promises' +import { copyFile, lstat, mkdir, mkdtemp, open, readFile, rm, stat } from 'node:fs/promises' import { basename, dirname, join, resolve } from 'node:path' import { Readable, Writable } from 'node:stream' import { fileURLToPath, pathToFileURL } from 'node:url' -import { ZipWriter } from '@zip.js/zip.js' +import { Uint8ArrayReader, ZipWriter } from '@zip.js/zip.js' import { normalizeArchive } from '@missing-elements/h5p-normalize' import { composePackage, readSource, writePackage } from './build.mjs' +import { bundleLibraries, contentTypes } from './bundle.mjs' import { MediaStore } from './media.mjs' import { Pack } from './pack.mjs' @@ -27,8 +28,19 @@ import { Pack } from './pack.mjs' * @typedef {{ libraries?: string, base?: string, fetch?: typeof fetch }} BuildOptions * `libraries`, a library pack's parts folder in place of the pinned pack; `base`, the folder a * source's file paths are read from; `fetch`, for media named by web address + * @typedef {{ output: string, type: string, parts: string[], size: number }} BundleResult + * `type`, the content type with the version bundled, `H5P.QuestionSet 1.21`; `parts`, the + * folders of the parts it holds; `size`, the zip's in bytes */ +/** + * The parts folder a pack is read from: the one named, or the pinned pack's. + * @param {string} [folder] + */ +function partsFolder(folder) { + return folder ? resolve(folder) : dirname(fileURLToPath(import.meta.resolve('@missing-elements/h5p-libraries/parts/index.json'))) +} + /** * The pack a build reads: the one this package pins, or the `parts/` folder of another copy. Read * from its `index.json`, which names the libraries carried for an H5P site's upload check too, @@ -36,7 +48,7 @@ import { Pack } from './pack.mjs' * @param {string} [folder] a folder holding `index.json` and the parts it names */ export async function openPack(folder) { - const directory = folder ? resolve(folder) : dirname(fileURLToPath(import.meta.resolve('@missing-elements/h5p-libraries/parts/index.json'))) + const directory = partsFolder(folder) const origin = folder ? directory : '@missing-elements/h5p-libraries' let index try { @@ -51,6 +63,11 @@ export async function openPack(folder) { try { return new Uint8Array(await readFile(fileURLToPath(entry.url))) } catch (error) { + // A content type's bundle holds what the type can hold, and no addon: a source whose text + // calls for one, or another type's library, needs its part added. + if (folder && /** @type {{ code?: unknown }} */ (error)?.code === 'ENOENT') { + throw new Error(`The parts folder ${directory} has no ${entry.folder}.h5p, which this build needs: add it from the library pack's parts/`) + } throw new Error(`Cannot read the library part ${entry.folder} from ${origin}`, { cause: error }) } }, 'reinstall the pack') @@ -105,9 +122,10 @@ export function nodeMedia({ base, temporary }) { * The refusal of an output that exists, with the file system's code, so a caller tells it from * a source's problems. * @param {string} output + * @param {string} [what] what refuses */ -function exists(output) { - return Object.assign(new Error(`${output} exists, and a build never writes over a file: name a new one`), { code: 'EEXIST' }) +function exists(output, what = 'a build') { + return Object.assign(new Error(`${output} exists, and ${what} never writes over a file: name a new one`), { code: 'EEXIST' }) } /** @@ -169,3 +187,103 @@ export async function buildPackage(input, output, options = {}) { if (temporary) await rm(temporary, { recursive: true, force: true }) } } + +/** The date on every entry of a bundle, as on the pack's parts: a bundle of the same pack is the same bytes. */ +const BUNDLE_DATE = new Date(2000, 0, 1, 2) + +/** + * A content type's bundle (`bundleLibraries`) written to `output`, which must not exist yet: a zip + * of `parts/`, a parts folder `h5p-author build --libraries parts` reads, with the pack's whole index + * and the parts a build of the type reads, and of `libraries.txt`, the licences those are used under, + * as the pack lists them beside its parts folder. Two bundles of the same pack unzipped in one place + * make one parts folder: their index is the same. + * @param {string} type a content type's machine name, such as `H5P.QuestionSet` + * @param {string} output + * @param {{ libraries?: string }} [options] `libraries`, a pack's parts folder in place of the pinned pack + * @returns {Promise} + */ +export async function writeBundle(type, output, options = {}) { + output = resolve(output) + if (await lstat(output).then(() => true, () => false)) throw exists(output, 'h5p-author') + if (!(await stat(dirname(output)).catch(() => null))?.isDirectory()) throw new Error(`${dirname(output)} is not a folder to write the bundle in`) + const files = await bundleFiles(partsFolder(options.libraries)) + const pack = await openPack(options.libraries) + try { + return await writeZip(pack, files, await bundleLibraries(pack, type), output) + } finally { + await pack.close() + } +} + +/** + * Every content type's bundle (`contentTypes`), written as `writeBundle` writes one, to + * `.zip` in `folder`, which is made if it is not there. Nothing is written over: where + * one of them is there already, nothing is written. + * @param {string} folder + * @param {{ libraries?: string }} [options] `libraries`, a pack's parts folder in place of the pinned pack + * @returns {Promise} by machine name + */ +export async function writeBundles(folder, options = {}) { + folder = resolve(folder) + const files = await bundleFiles(partsFolder(options.libraries)) + const pack = await openPack(options.libraries) + try { + const types = (await contentTypes(pack)).map((part) => part.library.machineName) + const outputs = types.map((type) => join(folder, `${type}.zip`)) + for (const output of outputs) if (await lstat(output).then(() => true, () => false)) throw exists(output, 'h5p-author') + await mkdir(folder, { recursive: true }) + /** @type {BundleResult[]} */ + const results = [] + for (const [i, type] of types.entries()) results.push(await writeZip(pack, files, await bundleLibraries(pack, type), outputs[i])) + return results + } finally { + await pack.close() + } +} + +/** + * What every bundle of a pack carries besides its parts: the index as the pack has it, and the + * licence list beside its parts folder, without which the parts are not handed on. + * @param {string} directory the pack's parts folder + */ +async function bundleFiles(directory) { + const index = await readFile(join(directory, 'index.json')).catch((error) => { + throw new Error(`${directory} is not a library pack's parts folder: it has no readable index.json`, { cause: error }) + }) + const listed = join(dirname(directory), 'libraries.txt') + const licences = await readFile(listed).catch((error) => { + throw new Error(`${listed} cannot be read: a bundle carries the licences its libraries are used under, the list a pack keeps beside its parts/ folder`, { cause: error }) + }) + return { index: new Uint8Array(index), licences: new Uint8Array(licences) } +} + +/** + * A bundle's zip, created at `output` and removed again if it cannot be written whole. + * @param {Pack} pack with every part of `parts` opened, and so checked against the index + * @param {{ index: Uint8Array, licences: Uint8Array }} files + * @param {import('./pack.mjs').IndexEntry[]} parts the content type's first + * @param {string} output + * @returns {Promise} + */ +async function writeZip(pack, { index, licences }, parts, output) { + const file = await open(output, 'wx').catch((error) => { + throw error?.code === 'EEXIST' ? exists(output, 'h5p-author') : error + }) + try { + try { + const writer = new ZipWriter(Writable.toWeb(file.createWriteStream()), { useWebWorkers: false, lastModDate: BUNDLE_DATE, extendedTimestamp: false }) + await writer.add('parts/index.json', new Uint8ArrayReader(index)) + // A part is a zip already: stored as it is, the bytes checked against the index as it opened. + for (const entry of parts) await writer.add(`parts/${entry.folder}.h5p`, pack.archiveOf(entry), { level: 0 }) + await writer.add('libraries.txt', new Uint8ArrayReader(licences)) + await writer.close() + } finally { + await file.close() + } + } catch (error) { + await rm(output, { force: true }) + throw error + } + const [main] = parts + return { output, type: `${main.machineName} ${main.majorVersion}.${main.minorVersion}`, parts: parts.map((entry) => entry.folder), size: (await stat(output)).size } +} diff --git a/packages/author/tests/bundle.test.ts b/packages/author/tests/bundle.test.ts new file mode 100644 index 00000000..e5e4c94b --- /dev/null +++ b/packages/author/tests/bundle.test.ts @@ -0,0 +1,179 @@ +import { execFile } from 'node:child_process' +import { copyFile, mkdir, mkdtemp, readFile, readdir, rm, stat, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { dirname, join, resolve } from 'node:path' +import { promisify } from 'node:util' +import { BlobReader, Uint8ArrayWriter, ZipReader } from '@zip.js/zip.js' +import { afterAll, beforeAll, describe, expect, it } from 'vitest' +import { buildPackage, bundleLibraries, contentTypes, missingForInstall, openPack, writeBundle } from '../lib/author.mjs' + +/* + * A content type's bundle (#166): the parts a build of the type reads, for a build where the pack + * cannot be installed and a file can be fetched. Everything the type can hold is in it, with what + * an H5P site's upload check wants, and no addon; a build from it is the build from the pack, or + * stops at the part it lacks, by name. + */ + +const parts = resolve(import.meta.dirname, '../../libraries/parts') +const bin = resolve(import.meta.dirname, '../bin/h5p-author.mjs') +const exec = promisify(execFile) +const META = { title: 'Rivers', language: 'en', license: 'CC BY', authors: [{ name: 'Ann Author', role: 'Author' }] } +const QUIZ = { + ...META, + type: 'H5P.QuestionSet', + params: { + questions: [ + { library: 'H5P.MultiChoice', params: { question: 'Which river is the longest in Europe?', answers: [{ text: 'Volga', correct: true }, { text: 'Danube', correct: false }] } }, + { library: 'H5P.TrueFalse', params: { question: 'The Rhine flows into the North Sea.', correct: 'true' } } + ] + } +} + +type Field = { name: string, type: string, options?: string[], field?: Field, fields?: Field[] } + +let dir: string +let pack: Awaited> +beforeAll(async () => { + dir = await mkdtemp(join(tmpdir(), 'h5p-author-bundle-')) + pack = await openPack() +}) +afterAll(async () => { + await pack?.close() + await rm(dir, { recursive: true, force: true }) +}) + +/** A zip's files by name, in its order. */ +async function unzipped(path: string) { + const reader = new ZipReader(new BlobReader(new Blob([await readFile(path)])), { checkSignature: true }) + try { + const files = new Map() + for (const entry of await reader.getEntries()) if (!entry.directory) files.set(entry.filename, await entry.getData(new Uint8ArrayWriter())) + return files + } finally { + await reader.close() + } +} + +/** A zip unzipped into a folder of its own, as a chat's sandbox unzips a bundle. */ +async function unzip(path: string, name: string) { + const folder = join(dir, name) + for (const [file, bytes] of await unzipped(path)) { + await mkdir(dirname(join(folder, file)), { recursive: true }) + await writeFile(join(folder, file), bytes) + } + return folder +} + +/** Content with the ids a build draws anew each time masked. */ +const masked = (content: unknown) => JSON.parse(JSON.stringify(content, (key, value) => (key === 'subContentId' ? '*' : value))) + +/** The `library.json` of each part, as the upload check reads them. */ +const librariesOf = (entries: Array<{ folder: string }>) => Promise.all(entries.map(async (entry) => (await pack.open(pack.libraries.find((each) => each.folder === entry.folder)!)).library)) + +describe('bundleLibraries', () => { + it('holds every question a Question Set offers, with what an H5P site\'s upload check wants, and no addon', async () => { + const bundle = await bundleLibraries(pack, 'H5P.QuestionSet') + const set = pack.newest('H5P.QuestionSet')! + expect(bundle[0]).toEqual(set) + const names = new Set(bundle.map((entry) => entry.machineName)) + const questions = ((await pack.open(set)).semantics as Field[]).find((field) => field.name === 'questions')!.field! + const offered = questions.options!.map((option) => option.split(' ')[0]).filter((name) => pack.newest(name)) + expect(offered.length).toBeGreaterThan(5) + for (const name of offered) expect(names).toContain(name) + expect(names).toContain('H5PEditor.QuestionSetTextualEditor') + // Every dependency a part names, preloaded, dynamic or editor, is there at its exact version. + expect(missingForInstall(await librariesOf(bundle))).toEqual([]) + expect(bundle.filter((entry) => entry.addTo !== undefined)).toEqual([]) + const size = (entries: Array<{ size: number }>) => entries.reduce((sum, entry) => sum + entry.size, 0) + expect(size(bundle)).toBeLessThan(size(pack.libraries) / 4) + }) + + it('makes every content type\'s complete for the upload check, without an addon', async () => { + const types = await contentTypes(pack) + expect(types.length).toBeGreaterThan(40) + for (const type of types) { + expect(type.library.runnable, type.folder).toBeTruthy() + expect(type.index.installOnly, type.folder).toBeUndefined() + const bundle = await bundleLibraries(pack, type.library.machineName) + expect(missingForInstall(await librariesOf(bundle)), type.folder).toEqual([]) + expect(bundle.some((entry) => entry.addTo !== undefined), type.folder).toBe(false) + } + }) + + it('refuses a library that is not a content type, and one the pack does not have', async () => { + await expect(bundleLibraries(pack, 'H5P.Question')).rejects.toThrow(/^H5P\.Question is not a content type on its own but a library content types use/) + await expect(bundleLibraries(pack, 'H5P.Nothing')).rejects.toThrow(/^The library pack has no content type H5P\.Nothing\. Its content types: .*H5P\.QuestionSet/) + }) +}) + +describe('writeBundle', () => { + it('writes the pack\'s whole index, the parts and the licences, the same bytes each time', async () => { + const output = join(dir, 'H5P.QuestionSet.zip') + const result = await writeBundle('H5P.QuestionSet', output) + expect(result).toMatchObject({ output, type: expect.stringMatching(/^H5P\.QuestionSet \d+\.\d+$/), size: (await stat(output)).size }) + const files = await unzipped(output) + expect([...files.keys()]).toEqual(['parts/index.json', ...result.parts.map((folder) => `parts/${folder}.h5p`), 'libraries.txt']) + expect(Buffer.from(files.get('parts/index.json')!).equals(await readFile(join(parts, 'index.json')))).toBe(true) + expect(Buffer.from(files.get('libraries.txt')!).equals(await readFile(join(parts, '../libraries.txt')))).toBe(true) + for (const folder of result.parts) expect(Buffer.from(files.get(`parts/${folder}.h5p`)!).equals(await readFile(join(parts, `${folder}.h5p`))), folder).toBe(true) + + const again = await writeBundle('H5P.QuestionSet', join(dir, 'again.zip')) + expect((await readFile(again.output)).equals(await readFile(output))).toBe(true) + await expect(writeBundle('H5P.QuestionSet', output)).rejects.toMatchObject({ code: 'EEXIST', message: `${output} exists, and h5p-author never writes over a file: name a new one` }) + }) + + it('builds what the pack builds', async () => { + const folder = await unzip((await writeBundle('H5P.QuestionSet', join(dir, 'quiz.zip'))).output, 'quiz') + const whole = await buildPackage(QUIZ, join(dir, 'whole.h5p')) + const bundled = await buildPackage(QUIZ, join(dir, 'bundled.h5p'), { libraries: join(folder, 'parts') }) + expect(bundled.libraries).toEqual(whole.libraries) + expect(bundled.install).toEqual(whole.install) + expect(bundled.manifest).toEqual(whole.manifest) + expect(masked(bundled.content)).toEqual(masked(whole.content)) + }) + + it('stops at an addon the text calls for, naming its part, and builds with it once it is there', async () => { + const folder = join(await unzip((await writeBundle('H5P.Blanks', join(dir, 'H5P.Blanks.zip'))).output, 'blanks'), 'parts') + const addon = pack.newest('H5P.MathDisplay')! + const squares = { ...META, type: 'H5P.Blanks', params: { questions: ['The square of 3, $$3^2$$, is *9*.'] } } + const output = join(dir, 'squares.h5p') + await expect(buildPackage(squares, output, { libraries: folder })).rejects.toThrow( + `The parts folder ${folder} has no ${addon.folder}.h5p, which this build needs: add it from the library pack's parts/` + ) + expect(await readdir(dir)).not.toContain('squares.h5p') + await copyFile(join(parts, `${addon.folder}.h5p`), join(folder, `${addon.folder}.h5p`)) + const built = await buildPackage(squares, output, { libraries: folder }) + expect(built.libraries).toContain(addon.folder) + expect(built.notes.map(({ message }) => message)).toContainEqual(expect.stringContaining(`${addon.machineName} ${addon.majorVersion}.${addon.minorVersion}`)) + }) + + it('refuses a pack without its licence list beside its parts', async () => { + const bare = join(dir, 'bare', 'parts') + await mkdir(bare, { recursive: true }) + await copyFile(join(parts, 'index.json'), join(bare, 'index.json')) + await expect(writeBundle('H5P.TrueFalse', join(dir, 'bare.zip'), { libraries: bare })).rejects.toThrow( + `${join(dir, 'bare', 'libraries.txt')} cannot be read: a bundle carries the licences its libraries are used under` + ) + expect(await readdir(dir)).not.toContain('bare.zip') + }) +}) + +describe('the bundle command', () => { + it('writes one content type\'s bundle, or with --all every content type\'s, by its name', async () => { + const { stdout } = await exec(process.execPath, [bin, 'bundle', 'H5P.TrueFalse', join(dir, 'tf.zip')]) + expect(stdout).toMatch(/^Bundled H5P\.TrueFalse \d+\.\d+ in \S+tf\.zip: \d+ parts from the library pack \d+\.\d+\.\d+, [\d.]+ (kB|MB)\. Unzip it beside a source and build with --libraries parts\.\n$/) + + const all = join(dir, 'all') + const { stdout: listed } = await exec(process.execPath, [bin, 'bundle', '--all', all]) + const types = (await contentTypes(pack)).map((type) => type.library.machineName) + expect((await readdir(all)).sort()).toEqual(types.map((type) => `${type}.zip`).sort()) + expect(listed).toContain(`Bundled ${types.length} content types in ${all} from the library pack `) + // Not one written over: nothing is written. + await rm(join(all, 'H5P.Blanks.zip')) + await expect(exec(process.execPath, [bin, 'bundle', '--all', all])).rejects.toMatchObject({ code: 1, stderr: expect.stringMatching(/\.zip exists, and h5p-author never writes over a file/) }) + expect(await readdir(all)).not.toContain('H5P.Blanks.zip') + + await expect(exec(process.execPath, [bin, 'bundle', 'H5P.TrueFalse'])).rejects.toMatchObject({ code: 1, stderr: expect.stringContaining('bundle takes a content type and an output') }) + await expect(exec(process.execPath, [bin, 'bundle', '--markdown', 'H5P.TrueFalse', 'x.zip'])).rejects.toMatchObject({ code: 1, stderr: expect.stringContaining('bundle does not take --markdown') }) + }) +}) diff --git a/packages/author/tests/package-types.ts b/packages/author/tests/package-types.ts index a8e7f7e3..cd58e342 100644 --- a/packages/author/tests/package-types.ts +++ b/packages/author/tests/package-types.ts @@ -1,13 +1,14 @@ // The package as a caller sees it, by its own name: through the `types` condition, so this checks the // published declarations, not the source. `pnpm typecheck` writes them first. import { - allowedTags, buildPackage, checkContent, ContentGenerator, GENERATED_START, loadLibraries, markupText, openPack, packageLibraries, plainText, schemaOf, SourceError, - type BuildResult, type CheckedContent, type LibraryRecord, type LibrarySource, type PackageFiles, type Source, type TranslationReport + allowedTags, buildPackage, bundleLibraries, checkContent, ContentGenerator, contentTypes, GENERATED_START, loadLibraries, markupText, openPack, packageLibraries, plainText, schemaOf, SourceError, + writeBundle, writeBundles, + type BuildResult, type BundleResult, type CheckedContent, type LibraryRecord, type LibrarySource, type PackageFiles, type Source, type TranslationReport } from '@missing-elements/h5p-author' // @ts-expect-error The source reader is internal: a helper the tests import from its module. import { readSource } from '@missing-elements/h5p-author' import { buildPackage as buildInPage, openPack as openInPage, type BuildResult as PageBuildResult } from '@missing-elements/h5p-author/browser' -import { checkContent as checkBeside, ContentGenerator as Generator, loadLibraries as loadBeside, type IndexEntry } from '@missing-elements/h5p-author/core' +import { bundleLibraries as bundleBeside, checkContent as checkBeside, ContentGenerator as Generator, loadLibraries as loadBeside, type IndexEntry } from '@missing-elements/h5p-author/core' // @ts-expect-error The core builds nothing: a build is Node's or a page's. import { buildPackage as noBuild } from '@missing-elements/h5p-author/core' @@ -82,3 +83,18 @@ export async function inAPage(pack: { libraries: IndexEntry[] }, picture: File): await libraries.close() } } + +/** A content type's bundle: its parts, and the zip a build without the pack reads. */ +export async function bundles(): Promise { + const one: BundleResult = await writeBundle('H5P.Blanks', 'H5P.Blanks.zip', { libraries: 'parts' }) + const pack = await openPack('parts') + try { + const entries: IndexEntry[] = await bundleLibraries(pack, 'H5P.Blanks') + const beside: IndexEntry[] = await bundleBeside(pack, 'H5P.Blanks') + const names: string[] = (await contentTypes(pack)).map((part) => part.library.machineName) + void [entries, beside, names, one.type, one.parts, one.size] + } finally { + await pack.close() + } + return [one, ...(await writeBundles('bundles'))] +} diff --git a/skills/h5p-author/SKILL.md b/skills/h5p-author/SKILL.md index b07dbf62..87492aff 100644 --- a/skills/h5p-author/SKILL.md +++ b/skills/h5p-author/SKILL.md @@ -8,8 +8,9 @@ license: MIT You write a short source: the content type and only what its defaults cannot know. `@missing-elements/h5p-author` fills in the rest from the libraries, copies them, and checks -every field. Needs Node 22 and a shell; nothing to install. Where `npx` cannot run, a page in -the user's browser builds it instead (step 3), so never end with commands for the user to run. +every field. Needs Node 22 and a shell; nothing to install. Where `npx` cannot install, it builds +from two downloaded files, or a page in the user's browser builds it (step 3), so never end with +commands for the user to run. Keep [LICENSE](./LICENSE) with the skill. ## 1. Choose the content type with the user @@ -82,12 +83,33 @@ Build in a working folder of your own, not in the folder the user named: npx @missing-elements/h5p-author@latest build source.json course.h5p ``` -**If it cannot run here, for any reason, the build page builds it instead.** No shell, no Node, -or no network (`EAI_AGAIN`, `ENOTFOUND`, a registry's 403, 404 or 503, as in ChatGPT): do not -retry, try another way to install it, or end with commands for the user to run once npm works. -[The build page](https://embed-my.github.io/build) builds the same package in the user's browser -from the source alone, with no install. Give them a link that carries the source, written with -Node or Python, which need no network: +**If it cannot install here, build from files instead.** No network for npm (`EAI_AGAIN`, +`ENOTFOUND`, a registry's 403, 404 or 503, as in ChatGPT): do not retry, try another way to +install it, or end with commands for the user to run once npm works. Where Node runs and you can +fetch a file (in ChatGPT, open each address with the browsing tool, then download it), fetch the +build tool and the content type's bundle, the libraries a build of that type reads, named by the +type's machine name: + +``` +https://embed-my.github.io/bundles/h5p-author.mjs +https://embed-my.github.io/bundles/H5P.QuestionSet.zip +``` + +```bash +unzip -q H5P.QuestionSet.zip # or: python3 -m zipfile -e H5P.QuestionSet.zip . +node h5p-author.mjs build source.json course.h5p --libraries parts +``` + +A build that stops at a part `parts/` lacks names it (an addon: H5P.MathDisplay, where the text +holds math): fetch that part into `parts/`, such as +`https://embed-my.github.io/bundles/parts/H5P.MathDisplay-1.0.h5p`, and build again. The verifier +cannot run there either: hand the package over **unverified**, with the build page's link below +for the user to play it before using it. + +**Where nothing can be fetched or run, the build page builds it.** No shell, no Node, or no +download: [the build page](https://embed-my.github.io/build) builds the same package in the user's +browser from the source alone, with no install. Give them a link that carries the source, written +with Node or Python, which need no network: ```bash node -e "console.log('https://embed-my.github.io/build#source=' + require('zlib').deflateRawSync(require('fs').readFileSync('source.json')).toString('base64url'))" From 519643a4528d1166bc33a361c8d3304cafa2504f Mon Sep 17 00:00:00 2001 From: Aleksandr Shevchuk Date: Mon, 9 Nov 2026 21:44:42 +0200 Subject: [PATCH 2/3] fix(author): bundle --all writes every bundle or none; the skill names the source's own type A later content type that failed left the bundles before it in the folder, and running the command again then refused them: those this call wrote are now taken away again. The skill's step 3 wrote H5P.QuestionSet.zip as if every source were a quiz; it now writes , the source's own, since another type's bundle lacks what this one needs. The README's and the design note's "stops naming the part" now say the build stops and names it. --- design/author.md | 2 +- packages/author/README.md | 9 +++++---- packages/author/lib/node.mjs | 11 +++++++++-- packages/author/tests/bundle.test.ts | 26 ++++++++++++++++++++++++-- skills/h5p-author/SKILL.md | 9 +++++---- 5 files changed, 44 insertions(+), 13 deletions(-) diff --git a/design/author.md b/design/author.md index e7feb1f0..a58968d9 100644 --- a/design/author.md +++ b/design/author.md @@ -288,7 +288,7 @@ to 3,000. Thirteen things the code does not say by itself: `writeBundle` zips them as `parts/`, with `libraries.txt`, without which the parts are not handed on. The bundle carries the pack's whole index, not its own entries: a build from it plans as it would from the pack, so it writes the same package (the skill's 32 examples were built both ways - and matched, but for the ids a build draws) or stops naming the part it lacks, where a cut-down + and matched, but for the ids a build draws) or stops and names the part it lacks, where a cut-down index would build a source with math without its addon and say nothing. No addon is in a bundle: H5P.MathDisplay, 2.8 MB, joins wherever text holds math, and would more than double most bundles, so a source that calls for it fetches that one part. A bundle rather than the parts, diff --git a/packages/author/README.md b/packages/author/README.md index 33209181..681e10d8 100644 --- a/packages/author/README.md +++ b/packages/author/README.md @@ -122,10 +122,11 @@ The libraries a build of one content type reads, for a build where the pack cann such as a chat's sandbox that reaches no npm registry: everything the type can hold, with the editor libraries an H5P site's upload check asks for. The zip holds `parts/`, the pack's whole index with those parts, and `libraries.txt`, the licences they are used under. A build from it -writes the package the pack would. An addon is not in it, so a source whose text calls for one, -H5P.MathDisplay for math, stops naming the part to add to `parts/` from the pack's. With pack -1.1.0 a question type's bundle is about 1 MB, a Question Set's 2.6 MB, a container's up to -8.4 MB. `--all ` writes one per content type, `.zip`: 55, 99 MB in all. +writes the package the pack would. An addon is not in it: where a source's text calls for one, +H5P.MathDisplay for math, the build stops and names the missing part, to copy into `parts/` from +the pack's own `parts/` folder. With pack 1.1.0 a question type's bundle is about 1 MB, a Question +Set's 2.6 MB, a container's up to 8.4 MB. `--all ` writes one per content type, +`.zip`: 55, 99 MB in all. If one fails, none is left behind. ## Options diff --git a/packages/author/lib/node.mjs b/packages/author/lib/node.mjs index 8e777049..c778caac 100644 --- a/packages/author/lib/node.mjs +++ b/packages/author/lib/node.mjs @@ -218,7 +218,8 @@ export async function writeBundle(type, output, options = {}) { /** * Every content type's bundle (`contentTypes`), written as `writeBundle` writes one, to * `.zip` in `folder`, which is made if it is not there. Nothing is written over: where - * one of them is there already, nothing is written. + * one of them is there already, nothing is written. All of them or none: one that fails takes those + * written before it away again, so the call can be run again as it was. * @param {string} folder * @param {{ libraries?: string }} [options] `libraries`, a pack's parts folder in place of the pinned pack * @returns {Promise} by machine name @@ -234,7 +235,13 @@ export async function writeBundles(folder, options = {}) { await mkdir(folder, { recursive: true }) /** @type {BundleResult[]} */ const results = [] - for (const [i, type] of types.entries()) results.push(await writeZip(pack, files, await bundleLibraries(pack, type), outputs[i])) + try { + for (const [i, type] of types.entries()) results.push(await writeZip(pack, files, await bundleLibraries(pack, type), outputs[i])) + } catch (error) { + // Only what this call wrote: each was created anew, after the check above found none there. + await Promise.all(results.map(({ output }) => rm(output, { force: true }))) + throw error + } return results } finally { await pack.close() diff --git a/packages/author/tests/bundle.test.ts b/packages/author/tests/bundle.test.ts index e5e4c94b..0783d498 100644 --- a/packages/author/tests/bundle.test.ts +++ b/packages/author/tests/bundle.test.ts @@ -1,11 +1,11 @@ import { execFile } from 'node:child_process' -import { copyFile, mkdir, mkdtemp, readFile, readdir, rm, stat, writeFile } from 'node:fs/promises' +import { copyFile, cp, mkdir, mkdtemp, readFile, readdir, rm, stat, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { dirname, join, resolve } from 'node:path' import { promisify } from 'node:util' import { BlobReader, Uint8ArrayWriter, ZipReader } from '@zip.js/zip.js' import { afterAll, beforeAll, describe, expect, it } from 'vitest' -import { buildPackage, bundleLibraries, contentTypes, missingForInstall, openPack, writeBundle } from '../lib/author.mjs' +import { buildPackage, bundleLibraries, contentTypes, missingForInstall, openPack, writeBundle, writeBundles } from '../lib/author.mjs' /* * A content type's bundle (#166): the parts a build of the type reads, for a build where the pack @@ -147,6 +147,28 @@ describe('writeBundle', () => { expect(built.notes.map(({ message }) => message)).toContainEqual(expect.stringContaining(`${addon.machineName} ${addon.majorVersion}.${addon.minorVersion}`)) }) + it('writes every content type\'s or none: one that fails takes those before it away again', async () => { + // A copy of the pack without a part that only a later content type's bundle reads: an older + // minor kept for the upload check, which listing the content types never opens. + const types = (await contentTypes(pack)).map((type) => type.library.machineName) + const bundles: string[][] = [] + for (const type of types) bundles.push((await bundleLibraries(pack, type)).map((entry) => entry.folder)) + const firstNeeding = (folder: string) => bundles.findIndex((folders) => folders.includes(folder)) + const missing = pack.libraries.filter((entry) => entry.installOnly).map((entry) => entry.folder).find((folder) => firstNeeding(folder) > 0) + expect(missing).toBeDefined() + const copy = join(dir, 'gappy') + await cp(parts, join(copy, 'parts'), { recursive: true }) + await copyFile(join(parts, '../libraries.txt'), join(copy, 'libraries.txt')) + await rm(join(copy, 'parts', `${missing}.h5p`)) + + const out = join(dir, 'gappy-bundles') + await expect(writeBundles(out, { libraries: join(copy, 'parts') })).rejects.toThrow(`has no ${missing}.h5p`) + expect(await readdir(out)).toEqual([]) + // Run again with the part back, it writes them all. + await copyFile(join(parts, `${missing}.h5p`), join(copy, 'parts', `${missing}.h5p`)) + expect(await writeBundles(out, { libraries: join(copy, 'parts') })).toHaveLength(types.length) + }) + it('refuses a pack without its licence list beside its parts', async () => { const bare = join(dir, 'bare', 'parts') await mkdir(bare, { recursive: true }) diff --git a/skills/h5p-author/SKILL.md b/skills/h5p-author/SKILL.md index 87492aff..9d2164e2 100644 --- a/skills/h5p-author/SKILL.md +++ b/skills/h5p-author/SKILL.md @@ -87,16 +87,17 @@ npx @missing-elements/h5p-author@latest build source.json course.h5p `ENOTFOUND`, a registry's 403, 404 or 503, as in ChatGPT): do not retry, try another way to install it, or end with commands for the user to run once npm works. Where Node runs and you can fetch a file (in ChatGPT, open each address with the browsing tool, then download it), fetch the -build tool and the content type's bundle, the libraries a build of that type reads, named by the -type's machine name: +build tool and the bundle of the source's own `type`, the libraries a build of that type reads. +In both addresses and commands below, `` is that machine name, such as `H5P.Blanks`: another +type's bundle lacks what this one needs. ``` https://embed-my.github.io/bundles/h5p-author.mjs -https://embed-my.github.io/bundles/H5P.QuestionSet.zip +https://embed-my.github.io/bundles/.zip ``` ```bash -unzip -q H5P.QuestionSet.zip # or: python3 -m zipfile -e H5P.QuestionSet.zip . +unzip -q .zip # or: python3 -m zipfile -e .zip . node h5p-author.mjs build source.json course.h5p --libraries parts ``` From c480fc26020723a6e9ab72cd3181128c76876ccd Mon Sep 17 00:00:00 2001 From: Aleksandr Shevchuk Date: Mon, 9 Nov 2026 21:56:51 +0200 Subject: [PATCH 3/3] fix(author): a failed bundle --all takes away only the files this call made The rollback removed each earlier bundle by its name, so a file another process had put at that name since was taken away with them. Each bundle's device and inode are now recorded as it is created, and the rollback, like a single bundle's own cleanup, removes a name only while it still holds that file. --- packages/author/lib/node.mjs | 53 +++++++++++++++++++++------- packages/author/tests/bundle.test.ts | 28 ++++++++++++++- 2 files changed, 68 insertions(+), 13 deletions(-) diff --git a/packages/author/lib/node.mjs b/packages/author/lib/node.mjs index c778caac..0f14bd08 100644 --- a/packages/author/lib/node.mjs +++ b/packages/author/lib/node.mjs @@ -31,6 +31,8 @@ import { Pack } from './pack.mjs' * @typedef {{ output: string, type: string, parts: string[], size: number }} BundleResult * `type`, the content type with the version bundled, `H5P.QuestionSet 1.21`; `parts`, the * folders of the parts it holds; `size`, the zip's in bytes + * @typedef {{ dev: bigint, ino: bigint }} Made a file this module created, by the device and + * inode it was given, which a file that takes its name later does not have */ /** @@ -209,7 +211,7 @@ export async function writeBundle(type, output, options = {}) { const files = await bundleFiles(partsFolder(options.libraries)) const pack = await openPack(options.libraries) try { - return await writeZip(pack, files, await bundleLibraries(pack, type), output) + return (await writeZip(pack, files, await bundleLibraries(pack, type), output)).result } finally { await pack.close() } @@ -219,7 +221,8 @@ export async function writeBundle(type, output, options = {}) { * Every content type's bundle (`contentTypes`), written as `writeBundle` writes one, to * `.zip` in `folder`, which is made if it is not there. Nothing is written over: where * one of them is there already, nothing is written. All of them or none: one that fails takes those - * written before it away again, so the call can be run again as it was. + * written before it away again, so the call can be run again as it was. A file another process has + * put at one of those names since is theirs, and stays. * @param {string} folder * @param {{ libraries?: string }} [options] `libraries`, a pack's parts folder in place of the pinned pack * @returns {Promise} by machine name @@ -233,16 +236,17 @@ export async function writeBundles(folder, options = {}) { const outputs = types.map((type) => join(folder, `${type}.zip`)) for (const output of outputs) if (await lstat(output).then(() => true, () => false)) throw exists(output, 'h5p-author') await mkdir(folder, { recursive: true }) - /** @type {BundleResult[]} */ - const results = [] + /** @type {{ result: BundleResult, made: Made }[]} */ + const written = [] try { - for (const [i, type] of types.entries()) results.push(await writeZip(pack, files, await bundleLibraries(pack, type), outputs[i])) + for (const [i, type] of types.entries()) written.push(await writeZip(pack, files, await bundleLibraries(pack, type), outputs[i])) } catch (error) { - // Only what this call wrote: each was created anew, after the check above found none there. - await Promise.all(results.map(({ output }) => rm(output, { force: true }))) + // Only what this call made: a name is taken away if it still holds the file this call + // created, not if another process has replaced that since. + await Promise.all(written.map(({ result, made }) => removeIfMade(result.output, made))) throw error } - return results + return written.map(({ result }) => result) } finally { await pack.close() } @@ -265,19 +269,39 @@ async function bundleFiles(directory) { } /** - * A bundle's zip, created at `output` and removed again if it cannot be written whole. + * Takes away a file this module created, unless the name holds another file by now. A file another + * process has put there since is theirs, and removing by the name alone would take it. A guard, + * not a lock: the check and the removal are two calls, and Node removes a file by its name, so a + * file that takes the name in the gap between them is still lost. + * @param {string} path + * @param {Made} made + * @returns {Promise} + * @internal + */ +export async function removeIfMade(path, made) { + const now = await lstat(path, { bigint: true }).catch(() => null) + if (now && now.dev === made.dev && now.ino === made.ino) await rm(path, { force: true }) +} + +/** + * A bundle's zip, created at `output`, and removed again if it cannot be written whole. * @param {Pack} pack with every part of `parts` opened, and so checked against the index * @param {{ index: Uint8Array, licences: Uint8Array }} files * @param {import('./pack.mjs').IndexEntry[]} parts the content type's first * @param {string} output - * @returns {Promise} + * @returns {Promise<{ result: BundleResult, made: Made }>} `made`, the file as created, for a caller + * that takes it away again if a later one fails */ async function writeZip(pack, { index, licences }, parts, output) { const file = await open(output, 'wx').catch((error) => { throw error?.code === 'EEXIST' ? exists(output, 'h5p-author') : error }) + /** @type {Made | undefined} */ + let made try { try { + const { dev, ino } = await file.stat({ bigint: true }) + made = { dev, ino } const writer = new ZipWriter(Writable.toWeb(file.createWriteStream()), { useWebWorkers: false, lastModDate: BUNDLE_DATE, extendedTimestamp: false }) await writer.add('parts/index.json', new Uint8ArrayReader(index)) // A part is a zip already: stored as it is, the bytes checked against the index as it opened. @@ -288,9 +312,14 @@ async function writeZip(pack, { index, licences }, parts, output) { await file.close() } } catch (error) { - await rm(output, { force: true }) + // The file as created, if it is still there; one that could not be looked at was made a moment ago. + if (made) await removeIfMade(output, made) + else await rm(output, { force: true }) throw error } const [main] = parts - return { output, type: `${main.machineName} ${main.majorVersion}.${main.minorVersion}`, parts: parts.map((entry) => entry.folder), size: (await stat(output)).size } + return { + result: { output, type: `${main.machineName} ${main.majorVersion}.${main.minorVersion}`, parts: parts.map((entry) => entry.folder), size: (await stat(output)).size }, + made: /** @type {Made} */ (made) + } } diff --git a/packages/author/tests/bundle.test.ts b/packages/author/tests/bundle.test.ts index 0783d498..da1093f8 100644 --- a/packages/author/tests/bundle.test.ts +++ b/packages/author/tests/bundle.test.ts @@ -1,11 +1,12 @@ import { execFile } from 'node:child_process' -import { copyFile, cp, mkdir, mkdtemp, readFile, readdir, rm, stat, writeFile } from 'node:fs/promises' +import { copyFile, cp, lstat, mkdir, mkdtemp, readFile, readdir, rename, rm, stat, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { dirname, join, resolve } from 'node:path' import { promisify } from 'node:util' import { BlobReader, Uint8ArrayWriter, ZipReader } from '@zip.js/zip.js' import { afterAll, beforeAll, describe, expect, it } from 'vitest' import { buildPackage, bundleLibraries, contentTypes, missingForInstall, openPack, writeBundle, writeBundles } from '../lib/author.mjs' +import { removeIfMade } from '../lib/node.mjs' /* * A content type's bundle (#166): the parts a build of the type reads, for a build where the pack @@ -169,6 +170,31 @@ describe('writeBundle', () => { expect(await writeBundles(out, { libraries: join(copy, 'parts') })).toHaveLength(types.length) }) + it('takes a file away only while it is the one this call made, not once another process has replaced it', async () => { + const identity = async (path: string) => { + const { dev, ino } = await lstat(path, { bigint: true }) + return { dev, ino } + } + // The file as made: taken away. + const mine = join(dir, 'made.zip') + await writeFile(mine, 'made') + await removeIfMade(mine, await identity(mine)) + await expect(stat(mine)).rejects.toMatchObject({ code: 'ENOENT' }) + + // Replaced by another process since: theirs, and it stays. The first file is moved aside, not + // deleted, so that its inode is still taken and the new file cannot be given the same one. + const theirs = join(dir, 'replaced.zip') + await writeFile(theirs, 'made') + const made = await identity(theirs) + await rename(theirs, join(dir, 'replaced-first.zip')) + await writeFile(theirs, 'theirs') + await removeIfMade(theirs, made) + expect(await readFile(theirs, 'utf8')).toBe('theirs') + + // Already gone: nothing to take, and no error. + await removeIfMade(mine, made) + }) + it('refuses a pack without its licence list beside its parts', async () => { const bare = join(dir, 'bare', 'parts') await mkdir(bare, { recursive: true })