Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,7 @@ Read it before changing which libraries a package plays with: the library check,
- The pack is split, a package downloads the parts it needs, and the pack's newer versions play in place of a package's older ones.
- `libraries` is a list, tried in order.
- Content written for a newer minor than any source has plays at the newest older one.
- The pack carries what an H5P site's upload check wants, and the player never serves it.

### The frame and the runtime: [`design/frame.md`](design/frame.md)

Expand Down
Binary file modified apps/demo/demo/content/glossary.h5p
Binary file not shown.
Binary file modified apps/demo/demo/content/quiz.h5p
Binary file not shown.
Binary file modified apps/demo/demo/content/video.h5p
Binary file not shown.
62 changes: 52 additions & 10 deletions apps/demo/scripts/build-demo-content.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@ import { tmpdir } from 'node:os'
import { basename, join, relative, resolve } from 'node:path'
import { BlobReader, BlobWriter, TextReader, Uint8ArrayReader, Uint8ArrayWriter, ZipReader, ZipWriter, configure } from '@zip.js/zip.js'
import { formatBytes, normalizeArchive } from '@missing-elements/h5p-normalize'
// The rule h5p-author checks its packages by and `apps/skills` checks every example with, reached
// by path as build-libraries.mjs reaches the pack's scripts.
import { missingForInstall } from '../../../packages/author/lib/install.mjs'
import { download, hubBundle as cachedBundle } from './lib/hub.mjs'

/**
Expand All @@ -16,6 +19,13 @@ import { download, hubBundle as cachedBundle } from './lib/hub.mjs'
* naming the main library alone, `content/`, and no library folders — the package the player
* page plays `untrusted` against the site's pack.
*
* Every other package also carries what an H5P site's upload check wants of its libraries (#146):
* H5P core refuses a package unless each library it carries has its preloaded, dynamic and editor
* dependencies with it at the exact major.minor, so a Moodle with no library installed refused
* the demo's quiz, video and glossary for their editor libraries. `h5p.json` lists only what
* plays, as the platforms' exports write it, and the player never reads the rest; they follow the
* libraries that play, and the normalizer keeps library folders in that order.
*
* The outputs are committed. This is a maintainer's tool with two network dependencies — the
* hub, and whatever media a manifest points at — and a deploy should not have either. Hub
* bundles are cached under the OS temp directory; `--refresh` fetches them again.
Expand Down Expand Up @@ -72,8 +82,8 @@ async function buildPackage(name) {
const entries = (await bundle.getEntries({ filenameValidation: 'tolerant' })).filter((entry) => !entry.directory)

// Every library the content actually needs: the main library's dependency tree, plus the
// tree of every sub-content library the params name. Nothing else from the bundle — in
// particular none of its editor libraries — goes into the package.
// tree of every sub-content library the params name. What else goes into the package is what
// an H5P site's upload check wants of these, below; nothing else from the bundle does.
const libraryJsons = new Map()
for (const entry of entries) {
const match = /^([^/]+)\/library\.json$/.exec(entry.filename)
Expand Down Expand Up @@ -104,9 +114,33 @@ async function buildPackage(name) {
visit(manifest.mainLibrary)
for (const used of JSON.stringify(content).matchAll(/"library":\s*"([^" ]+) (\d+)\.(\d+)"/g)) visit(used[1])

const folderOf = (library) => `${library.machineName}-${library.majorVersion}.${library.minorVersion}`
const folderOf = (library) => `${library.machineName}-${Number(library.majorVersion)}.${Number(library.minorVersion)}`
const folders = new Set([...needed.values()].map(folderOf))

// What an H5P site's upload check wants of them (#146): every dependency a carried library
// names, preloaded, dynamic or editor, at its exact major.minor, and what those name in turn.
// The bundle carries its content type's whole closure, so each is in it, an older minor of a
// library included (the QuestionSet bundle's H5P.Question 1.4 beside the 1.5 that plays).
/** @type {Set<string>} */
const install = new Set()
if (!manifest.stripped) {
const queue = [...needed.values()]
while (queue.length > 0) {
const library = queue.shift()
for (const dependency of [...(library.preloadedDependencies ?? []), ...(library.dynamicDependencies ?? []), ...(library.editorDependencies ?? [])]) {
const folder = folderOf(dependency)
if (folders.has(folder) || install.has(folder)) continue
const found = libraryJsons.get(folder)
if (!found) throw new Error(`${name}: ${folder} is not in the hub bundle for ${manifest.mainLibrary}, which ${folderOf(library)} needs to install on an H5P site`)
install.add(folder)
queue.push(found)
}
}
// The same rule as h5p-author's check, over what is about to be written.
const missing = missingForInstall([...needed.values(), ...[...install].map((folder) => libraryJsons.get(folder))])
if (missing.length > 0) throw new Error(`${name}: an H5P site would refuse it for ${missing.map(({ library, needs }) => `${library} without ${needs}`).join(', ')}`)
}

const h5pJson = {
title: manifest.title,
language: 'en',
Expand Down Expand Up @@ -139,13 +173,18 @@ async function buildPackage(name) {
await writer.add(`content/${path}`, new Uint8ArrayReader(new Uint8Array(await fetched(url))))
}

// The libraries that play, then the ones an H5P site's upload check wants with them.
let libraryBytes = 0
for (const entry of entries) {
const folder = entry.filename.slice(0, entry.filename.indexOf('/'))
if (manifest.stripped || !folders.has(folder)) continue
const bytes = await entry.getData(new Uint8ArrayWriter())
libraryBytes += bytes.length
await writer.add(entry.filename, new Uint8ArrayReader(bytes), { lastModDate: entry.lastModDate })
let installBytes = 0
for (const wanted of [folders, install]) {
for (const entry of entries) {
const folder = entry.filename.slice(0, entry.filename.indexOf('/'))
if (manifest.stripped || !wanted.has(folder)) continue
const bytes = await entry.getData(new Uint8ArrayWriter())
if (wanted === folders) libraryBytes += bytes.length
else installBytes += bytes.length
await writer.add(entry.filename, new Uint8ArrayReader(bytes), { lastModDate: entry.lastModDate })
}
}
await bundle.close()

Expand All @@ -158,7 +197,10 @@ async function buildPackage(name) {

console.log(
`[demo-content] ${name}.h5p ${formatBytes(report.bytesOut ?? 0)}: ${manifest.mainLibrary}, ` +
(manifest.stripped ? 'content only, ' : `${folders.size} libraries (${formatBytes(libraryBytes)} uncompressed), `) +
(manifest.stripped
? 'content only, '
: `${folders.size} libraries (${formatBytes(libraryBytes)} uncompressed)` +
(install.size ? ` and ${install.size} more for an H5P site's upload check (${formatBytes(installBytes)}), ` : ', ')) +
`${report.entries.filter((entry) => entry.target === 'store').length} media entries stored` +
(report.entries.some((entry) => entry.action === 'faststart') ? ', mp4 index moved to front' : '')
)
Expand Down
86 changes: 74 additions & 12 deletions apps/demo/scripts/build-libraries.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,13 @@ import { HUB, download as cachedDownload, hubBundle as cachedBundle } from './li
* Every content type below is fetched from the hub (cached under the OS temp directory, shared
* with `build-demo-content.mjs`; `--refresh` fetches again), and so is every addon in `ADDONS`
* from where H5P Group publishes it, and every library folder in the
* bundles is kept except the editor's, at the newest minor of each major: H5P keeps a minor
* bundles is kept at the newest minor of each major: H5P keeps a minor
* compatible with the ones before it, and the player resolves a request for 1.26 to 1.27. A
* package that needs a newer minor than the pack has is refused, naming the library.
* package that needs a newer minor than the pack has is refused, naming the library. With them,
* since pack 1.1.0, what an H5P site's upload check wants of those libraries (#146): the editor
* libraries, and the older minor a library names exactly, at the exact versions. `splitPack`
* marks those `installOnly` in the index; the player never serves one, and h5p-author carries
* them into a package.
*
* The outputs are committed, like the demo content: a deploy should not depend on the hub.
* Refresh them before a release. A library whose `library.json` declares no licence has to be
Expand Down Expand Up @@ -85,6 +89,29 @@ const UNDECLARED = {
'H5P.Nil': 'MIT (README of github.com/h5p/h5p-nil)',
'H5P.Table': 'MIT (README of github.com/h5p/h5p-table)',
'H5P.Text': 'MIT (README of github.com/h5p/h5p-text)',
'H5PEditor.ARMarkerGenerator': 'MIT (LICENSE of github.com/otacke/h5p-editor-ar-marker-generator)',
'H5PEditor.BranchingQuestion': 'MIT (README of github.com/h5p/h5p-editor-branching-question)',
'H5PEditor.ColorSelector': 'MIT (README of github.com/h5p/h5p-editor-color-selector)',
'H5PEditor.CoursePresentation': 'MIT (README of github.com/h5p/h5p-editor-course-presentation)',
'H5PEditor.DateTime': 'MIT (LICENSE of github.com/otacke/h5p-editor-datetime)',
'H5PEditor.DragQuestion': 'MIT (README of github.com/h5p/h5p-editor-drag-question)',
'H5PEditor.Duration': 'MIT (README of github.com/h5p/h5p-editor-duration)',
'H5PEditor.GameMap': 'MIT (LICENSE of github.com/otacke/h5p-editor-game-map)',
'H5PEditor.ImageCoordinateSelector': 'MIT (README of github.com/h5p/h5p-editor-image-coordinate-selector)',
'H5PEditor.ImageHotspotQuestion': 'MIT (README of github.com/h5p/h5p-editor-image-hotspot-question)',
'H5PEditor.ImageMultipleHotspotQuestion': 'MIT (README of github.com/lukemuller/h5p-editor-image-multiple-hotspot-question)',
'H5PEditor.ImportantDescription': 'MIT (LICENSE of github.com/otacke/h5p-editor-important-description)',
'H5PEditor.InfoWall': 'MIT (LICENSE of github.com/otacke/h5p-editor-info-wall)',
'H5PEditor.InfoWallPassiveList': 'MIT (LICENSE of github.com/otacke/h5p-editor-info-wall-passive-list)',
'H5PEditor.InputRow': 'MIT (LICENSE of github.com/otacke/h5p-editor-input-row)',
'H5PEditor.KewArCode': 'MIT (LICENSE of github.com/otacke/h5p-editor-kewar-code)',
'H5PEditor.RadioGroup': 'MIT (README of github.com/h5p/h5p-editor-radio-group)',
'H5PEditor.RadioSelector': 'MIT (README of github.com/h5p/h5p-editor-radio-selector)',
'H5PEditor.Shape': 'MIT (README of github.com/h5p/h5p-editor-shape)',
'H5PEditor.ThreeDModelLoader': 'MIT (LICENSE of github.com/otacke/h5p-editor-3d-model-loader)',
'H5PEditor.Timecode': 'MIT (README of github.com/h5p/h5p-editor-timecode)',
'H5PEditor.UrlField': 'MIT (README of github.com/h5p/h5p-editor-url-field)',
'H5PEditor.Wizard': 'MIT (README of github.com/h5p/h5p-editor-wizard)',
flowplayer: 'GPL-3.0 (header of scripts/flowplayer-3.2.12.min.js)'
}

Expand All @@ -104,8 +131,8 @@ const hubBundle = (machineName) => cachedBundle(machineName, { refresh, optional
*/
const download = (url, name, optional = false) => cachedDownload(url, name, { refresh, optional, log: LOG })

/** folder -> { library, files: Map<name, Uint8Array> }, keeping the newest minor per major. */
const kept = new Map()
/** `<machineName>-<major>.<minor>` -> { folder, library, files: Map<name, Uint8Array> }: every version the bundles carry, at its newest patch. */
const seen = new Map()
const skipped = []

for (const machineName of CONTENT_TYPES) {
Expand All @@ -118,7 +145,7 @@ for (const machineName of CONTENT_TYPES) {
}
for (const url of ADDONS) await keepLibraries(await download(url, url.slice(url.lastIndexOf('/') + 1)))

/** Keeps every library folder in a bundle that is newer than the one kept so far. */
/** Keeps every library folder in a bundle, each version at the newest patch seen so far. */
async function keepLibraries(path) {
const reader = new ZipReader(new BlobReader(await openAsBlob(path)))
const entries = (await reader.getEntries({ filenameValidation: 'tolerant' })).filter((entry) => !entry.directory)
Expand All @@ -128,7 +155,7 @@ async function keepLibraries(path) {
const slash = entry.filename.indexOf('/')
if (slash < 0) continue
const folder = entry.filename.slice(0, slash)
if (folder === 'content' || folder.startsWith('H5PEditor.')) continue
if (folder === 'content') continue
if (!folders.has(folder)) folders.set(folder, [])
folders.get(folder).push(entry)
}
Expand All @@ -137,18 +164,46 @@ async function keepLibraries(path) {
const libraryEntry = files.find((entry) => entry.filename === `${folder}/library.json`)
if (!libraryEntry) continue
const library = JSON.parse(new TextDecoder().decode(await libraryEntry.getData(new Uint8ArrayWriter())))
const key = `${library.machineName}-${library.majorVersion}`
const current = kept.get(key)
const current = seen.get(versionOf(library))
if (current && !newer(library, current.library)) continue

const contents = new Map()
for (const entry of files) contents.set(entry.filename, await entry.getData(new Uint8ArrayWriter()))
kept.set(key, { folder, library, files: contents })
seen.set(versionOf(library), { folder, library, files: contents })
}
await reader.close()
}

const libraries = [...kept.values()].sort((a, b) => a.folder.localeCompare(b.folder))
// What the player serves: the newest minor of each major, the editor's libraries left out.
const played = new Map()
for (const version of seen.values()) {
if (version.library.machineName.startsWith('H5PEditor.')) continue
const key = `${version.library.machineName}-${version.library.majorVersion}`
const current = played.get(key)
if (!current || newer(version.library, current.library)) played.set(key, version)
}

// With them, what an H5P site's upload check wants (#146): every library a carried one names,
// preloaded, dynamic or editor, at the exact major.minor, and what those name in turn. A bundle
// carries its content type's whole closure, so each is in `seen`; one that is not stops the build.
const carried = new Map([...played.values()].map((version) => [versionOf(version.library), version]))
const queue = [...carried.values()]
while (queue.length > 0) {
const { folder, library } = queue.shift()
for (const dependency of [...(library.preloadedDependencies ?? []), ...(library.dynamicDependencies ?? []), ...(library.editorDependencies ?? [])]) {
const wanted = versionOf(dependency)
if (carried.has(wanted)) continue
const found = seen.get(wanted)
if (!found) {
console.error(`${LOG} ${folder} needs ${wanted}, which no bundle carries`)
process.exit(1)
}
carried.set(wanted, found)
queue.push(found)
}
}

const libraries = [...carried.values()].sort((a, b) => a.folder.localeCompare(b.folder))

const unknown = libraries.filter(({ library }) => !library.license && !UNDECLARED[library.machineName])
if (unknown.length > 0) {
Expand All @@ -169,7 +224,9 @@ await writePackModule(parts, outDir)

const lines = [
'Library pack of H5P Offline Player by Missing Elements: the H5P hub runtime libraries the installable app carries,',
'so that a package exported without its own libraries plays offline. Each library is used here',
'so that a package exported without its own libraries plays offline, and with them the editor',
"libraries and older minors an H5P site's upload check wants of them, which h5p-author carries",
'into a package and the player never serves (installOnly in parts/index.json). Each library is used here',
'under the licence below; its source is the library folder itself, as shipped by the hub',
`(${HUB}<content type>), or for an addon as H5P Group publishes it (${ADDONS.join(', ')}).`,
'Built by apps/demo/scripts/build-libraries.mjs.',
Expand All @@ -185,11 +242,16 @@ const lines = [
await writeFile(join(outDir, 'libraries.txt'), lines.join('\n'))

console.log(
`[libraries] ${libraries.length} libraries from ${CONTENT_TYPES.length - skipped.length} content types -> packages/libraries/libraries.h5p ${formatBytes(pack.length)}, and ${parts.length} parts` +
`[libraries] ${played.size} libraries the player serves and ${libraries.length - played.size} for H5P sites' upload check, from ${CONTENT_TYPES.length - skipped.length} content types -> packages/libraries/libraries.h5p ${formatBytes(pack.length)}, and ${parts.length} parts` +
(skipped.length ? ` (not on the hub: ${skipped.join(', ')})` : '')
)

/** Whether `a` is a newer release than `b` of the same major. */
function newer(a, b) {
return a.minorVersion !== b.minorVersion ? a.minorVersion > b.minorVersion : (a.patchVersion ?? 0) > (b.patchVersion ?? 0)
}

/** `<machineName>-<major>.<minor>`, of a `library.json` or a dependency it names. */
function versionOf(library) {
return `${library.machineName}-${library.majorVersion}.${library.minorVersion}`
}
6 changes: 5 additions & 1 deletion apps/skills/tests/author-verify.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { tmpdir } from 'node:os'
import { join, resolve } from 'node:path'
import { crc32, deflateSync } from 'node:zlib'
import { BlobReader, TextWriter, Uint8ArrayWriter, ZipReader, type FileEntry } from '@zip.js/zip.js'
import { buildPackage, checkContent, loadLibraries, packageLibraries, schemaOf } from '@missing-elements/h5p-author'
import { buildPackage, checkContent, loadLibraries, missingForInstall, packageLibraries, schemaOf } from '@missing-elements/h5p-author'
import { verifyPackage } from '@missing-elements/h5p-verify'
import { afterAll, beforeAll, describe, expect, it } from 'vitest'

Expand Down Expand Up @@ -122,6 +122,10 @@ async function checksAsSaved(file: string) {
const checked = checkContent(await loadLibraries(await packageLibraries(files), content), content)
expect([checked.problems, checked.notes]).toEqual([[], []])
expect(checked.params).toEqual(params)
// What H5P core's upload check wants, which a Moodle with no library installed refused of 28
// of these 32 before (#146): every library a carried one names, at the exact version, carried.
const libraries = await Promise.all([...entries.keys()].filter((name) => /^[^/]+\/library\.json$/.test(name)).map(async (name) => JSON.parse(await files.read(name))))
expect(missingForInstall(libraries)).toEqual([])
} finally {
await reader.close()
}
Expand Down
Loading
Loading