diff --git a/design/serving.md b/design/serving.md index 6d657f2f..07f35984 100644 --- a/design/serving.md +++ b/design/serving.md @@ -251,7 +251,12 @@ How a package's bytes reach the H5P runtime: the Service Worker's virtual file s the Jobs worker publishes what it has found as a `ForwardIndexSnapshot` beside the watermark, announced on the same channel; the Service Worker builds a `partial` `PackageReader` from it and serves from that until the archive is whole, when a reader from the central directory — the - authority — replaces it. Three rules keep a partial index honest: + authority — replaces it. Each progress report goes out once its snapshot is stored + (`IndexPublisher`, #144): the element asks whether the frame can boot on every report, and the + worker answers from the stored snapshot, so a report sent first was answered from the one + before, and the next report was a chunk away. For a package under two chunks the first report + is the last before the end, and the frame waited for the whole download; CI's Firefox runner + lost that race now and then (#75). Three rules keep a partial index honest: - **An entry is in the snapshot only once its bytes have all arrived.** Recording it off its header alone would let the worker reach for data that is not there yet. - **The element boots only when `bootReady()` holds:** `h5p.json` has arrived and every diff --git a/packages/player/src/jobs/index-publisher.ts b/packages/player/src/jobs/index-publisher.ts new file mode 100644 index 00000000..4a178a85 --- /dev/null +++ b/packages/player/src/jobs/index-publisher.ts @@ -0,0 +1,56 @@ +import type { ForwardIndexSnapshot } from '../shared/forward-index.js' + +/** + * What a download from a host that ignores `Range` publishes as it goes: the entries its local + * headers have given up so far, stored beside the watermark for the Service Worker to serve from, + * and the progress reports, each sent once what it reports is stored. + * + * The order is the point (#144). On each report the element asks the Service Worker whether the + * frame can boot yet (`downloadAndIndex`), and the worker answers from the stored snapshot. A + * report sent before its snapshot was stored was answered from the one before, and nothing asked + * again until the next report, a chunk later. For a package under two chunks, the first report is + * the last before the end, so the frame waited for the whole download. + */ +export class IndexPublisher { + private published = -1 + private chain: Promise = Promise.resolve() + private refused = false + + constructor( + private readonly scanner: { snapshot(): ForwardIndexSnapshot; finish(): void }, + private readonly write: (snapshot: ForwardIndexSnapshot) => Promise + ) {} + + /** + * Stores what the scanner has found, when it has found more than was stored; with `final`, its + * last word, whatever it holds. Serialised: two snapshots racing to the same record could land + * the older one last. A snapshot the store refuses is dropped and the chain goes on from the + * next one: the index is what lets the frame boot before the download is whole, and the central + * directory is the authority once it is, so a refused write costs an early boot and never the + * download. Caught on each link, because a rejected link would skip every later one and come + * back as an unhandled rejection on each `void` call, and fail the download at the final publish. + */ + publish(final: boolean): Promise { + if (final) this.scanner.finish() + const snapshot = this.scanner.snapshot() + if (!final && snapshot.entries.length === this.published) return this.chain + this.published = snapshot.entries.length + this.chain = this.chain + .then(() => this.write(snapshot)) + .catch((error: unknown) => { + if (this.refused) return + this.refused = true + console.warn('[h5p-player] the forward index could not be written; the frame boots once the download is whole:', error) + }) + return this.chain + } + + /** + * Sends a progress report once the snapshot of what has arrived is stored, or refused. With + * nothing new to store, it waits only for the writes already under way, so reports go out in + * the order they were made, and the last of them before the final publish. + */ + report(send: () => void): void { + void this.publish(false).then(send) + } +} diff --git a/packages/player/src/jobs/jobs-worker.ts b/packages/player/src/jobs/jobs-worker.ts index b88c51a1..d05837d4 100644 --- a/packages/player/src/jobs/jobs-worker.ts +++ b/packages/player/src/jobs/jobs-worker.ts @@ -21,6 +21,7 @@ import { streamSpan } from '../shared/span-stream.js' import { inflated } from '../shared/stream-utils.js' import { quotaMessage } from '../shared/storage.js' import { createChunkWriter } from './chunk-writer.js' +import { IndexPublisher } from './index-publisher.js' import { JobQueue, type ExtractJob } from './job-queue.js' import { warmPackage } from './warm.js' @@ -273,29 +274,9 @@ async function downloadArchive( ) } - let published = -1 - let publishing: Promise = Promise.resolve() - let indexRefused = false - const publishIndex = (final: boolean) => { - if (final) scanner.finish() - const snapshot = scanner.snapshot() - if (!final && snapshot.entries.length === published) return publishing - published = snapshot.entries.length - // Serialised: two snapshots racing to the same record could land the older one last. A - // snapshot the store refuses is dropped and the chain goes on from the next one: the index - // is what lets the frame boot before the download is whole, and the central directory is - // the authority once it is, so a refused write costs an early boot and never the download. - // Caught on each link, because a rejected link would skip every later one and come back as - // an unhandled rejection on each `void` call, and fail the download at the final publish. - publishing = publishing - .then(() => store.setForwardIndex(snapshot)) - .catch((error: unknown) => { - if (indexRefused) return - indexRefused = true - console.warn('[h5p-player] the forward index could not be written; the frame boots once the download is whole:', error) - }) - return publishing - } + // Each progress report goes out once its snapshot is stored: the element asks whether the frame + // can boot on every report, and the Service Worker answers from the stored snapshot (#144). + const index = new IndexPublisher(scanner, (snapshot) => store.setForwardIndex(snapshot)) const indexer = new TransformStream({ transform(chunk, controller) { @@ -311,10 +292,7 @@ async function downloadArchive( entry: ARCHIVE_ENTRY, totalSize, startOffset, - onProgress: (loaded) => { - send({ type: 'progress', pkgId, loaded, total: totalSize }) - void publishIndex(false) - } + onProgress: (loaded) => index.report(() => send({ type: 'progress', pkgId, loaded, total: totalSize })) }), { signal } ) @@ -327,7 +305,7 @@ async function downloadArchive( 'downloads, so the whole archive has to be stored before it can play.' ) } - await publishIndex(true) + await index.publish(true) // Read back, because WebKit can lose it: measured on 2026-10-13 in Playwright's WebKit, a // package downloaded whole, its element removed — which terminates this worker — and played diff --git a/packages/player/tests/unit/index-publisher.test.ts b/packages/player/tests/unit/index-publisher.test.ts new file mode 100644 index 00000000..e906deab --- /dev/null +++ b/packages/player/tests/unit/index-publisher.test.ts @@ -0,0 +1,131 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { IndexPublisher } from '../../src/jobs/index-publisher' +import type { ForwardEntry, ForwardIndexSnapshot } from '../../src/shared/forward-index' + +/** + * The forward index a download publishes, and the progress reports after it (#144). The element + * asks the Service Worker whether the frame can boot on each report, and the worker answers from + * the stored snapshot: a report that overtook its write was answered from the one before. + */ + +/** A scanner the test feeds entries to by name. */ +function scanner() { + const names: string[] = [] + let finished = false + return { + add: (name: string) => void names.push(name), + get finished() { + return finished + }, + finish: () => void (finished = true), + snapshot: (): ForwardIndexSnapshot => ({ + entries: names.map((name) => ({ name }) as unknown as ForwardEntry), + parsedTo: names.length, + done: finished, + stopped: null + }) + } +} + +/** A store whose writes land when the test says, each named by how many entries it holds. */ +function store() { + const writes: Array<{ entries: number; land: () => void; refuse: (error: unknown) => void }> = [] + const write = (snapshot: ForwardIndexSnapshot) => + new Promise((land, refuse) => writes.push({ entries: snapshot.entries.length, land, refuse })) + return { writes, write } +} + +/** Long enough for every settled promise's callbacks to have run. */ +const settled = () => new Promise((resolve) => setTimeout(resolve, 0)) + +describe('IndexPublisher', () => { + afterEach(() => vi.restoreAllMocks()) + + it('sends a report only once the snapshot it reports is stored', async () => { + const found = scanner() + const { writes, write } = store() + const index = new IndexPublisher(found, write) + const sent: string[] = [] + + found.add('h5p.json') + index.report(() => sent.push('256 kB')) + await settled() + // The write is under way: a report now would be answered from no snapshot at all. + expect(writes.map((each) => each.entries)).toEqual([1]) + expect(sent).toEqual([]) + + writes[0].land() + await settled() + expect(sent).toEqual(['256 kB']) + }) + + it('keeps the reports in order: one with nothing new waits for the writes under way', async () => { + const found = scanner() + const { writes, write } = store() + const index = new IndexPublisher(found, write) + const sent: string[] = [] + + found.add('h5p.json') + index.report(() => sent.push('first')) + // Nothing new: no write of its own, and it goes out after the first. + index.report(() => sent.push('second')) + found.add('content/content.json') + index.report(() => sent.push('third')) + await settled() + // One write at a time: the third's starts once the first's has landed. + expect(writes.map((each) => each.entries)).toEqual([1]) + + writes[0].land() + await settled() + expect(sent).toEqual(['first', 'second']) + expect(writes.map((each) => each.entries)).toEqual([1, 2]) + + writes[1].land() + await settled() + expect(sent).toEqual(['first', 'second', 'third']) + }) + + it('still sends the report when the store refuses the snapshot, and says so once', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + const found = scanner() + const { writes, write } = store() + const index = new IndexPublisher(found, write) + const sent: string[] = [] + + found.add('h5p.json') + index.report(() => sent.push('first')) + found.add('content/content.json') + index.report(() => sent.push('second')) + await settled() + writes[0].refuse(new Error('QuotaExceededError')) + await settled() + writes[1].refuse(new Error('QuotaExceededError')) + await settled() + + expect(sent).toEqual(['first', 'second']) + expect(warn).toHaveBeenCalledTimes(1) + }) + + it('publishes the finished scan at the end, after every write before it, even with nothing new', async () => { + const found = scanner() + const { writes, write } = store() + const index = new IndexPublisher(found, write) + + found.add('h5p.json') + index.report(() => {}) + let ended = false + const end = index.publish(true).then(() => void (ended = true)) + await settled() + expect(found.finished).toBe(true) + expect(writes.map((each) => each.entries)).toEqual([1]) + + writes[0].land() + await settled() + // The final snapshot is written although it holds no entry more: it is the one marked done. + expect(writes.map((each) => each.entries)).toEqual([1, 1]) + expect(ended).toBe(false) + writes[1].land() + await end + expect(ended).toBe(true) + }) +})