From 19a1a98737eb7292e1e81146bfbc1f4e4b770961 Mon Sep 17 00:00:00 2001 From: Aleksandr Shevchuk Date: Sat, 7 Nov 2026 16:11:05 +0200 Subject: [PATCH] fix(player): send a download's progress once its forward index is stored On a host that ignores Range, the Jobs worker sent each progress report before it stored the snapshot of the entries found so far. The element asks whether the frame can boot on every report, and the Service Worker answers from the stored snapshot, so a report that overtook its write was answered from the one before, and the next report was a chunk away. For a package under two chunks that meant the whole download. The publishing moves into IndexPublisher, whose report() sends once the write it waits for has landed or been refused, in order. Closes #144. --- design/serving.md | 7 +- packages/player/src/jobs/index-publisher.ts | 56 ++++++++ packages/player/src/jobs/jobs-worker.ts | 34 +---- .../player/tests/unit/index-publisher.test.ts | 131 ++++++++++++++++++ 4 files changed, 199 insertions(+), 29 deletions(-) create mode 100644 packages/player/src/jobs/index-publisher.ts create mode 100644 packages/player/tests/unit/index-publisher.test.ts diff --git a/design/serving.md b/design/serving.md index 6d657f2f..07f35984 100644 --- a/design/serving.md +++ b/design/serving.md @@ -251,7 +251,12 @@ How a package's bytes reach the H5P runtime: the Service Worker's virtual file s the Jobs worker publishes what it has found as a `ForwardIndexSnapshot` beside the watermark, announced on the same channel; the Service Worker builds a `partial` `PackageReader` from it and serves from that until the archive is whole, when a reader from the central directory — the - authority — replaces it. Three rules keep a partial index honest: + authority — replaces it. Each progress report goes out once its snapshot is stored + (`IndexPublisher`, #144): the element asks whether the frame can boot on every report, and the + worker answers from the stored snapshot, so a report sent first was answered from the one + before, and the next report was a chunk away. For a package under two chunks the first report + is the last before the end, and the frame waited for the whole download; CI's Firefox runner + lost that race now and then (#75). Three rules keep a partial index honest: - **An entry is in the snapshot only once its bytes have all arrived.** Recording it off its header alone would let the worker reach for data that is not there yet. - **The element boots only when `bootReady()` holds:** `h5p.json` has arrived and every diff --git a/packages/player/src/jobs/index-publisher.ts b/packages/player/src/jobs/index-publisher.ts new file mode 100644 index 00000000..4a178a85 --- /dev/null +++ b/packages/player/src/jobs/index-publisher.ts @@ -0,0 +1,56 @@ +import type { ForwardIndexSnapshot } from '../shared/forward-index.js' + +/** + * What a download from a host that ignores `Range` publishes as it goes: the entries its local + * headers have given up so far, stored beside the watermark for the Service Worker to serve from, + * and the progress reports, each sent once what it reports is stored. + * + * The order is the point (#144). On each report the element asks the Service Worker whether the + * frame can boot yet (`downloadAndIndex`), and the worker answers from the stored snapshot. A + * report sent before its snapshot was stored was answered from the one before, and nothing asked + * again until the next report, a chunk later. For a package under two chunks, the first report is + * the last before the end, so the frame waited for the whole download. + */ +export class IndexPublisher { + private published = -1 + private chain: Promise = Promise.resolve() + private refused = false + + constructor( + private readonly scanner: { snapshot(): ForwardIndexSnapshot; finish(): void }, + private readonly write: (snapshot: ForwardIndexSnapshot) => Promise + ) {} + + /** + * Stores what the scanner has found, when it has found more than was stored; with `final`, its + * last word, whatever it holds. Serialised: two snapshots racing to the same record could land + * the older one last. A snapshot the store refuses is dropped and the chain goes on from the + * next one: the index is what lets the frame boot before the download is whole, and the central + * directory is the authority once it is, so a refused write costs an early boot and never the + * download. Caught on each link, because a rejected link would skip every later one and come + * back as an unhandled rejection on each `void` call, and fail the download at the final publish. + */ + publish(final: boolean): Promise { + if (final) this.scanner.finish() + const snapshot = this.scanner.snapshot() + if (!final && snapshot.entries.length === this.published) return this.chain + this.published = snapshot.entries.length + this.chain = this.chain + .then(() => this.write(snapshot)) + .catch((error: unknown) => { + if (this.refused) return + this.refused = true + console.warn('[h5p-player] the forward index could not be written; the frame boots once the download is whole:', error) + }) + return this.chain + } + + /** + * Sends a progress report once the snapshot of what has arrived is stored, or refused. With + * nothing new to store, it waits only for the writes already under way, so reports go out in + * the order they were made, and the last of them before the final publish. + */ + report(send: () => void): void { + void this.publish(false).then(send) + } +} diff --git a/packages/player/src/jobs/jobs-worker.ts b/packages/player/src/jobs/jobs-worker.ts index b88c51a1..d05837d4 100644 --- a/packages/player/src/jobs/jobs-worker.ts +++ b/packages/player/src/jobs/jobs-worker.ts @@ -21,6 +21,7 @@ import { streamSpan } from '../shared/span-stream.js' import { inflated } from '../shared/stream-utils.js' import { quotaMessage } from '../shared/storage.js' import { createChunkWriter } from './chunk-writer.js' +import { IndexPublisher } from './index-publisher.js' import { JobQueue, type ExtractJob } from './job-queue.js' import { warmPackage } from './warm.js' @@ -273,29 +274,9 @@ async function downloadArchive( ) } - let published = -1 - let publishing: Promise = Promise.resolve() - let indexRefused = false - const publishIndex = (final: boolean) => { - if (final) scanner.finish() - const snapshot = scanner.snapshot() - if (!final && snapshot.entries.length === published) return publishing - published = snapshot.entries.length - // Serialised: two snapshots racing to the same record could land the older one last. A - // snapshot the store refuses is dropped and the chain goes on from the next one: the index - // is what lets the frame boot before the download is whole, and the central directory is - // the authority once it is, so a refused write costs an early boot and never the download. - // Caught on each link, because a rejected link would skip every later one and come back as - // an unhandled rejection on each `void` call, and fail the download at the final publish. - publishing = publishing - .then(() => store.setForwardIndex(snapshot)) - .catch((error: unknown) => { - if (indexRefused) return - indexRefused = true - console.warn('[h5p-player] the forward index could not be written; the frame boots once the download is whole:', error) - }) - return publishing - } + // Each progress report goes out once its snapshot is stored: the element asks whether the frame + // can boot on every report, and the Service Worker answers from the stored snapshot (#144). + const index = new IndexPublisher(scanner, (snapshot) => store.setForwardIndex(snapshot)) const indexer = new TransformStream({ transform(chunk, controller) { @@ -311,10 +292,7 @@ async function downloadArchive( entry: ARCHIVE_ENTRY, totalSize, startOffset, - onProgress: (loaded) => { - send({ type: 'progress', pkgId, loaded, total: totalSize }) - void publishIndex(false) - } + onProgress: (loaded) => index.report(() => send({ type: 'progress', pkgId, loaded, total: totalSize })) }), { signal } ) @@ -327,7 +305,7 @@ async function downloadArchive( 'downloads, so the whole archive has to be stored before it can play.' ) } - await publishIndex(true) + await index.publish(true) // Read back, because WebKit can lose it: measured on 2026-10-13 in Playwright's WebKit, a // package downloaded whole, its element removed — which terminates this worker — and played diff --git a/packages/player/tests/unit/index-publisher.test.ts b/packages/player/tests/unit/index-publisher.test.ts new file mode 100644 index 00000000..e906deab --- /dev/null +++ b/packages/player/tests/unit/index-publisher.test.ts @@ -0,0 +1,131 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { IndexPublisher } from '../../src/jobs/index-publisher' +import type { ForwardEntry, ForwardIndexSnapshot } from '../../src/shared/forward-index' + +/** + * The forward index a download publishes, and the progress reports after it (#144). The element + * asks the Service Worker whether the frame can boot on each report, and the worker answers from + * the stored snapshot: a report that overtook its write was answered from the one before. + */ + +/** A scanner the test feeds entries to by name. */ +function scanner() { + const names: string[] = [] + let finished = false + return { + add: (name: string) => void names.push(name), + get finished() { + return finished + }, + finish: () => void (finished = true), + snapshot: (): ForwardIndexSnapshot => ({ + entries: names.map((name) => ({ name }) as unknown as ForwardEntry), + parsedTo: names.length, + done: finished, + stopped: null + }) + } +} + +/** A store whose writes land when the test says, each named by how many entries it holds. */ +function store() { + const writes: Array<{ entries: number; land: () => void; refuse: (error: unknown) => void }> = [] + const write = (snapshot: ForwardIndexSnapshot) => + new Promise((land, refuse) => writes.push({ entries: snapshot.entries.length, land, refuse })) + return { writes, write } +} + +/** Long enough for every settled promise's callbacks to have run. */ +const settled = () => new Promise((resolve) => setTimeout(resolve, 0)) + +describe('IndexPublisher', () => { + afterEach(() => vi.restoreAllMocks()) + + it('sends a report only once the snapshot it reports is stored', async () => { + const found = scanner() + const { writes, write } = store() + const index = new IndexPublisher(found, write) + const sent: string[] = [] + + found.add('h5p.json') + index.report(() => sent.push('256 kB')) + await settled() + // The write is under way: a report now would be answered from no snapshot at all. + expect(writes.map((each) => each.entries)).toEqual([1]) + expect(sent).toEqual([]) + + writes[0].land() + await settled() + expect(sent).toEqual(['256 kB']) + }) + + it('keeps the reports in order: one with nothing new waits for the writes under way', async () => { + const found = scanner() + const { writes, write } = store() + const index = new IndexPublisher(found, write) + const sent: string[] = [] + + found.add('h5p.json') + index.report(() => sent.push('first')) + // Nothing new: no write of its own, and it goes out after the first. + index.report(() => sent.push('second')) + found.add('content/content.json') + index.report(() => sent.push('third')) + await settled() + // One write at a time: the third's starts once the first's has landed. + expect(writes.map((each) => each.entries)).toEqual([1]) + + writes[0].land() + await settled() + expect(sent).toEqual(['first', 'second']) + expect(writes.map((each) => each.entries)).toEqual([1, 2]) + + writes[1].land() + await settled() + expect(sent).toEqual(['first', 'second', 'third']) + }) + + it('still sends the report when the store refuses the snapshot, and says so once', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + const found = scanner() + const { writes, write } = store() + const index = new IndexPublisher(found, write) + const sent: string[] = [] + + found.add('h5p.json') + index.report(() => sent.push('first')) + found.add('content/content.json') + index.report(() => sent.push('second')) + await settled() + writes[0].refuse(new Error('QuotaExceededError')) + await settled() + writes[1].refuse(new Error('QuotaExceededError')) + await settled() + + expect(sent).toEqual(['first', 'second']) + expect(warn).toHaveBeenCalledTimes(1) + }) + + it('publishes the finished scan at the end, after every write before it, even with nothing new', async () => { + const found = scanner() + const { writes, write } = store() + const index = new IndexPublisher(found, write) + + found.add('h5p.json') + index.report(() => {}) + let ended = false + const end = index.publish(true).then(() => void (ended = true)) + await settled() + expect(found.finished).toBe(true) + expect(writes.map((each) => each.entries)).toEqual([1]) + + writes[0].land() + await settled() + // The final snapshot is written although it holds no entry more: it is the one marked done. + expect(writes.map((each) => each.entries)).toEqual([1, 1]) + expect(ended).toBe(false) + writes[1].land() + await end + expect(ended).toBe(true) + }) +})