Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion design/serving.md
Original file line number Diff line number Diff line change
Expand Up @@ -251,7 +251,12 @@ How a package's bytes reach the H5P runtime: the Service Worker's virtual file s
the Jobs worker publishes what it has found as a `ForwardIndexSnapshot` beside the watermark,
announced on the same channel; the Service Worker builds a `partial` `PackageReader` from it and
serves from that until the archive is whole, when a reader from the central directory — the
authority — replaces it. Three rules keep a partial index honest:
authority — replaces it. Each progress report goes out once its snapshot is stored
(`IndexPublisher`, #144): the element asks whether the frame can boot on every report, and the
worker answers from the stored snapshot, so a report sent first was answered from the one
before, and the next report was a chunk away. For a package under two chunks the first report
is the last before the end, and the frame waited for the whole download; CI's Firefox runner
lost that race now and then (#75). Three rules keep a partial index honest:
- **An entry is in the snapshot only once its bytes have all arrived.** Recording it off its
header alone would let the worker reach for data that is not there yet.
- **The element boots only when `bootReady()` holds:** `h5p.json` has arrived and every
Expand Down
56 changes: 56 additions & 0 deletions packages/player/src/jobs/index-publisher.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
import type { ForwardIndexSnapshot } from '../shared/forward-index.js'

/**
* What a download from a host that ignores `Range` publishes as it goes: the entries its local
* headers have given up so far, stored beside the watermark for the Service Worker to serve from,
* and the progress reports, each sent once what it reports is stored.
*
* The order is the point (#144). On each report the element asks the Service Worker whether the
* frame can boot yet (`downloadAndIndex`), and the worker answers from the stored snapshot. A
* report sent before its snapshot was stored was answered from the one before, and nothing asked
* again until the next report, a chunk later. For a package under two chunks, the first report is
* the last before the end, so the frame waited for the whole download.
*/
export class IndexPublisher {
private published = -1
private chain: Promise<void> = Promise.resolve()
private refused = false

constructor(
private readonly scanner: { snapshot(): ForwardIndexSnapshot; finish(): void },
private readonly write: (snapshot: ForwardIndexSnapshot) => Promise<void>
) {}

/**
* Stores what the scanner has found, when it has found more than was stored; with `final`, its
* last word, whatever it holds. Serialised: two snapshots racing to the same record could land
* the older one last. A snapshot the store refuses is dropped and the chain goes on from the
* next one: the index is what lets the frame boot before the download is whole, and the central
* directory is the authority once it is, so a refused write costs an early boot and never the
* download. Caught on each link, because a rejected link would skip every later one and come
* back as an unhandled rejection on each `void` call, and fail the download at the final publish.
*/
publish(final: boolean): Promise<void> {
if (final) this.scanner.finish()
const snapshot = this.scanner.snapshot()
if (!final && snapshot.entries.length === this.published) return this.chain
this.published = snapshot.entries.length
this.chain = this.chain
.then(() => this.write(snapshot))
.catch((error: unknown) => {
if (this.refused) return
this.refused = true
console.warn('[h5p-player] the forward index could not be written; the frame boots once the download is whole:', error)
})
return this.chain
}

/**
* Sends a progress report once the snapshot of what has arrived is stored, or refused. With
* nothing new to store, it waits only for the writes already under way, so reports go out in
* the order they were made, and the last of them before the final publish.
*/
report(send: () => void): void {
void this.publish(false).then(send)
}
}
34 changes: 6 additions & 28 deletions packages/player/src/jobs/jobs-worker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import { streamSpan } from '../shared/span-stream.js'
import { inflated } from '../shared/stream-utils.js'
import { quotaMessage } from '../shared/storage.js'
import { createChunkWriter } from './chunk-writer.js'
import { IndexPublisher } from './index-publisher.js'
import { JobQueue, type ExtractJob } from './job-queue.js'
import { warmPackage } from './warm.js'

Expand Down Expand Up @@ -273,29 +274,9 @@ async function downloadArchive(
)
}

let published = -1
let publishing: Promise<void> = Promise.resolve()
let indexRefused = false
const publishIndex = (final: boolean) => {
if (final) scanner.finish()
const snapshot = scanner.snapshot()
if (!final && snapshot.entries.length === published) return publishing
published = snapshot.entries.length
// Serialised: two snapshots racing to the same record could land the older one last. A
// snapshot the store refuses is dropped and the chain goes on from the next one: the index
// is what lets the frame boot before the download is whole, and the central directory is
// the authority once it is, so a refused write costs an early boot and never the download.
// Caught on each link, because a rejected link would skip every later one and come back as
// an unhandled rejection on each `void` call, and fail the download at the final publish.
publishing = publishing
.then(() => store.setForwardIndex(snapshot))
.catch((error: unknown) => {
if (indexRefused) return
indexRefused = true
console.warn('[h5p-player] the forward index could not be written; the frame boots once the download is whole:', error)
})
return publishing
}
// Each progress report goes out once its snapshot is stored: the element asks whether the frame
// can boot on every report, and the Service Worker answers from the stored snapshot (#144).
const index = new IndexPublisher(scanner, (snapshot) => store.setForwardIndex(snapshot))

const indexer = new TransformStream<Uint8Array, Uint8Array>({
transform(chunk, controller) {
Expand All @@ -311,10 +292,7 @@ async function downloadArchive(
entry: ARCHIVE_ENTRY,
totalSize,
startOffset,
onProgress: (loaded) => {
send({ type: 'progress', pkgId, loaded, total: totalSize })
void publishIndex(false)
}
onProgress: (loaded) => index.report(() => send({ type: 'progress', pkgId, loaded, total: totalSize }))
}),
{ signal }
)
Expand All @@ -327,7 +305,7 @@ async function downloadArchive(
'downloads, so the whole archive has to be stored before it can play.'
)
}
await publishIndex(true)
await index.publish(true)

// Read back, because WebKit can lose it: measured on 2026-10-13 in Playwright's WebKit, a
// package downloaded whole, its element removed — which terminates this worker — and played
Expand Down
131 changes: 131 additions & 0 deletions packages/player/tests/unit/index-publisher.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import { IndexPublisher } from '../../src/jobs/index-publisher'
import type { ForwardEntry, ForwardIndexSnapshot } from '../../src/shared/forward-index'

/**
* The forward index a download publishes, and the progress reports after it (#144). The element
* asks the Service Worker whether the frame can boot on each report, and the worker answers from
* the stored snapshot: a report that overtook its write was answered from the one before.
*/

/** A scanner the test feeds entries to by name. */
function scanner() {
const names: string[] = []
let finished = false
return {
add: (name: string) => void names.push(name),
get finished() {
return finished
},
finish: () => void (finished = true),
snapshot: (): ForwardIndexSnapshot => ({
entries: names.map((name) => ({ name }) as unknown as ForwardEntry),
parsedTo: names.length,
done: finished,
stopped: null
})
}
}

/** A store whose writes land when the test says, each named by how many entries it holds. */
function store() {
const writes: Array<{ entries: number; land: () => void; refuse: (error: unknown) => void }> = []
const write = (snapshot: ForwardIndexSnapshot) =>
new Promise<void>((land, refuse) => writes.push({ entries: snapshot.entries.length, land, refuse }))
return { writes, write }
}

/** Long enough for every settled promise's callbacks to have run. */
const settled = () => new Promise((resolve) => setTimeout(resolve, 0))

describe('IndexPublisher', () => {
afterEach(() => vi.restoreAllMocks())

it('sends a report only once the snapshot it reports is stored', async () => {
const found = scanner()
const { writes, write } = store()
const index = new IndexPublisher(found, write)
const sent: string[] = []

found.add('h5p.json')
index.report(() => sent.push('256 kB'))
await settled()
// The write is under way: a report now would be answered from no snapshot at all.
expect(writes.map((each) => each.entries)).toEqual([1])
expect(sent).toEqual([])

writes[0].land()
await settled()
expect(sent).toEqual(['256 kB'])
})

it('keeps the reports in order: one with nothing new waits for the writes under way', async () => {
const found = scanner()
const { writes, write } = store()
const index = new IndexPublisher(found, write)
const sent: string[] = []

found.add('h5p.json')
index.report(() => sent.push('first'))
// Nothing new: no write of its own, and it goes out after the first.
index.report(() => sent.push('second'))
found.add('content/content.json')
index.report(() => sent.push('third'))
await settled()
// One write at a time: the third's starts once the first's has landed.
expect(writes.map((each) => each.entries)).toEqual([1])

writes[0].land()
await settled()
expect(sent).toEqual(['first', 'second'])
expect(writes.map((each) => each.entries)).toEqual([1, 2])

writes[1].land()
await settled()
expect(sent).toEqual(['first', 'second', 'third'])
})

it('still sends the report when the store refuses the snapshot, and says so once', async () => {
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
const found = scanner()
const { writes, write } = store()
const index = new IndexPublisher(found, write)
const sent: string[] = []

found.add('h5p.json')
index.report(() => sent.push('first'))
found.add('content/content.json')
index.report(() => sent.push('second'))
await settled()
writes[0].refuse(new Error('QuotaExceededError'))
await settled()
writes[1].refuse(new Error('QuotaExceededError'))
await settled()

expect(sent).toEqual(['first', 'second'])
expect(warn).toHaveBeenCalledTimes(1)
})

it('publishes the finished scan at the end, after every write before it, even with nothing new', async () => {
const found = scanner()
const { writes, write } = store()
const index = new IndexPublisher(found, write)

found.add('h5p.json')
index.report(() => {})
let ended = false
const end = index.publish(true).then(() => void (ended = true))
await settled()
expect(found.finished).toBe(true)
expect(writes.map((each) => each.entries)).toEqual([1])

writes[0].land()
await settled()
// The final snapshot is written although it holds no entry more: it is the one marked done.
expect(writes.map((each) => each.entries)).toEqual([1, 1])
expect(ended).toBe(false)
writes[1].land()
await end
expect(ended).toBe(true)
})
})
Loading