From 42c8b728e86f9d34662bbff4096f9d0aac31c64b Mon Sep 17 00:00:00 2001 From: Aleksandr Shevchuk Date: Sat, 7 Nov 2026 16:30:18 +0200 Subject: [PATCH] feat(player): keep the blocks an index was read from with the package The block cache under small reads over HTTP lives in the Service Worker's memory, and Chromium stops an idle worker after about thirty seconds, so a reload read the archive's tail and head again before the worker answered anything. At the end of each index of a whole reader over HTTP, the worker now writes the blocks that reader's small reads went through into the package's cache, with the validator and size they were read under, and openReader puts them back before zip.js opens the archive. Only with a validator; a copy that does not match or does not read back whole is read around. On the demo's quiz, from a host paced to 1 MB/s with 100 ms on every request, a reload after the worker stopped is the probe alone, ready at 0.21 s, where it took four requests and 0.64 s. Closes #136. --- AGENTS.md | 2 +- design/serving.md | 24 +++++- docs/setup.md | 5 +- packages/player/src/shared/block-cache.ts | 34 ++++++++- packages/player/src/shared/chunk-store.ts | 73 +++++++++++++++++++ packages/player/src/shared/source.ts | 40 +++++++++- packages/player/src/sw/mount.ts | 61 +++++++++++++++- .../browser-emulated/worker-stopped.test.ts | 46 ++++++++++++ .../player/tests/unit/block-cache.test.ts | 44 +++++++++++ .../tests/unit/chunk-store-write.test.ts | 25 +++++++ .../tests/unit/range-http-handle.test.ts | 28 ++++++- packages/player/vite.plugins.ts | 3 + 12 files changed, 376 insertions(+), 9 deletions(-) create mode 100644 packages/player/tests/browser-emulated/worker-stopped.test.ts diff --git a/AGENTS.md b/AGENTS.md index 46ca79d2..09d1e9b6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -149,7 +149,7 @@ Read it before changing the Service Worker's file server (`src/sw/mount.ts`, `pa - `SourceReader` overrides `createReadable`. - A large span over HTTP is then pulled by several connections at once. -- Small reads over HTTP go through a block cache, and a rebuilt reader finds it. +- Small reads over HTTP go through a block cache, and a rebuilt reader finds it; with a validator, so does a worker started again. - Waiting on the watermark is bounded by a stall, not by a wall clock. - A ranged read survives the link, and a body the server gives up on ends short. - A body that streams holds its fetch event open. diff --git a/design/serving.md b/design/serving.md index 6d657f2f..1c257b87 100644 --- a/design/serving.md +++ b/design/serving.md @@ -36,7 +36,7 @@ How a package's bytes reach the H5P runtime: the Service Worker's virtual file s keeps a request's bytes as they arrive and the consumer follows it live when its turn comes, and the window fills only after the first byte, which costs a per-connection-capped host one round trip and nothing else. Same 40 MB at 4 Mbit/s now: first byte in under two seconds. -- **Small reads over HTTP go through a block cache, and a rebuilt reader finds it.** zip.js reads +- **Small reads over HTTP go through a block cache, and a rebuilt reader finds it; with a validator, so does a worker started again.** zip.js reads an index in pieces, one after another: 65,557 bytes from the end to find the end-of-central- directory record, then the directory — which in a package of a few hundred entries lies inside those same bytes — then each entry's 30-byte local header, its extra field when it has one, and @@ -57,7 +57,8 @@ How a package's bytes reach the H5P runtime: the Service Worker's virtual file s them in pieces (`blocksFor`). The cache is one per worker, made on first use so the element's bundle does not carry it, keyed by the archive's URL, validator and size — the package id's ingredients — so the new handle a rebuilt reader opens finds the blocks - the first one fetched; it dies with the worker, and it is never authority. The quiz now costs + the first one fetched; it dies with the worker, but for the blocks an index was read from, and + it is never authority. The quiz now costs four: the probe, the tail, the head and the warm. Two things left out on purpose, by the owner's decision the same day: a whole-file download below a size, which with the cache saves one request and needs a threshold; and rules per content type for what to fetch first, which would @@ -65,6 +66,25 @@ How a package's bytes reach the H5P runtime: the Service Worker's virtual file s `tests/unit/range-http-handle.test.ts` a real archive's index over a host that honours `Range`, and `tests/browser/range-requests.test.ts` counts what a load costs at `/counted/`, a host that writes down every request it takes. + + The blocks an index was read from outlive the worker (2026-11-07, #136). Chromium stops an idle + worker after about thirty seconds, so the usual revisit found the cache empty and read the tail + and the head again, one after the other, before the worker answered anything. Measured on the + demo's quiz from a host paced to 1 MB/s with 100 ms on every request, the worker stopped + through CDP between the loads: a reload took four requests and was ready at 0.64 s. Now, at the + end of each `index` of a whole reader over HTTP, `keepIndexBlocks` writes the blocks that + reader's small reads went through (`blocksReadBy`: the index, `content.json`, whatever the load + read before the boot; the player names no file) into the package's cache, with the validator + and the size they were read under. `openReader` puts them back (`seedBlocks`) before zip.js + opens the archive, when both still match, and the same reload is the probe alone, ready at + 0.21 s. Only with a validator: without one, an archive republished at the same URL keeps its + package id, and old blocks would send zip.js to the old offsets, so such a host reads the index + again. A copy that is missing, cut short or of the wrong length is read around, never trusted + (`BlockCache.seed` checks each block's length for its place), and it goes with the package's + cache when that is evicted. The bytes are kept rather than a parsed index, so zip.js stays the + only parser, and the price is one Cache API read in place of a round trip. + `tests/browser-emulated/worker-stopped.test.ts` stops the worker and counts the reload's + requests at `/counted/` with an `ETag` and without. - **Waiting on the watermark is bounded by a stall, not by a wall clock.** Inflating a few hundred megabytes over a network legitimately takes minutes, and a request for the *tail* of such an entry cannot be answered until it finishes: an mp4 whose `moov` index sits at the end — common, diff --git a/docs/setup.md b/docs/setup.md index f7a5a80b..569c3d84 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -14,7 +14,10 @@ caching and upgrades. - The page on `https://`, or `localhost` while developing: the player needs a Service Worker. - CORS headers on the package host, so the page can read the `.h5p`. `Range` support is optional: - without it the archive is downloaded once and played from the cache. + without it the archive is downloaded once and played from the cache. On another origin, add + `Access-Control-Expose-Headers: ETag` (or `Last-Modified`): the player can then tell an archive + republished at the same URL from the old one, and a reload after the browser stopped its + worker reads nothing of the archive but one byte. - A browser with Service Workers, the Cache API, IndexedDB, Web Locks and `DecompressionStream`; [Compatibility](compatibility.md) says which are tested. diff --git a/packages/player/src/shared/block-cache.ts b/packages/player/src/shared/block-cache.ts index 5448f725..08ce742d 100644 --- a/packages/player/src/shared/block-cache.ts +++ b/packages/player/src/shared/block-cache.ts @@ -21,7 +21,10 @@ import type { ByteRange } from './range.js' * bytes out from under it. * * Nothing here is authority: the key names the bytes — the archive's URL, validator and size, the - * ingredients of its package id — and the cache dies with the worker that holds it. + * ingredients of its package id — and the cache dies with the worker that holds it. The blocks + * an index was read from are the exception: kept with the package when the archive has a + * validator (`blocksOf`), and put back by the next worker (`seed`), so a worker the browser + * stopped does not read the index again (#136). */ export class BlockCache { /** Held blocks by `blockId`, oldest use first: a use moves a block to the end. */ @@ -94,6 +97,35 @@ export class BlockCache { return out } + /** + * The blocks of `key` held among `indices`, in that order, for keeping beyond this worker + * (#136): a worker the browser stops takes its cache with it, and the next one read the index + * again. One not held is left out. + */ + blocksOf(key: string, indices: Iterable): Array<{ index: number; bytes: Uint8Array }> { + const out: Array<{ index: number; bytes: Uint8Array }> = [] + for (const index of indices) { + const bytes = this.blocks.get(blockId(key, index)) + if (bytes) out.push({ index, bytes }) + } + return out + } + + /** + * Holds blocks of `key`, in an archive of `size` bytes, that an earlier worker read: from then + * on a read finds them as if fetched. A block of the wrong length for its place is left out, + * and so is one already held or on its way, which is as good or newer. + */ + seed(key: string, size: number, blocks: Iterable<{ index: number; bytes: Uint8Array }>): void { + for (const { index, bytes } of blocks) { + const id = blockId(key, index) + const expected = Math.min(this.blockSize, size - index * this.blockSize) + if (!Number.isInteger(index) || index < 0 || expected <= 0 || bytes.length !== expected) continue + if (this.blocks.has(id) || this.pending.has(id)) continue + this.keep(id, bytes) + } + } + /** Forgets every block held. Blocks on their way still arrive, and are kept. */ clear(): void { this.blocks.clear() diff --git a/packages/player/src/shared/chunk-store.ts b/packages/player/src/shared/chunk-store.ts index b214c128..7c642e21 100644 --- a/packages/player/src/shared/chunk-store.ts +++ b/packages/player/src/shared/chunk-store.ts @@ -80,6 +80,59 @@ function forwardIndexKey(pkgId: string): string { return `${CHUNK_KEY_ORIGIN}${pkgId}/chunk/${encodeEntry(ARCHIVE_ENTRY)}${FORWARD_INDEX_MARKER}` } +/** Beside the package's entries, not among them: the blocks are pieces of the archive, not of an entry. */ +function indexBlocksKey(pkgId: string): string { + return `${CHUNK_KEY_ORIGIN}${pkgId}/index-blocks` +} + +/** + * The blocks an archive's index was read from over HTTP, kept with the package (#136), and what + * they were read under: the host's validator and the archive's size. A worker started again puts + * them back in its block cache only when both still match. + */ +export interface IndexBlocks { + validator: string + size: number + blocks: Array<{ index: number; bytes: Uint8Array }> +} + +/** One body: four bytes giving the length of a JSON head, the head, then each block's bytes in turn. */ +function encodeIndexBlocks({ validator, size, blocks }: IndexBlocks): Uint8Array { + const head = new TextEncoder().encode( + JSON.stringify({ validator, size, blocks: blocks.map(({ index, bytes }) => ({ index, length: bytes.length })) }) + ) + const body = new Uint8Array(4 + head.length + blocks.reduce((sum, { bytes }) => sum + bytes.length, 0)) + new DataView(body.buffer).setUint32(0, head.length) + body.set(head, 4) + let at = 4 + head.length + for (const { bytes } of blocks) { + body.set(bytes, at) + at += bytes.length + } + return body +} + +/** `undefined` for a body that is not what `encodeIndexBlocks` writes, whole. */ +function decodeIndexBlocks(body: Uint8Array): IndexBlocks | undefined { + if (body.length < 4) return undefined + const length = new DataView(body.buffer, body.byteOffset, body.byteLength).getUint32(0) + if (4 + length > body.length) return undefined + const head = JSON.parse(new TextDecoder().decode(body.subarray(4, 4 + length))) as { + validator?: unknown + size?: unknown + blocks?: Array<{ index?: unknown; length?: unknown }> + } + if (typeof head.validator !== 'string' || typeof head.size !== 'number' || !Array.isArray(head.blocks)) return undefined + const blocks: IndexBlocks['blocks'] = [] + let at = 4 + length + for (const { index, length: bytes } of head.blocks) { + if (typeof index !== 'number' || typeof bytes !== 'number' || at + bytes > body.length) return undefined + blocks.push({ index, bytes: body.slice(at, at + bytes) }) + at += bytes + } + return at === body.length ? { validator: head.validator, size: head.size, blocks } : undefined +} + /** A write the browser refused for lack of space, in either spelling. */ export function isQuotaError(error: unknown): boolean { return ( @@ -342,6 +395,26 @@ export class ChunkStore { }) } + /** + * The blocks the archive's index was read from over HTTP, as a worker kept them (#136): + * `undefined` when none were kept, or what was kept does not read back whole. + */ + async getIndexBlocks(): Promise { + const cache = await this.cache() + const response = await cache.match(indexBlocksKey(this.pkgId)) + if (!response) return undefined + try { + return decodeIndexBlocks(new Uint8Array(await response.arrayBuffer())) + } catch { + return undefined + } + } + + async setIndexBlocks(kept: IndexBlocks): Promise { + const body = encodeIndexBlocks(kept) + await this.write(indexBlocksKey(this.pkgId), () => new Response(body, { headers: { 'content-type': 'application/octet-stream' } })) + } + /* ---------------------------------------------------------------- writing and eviction */ /** diff --git a/packages/player/src/shared/source.ts b/packages/player/src/shared/source.ts index 060f56d2..51224143 100644 --- a/packages/player/src/shared/source.ts +++ b/packages/player/src/shared/source.ts @@ -203,16 +203,50 @@ export function forgetBlocks(): void { blocks?.clear() } +type RangeHttpDescriptor = Extract + +/** What names an archive's bytes in the block cache: the package id's ingredients, and the size. */ +function blockKeyOf(descriptor: RangeHttpDescriptor, size: number): string { + return `${descriptor.url}\u0000${descriptor.validator ?? ''}\u0000${size}` +} + +/** A block of an archive, by its place: block `index` starts at `index * BLOCK_SIZE`. */ +export interface KeptBlock { + index: number + bytes: Uint8Array +} + +/** + * The blocks `handle`'s small reads went through, held in this worker's cache, for keeping with + * the package (#136): what the index and a load's reads before the boot needed. Only a handle over + * HTTP has any. + */ +export function blocksReadBy(handle: SourceHandle): KeptBlock[] { + return handle instanceof RangeHttpHandle ? handle.blocksRead() : [] +} + +/** Puts back blocks an earlier worker read of the archive at `descriptor`, `size` bytes long. */ +export function seedBlocks(descriptor: RangeHttpDescriptor, size: number, kept: Iterable): void { + blockCache().seed(blockKeyOf(descriptor, size), size, kept) +} + class RangeHttpHandle implements SourceHandle { received = 0 /** What names these bytes in the block cache: the package id's ingredients, and the size. */ private readonly blockKey: string + /** The blocks this handle's small reads went through, fetched or found held. */ + private readonly used = new Set() constructor( - readonly descriptor: Extract, + readonly descriptor: RangeHttpDescriptor, readonly size: number ) { - this.blockKey = `${descriptor.url}\u0000${descriptor.validator ?? ''}\u0000${size}` + this.blockKey = blockKeyOf(descriptor, size) + } + + /** The blocks this handle's small reads went through that the cache still holds, in archive order. */ + blocksRead(): KeptBlock[] { + return blockCache().blocksOf(this.blockKey, [...this.used].sort((a, b) => a - b)) } /** Counts a body's bytes as they pass, without holding any of them. */ @@ -268,6 +302,8 @@ class RangeHttpHandle implements SourceHandle { */ async read(range: ByteRange): Promise { if (range.end - range.start + 1 > BLOCK_CACHE_MAX_READ) return this.fetchBytes(range) + const last = Math.floor(Math.min(range.end, this.size - 1) / BLOCK_SIZE) + for (let index = Math.floor(range.start / BLOCK_SIZE); index <= last; index += 1) this.used.add(index) return blockCache().read(this.blockKey, this.size, range, (missing) => this.fetchBytes(missing)) } diff --git a/packages/player/src/sw/mount.ts b/packages/player/src/sw/mount.ts index 6344d457..13acb904 100644 --- a/packages/player/src/sw/mount.ts +++ b/packages/player/src/sw/mount.ts @@ -39,7 +39,7 @@ import { type WorkerReply } from '../shared/protocol.js' import { contentRange, parseRange, type ByteRange } from '../shared/range.js' -import { openSource } from '../shared/source.js' +import { blocksReadBy, openSource, seedBlocks, type SourceHandle } from '../shared/source.js' import * as db from '../shared/idb.js' import { dependenciesIn } from '../shared/content-libraries.js' import { libraryFolderNames } from '../shared/library-folders.js' @@ -176,6 +176,16 @@ class VirtualServer { * archive without another attempt to cache them. */ private storageFullUntil = 0 + /** + * The blocks kept with each package, or put back from it, in this worker, by their indices: an + * index that read none beyond them writes nothing. + */ + private readonly keptIndexBlocks = new Map>() + /** + * Readers whose frame has booted. What one reads after that is the runtime's, not the index's: + * a second load in a worker still running reuses the reader, and keeps nothing more. + */ + private readonly booted = new WeakSet() constructor( private readonly worker: ServiceWorkerGlobalScope, @@ -183,9 +193,54 @@ class VirtualServer { ) { installEvictionPolicy((pkgId) => { for (const key of this.readers.keys()) if (key.startsWith(pkgId)) this.readers.delete(key) + this.keptIndexBlocks.delete(pkgId) }) } + /** + * Keeps the blocks this reader's index was read from with the package (#136). The block cache + * dies with the worker, and Chromium stops an idle one after about thirty seconds, so a reload + * read the tail, `h5p.json` and `content.json` again before the first answer. Only for an + * archive over HTTP whose host gave a validator: without one, an archive republished at the same + * URL keeps its package id, and old blocks would send zip.js to the old offsets. The blocks are + * whatever this reader's small reads went through before its frame booted, the index and what + * the load read for the plan and the check; the player names no file. Written again only when a + * load read a block beyond those kept. A copy that cannot be written is no copy. + */ + private async keepIndexBlocks(reader: PackageReader): Promise { + const { handle } = reader + const { descriptor } = handle + if (descriptor.type !== 'range-http' || !descriptor.validator || this.booted.has(reader)) return + const blocks = blocksReadBy(handle) + const kept = this.keptIndexBlocks.get(reader.pkgId) + if (blocks.length === 0 || (kept && blocks.every(({ index }) => kept.has(index)))) return + try { + await new ChunkStore(reader.pkgId).setIndexBlocks({ validator: descriptor.validator, size: handle.size, blocks }) + this.keptIndexBlocks.set(reader.pkgId, new Set(blocks.map(({ index }) => index))) + } catch { + // Read from the network next time, as before. + } + } + + /** + * Puts back the blocks an earlier worker kept for this archive, when the validator and the size + * they were read under still match the ones this load was given: zip.js then reads the index + * from memory, as a reader built again after `attach-libraries` does. A copy that is missing or + * cannot be read means a read from the network, as before; it is never the authority. + */ + private async seedIndexBlocks(pkgId: string, handle: SourceHandle): Promise { + const { descriptor } = handle + if (descriptor.type !== 'range-http' || !descriptor.validator) return + try { + const kept = await new ChunkStore(pkgId).getIndexBlocks() + if (!kept || kept.validator !== descriptor.validator || kept.size !== handle.size) return + seedBlocks(descriptor, handle.size, kept.blocks) + this.keptIndexBlocks.set(pkgId, new Set(kept.blocks.map(({ index }) => index))) + } catch { + // The network answers. + } + } + /* ---------------------------------------------------------------- control channel */ async handleMessage(event: ExtendableMessageEvent): Promise { @@ -248,6 +303,8 @@ class VirtualServer { mainLibrary: typeof reader.manifest.mainLibrary === 'string' ? reader.manifest.mainLibrary : undefined, ...(downgraded.length > 0 ? { downgraded } : {}) }) + // After the reply, so the load does not wait on it. + if (!reader.partial) await this.keepIndexBlocks(reader) return } @@ -415,6 +472,7 @@ class VirtualServer { // `let`: a wait on a partial reader hands back the reader to go on with, which is a new one // once the archive is whole, as in `serveEntry`. let reader = await this.refresh(load, await this.reader(load)) + this.booted.add(reader) const manifest = reader.hasFallbacks ? await reader.mergedManifest() : reader.manifest // A download that stops moving is a stall, answered with a retry as an entry's is, and not a // library the package lacks: a lookup that found one ends the walk's waiting. @@ -841,6 +899,7 @@ class VirtualServer { } const handle = await openSource(pkgId, record.source, file) + await this.seedIndexBlocks(pkgId, handle) // Not held to its libraries here: an `index` does that, after any plan from the pack, and a // bundle or a part registered to supply libraries to something else never is. diff --git a/packages/player/tests/browser-emulated/worker-stopped.test.ts b/packages/player/tests/browser-emulated/worker-stopped.test.ts new file mode 100644 index 00000000..7fbd970a --- /dev/null +++ b/packages/player/tests/browser-emulated/worker-stopped.test.ts @@ -0,0 +1,46 @@ +/// +import { cdp } from 'vitest/browser' +import { afterEach, describe, expect, it } from 'vitest' +import '../../src/h5p-offline-player' +import { clearPackageCaches, play } from '../browser/utils' + +/** + * A load after the browser stopped the Service Worker (#136). Chromium stops an idle worker after + * about thirty seconds, and the block cache under small reads lives in its memory, so the next load + * read the archive's index again: the tail and the head, one after the other, before the worker + * answered anything. The blocks the index was read from are kept with the package when the host + * gives a validator. CDP stops the worker here, as DevTools' Stop does; `/counted/` writes down + * every request the host takes, and gives an `ETag` with `&etag=1`. + */ + +const rangesOf = async (log: string): Promise => (await fetch(`/counted/__log?log=${log}`)).json() + +/** Plays `src`, removes the player, stops the worker, and plays it again: the second load's requests. */ +async function reloadAfterStop(src: string, log: string): Promise { + const first = await play(src) + const before = (await rangesOf(log)).length + first.remove() + await cdp().send('ServiceWorker.enable') + await cdp().send('ServiceWorker.stopAllWorkers') + await play(src) + return (await rangesOf(log)).slice(before) +} + +describe('a load after the browser stopped the worker', () => { + afterEach(() => clearPackageCaches()) + + it('asks the host for the probe alone when it gave a validator', async () => { + const log = crypto.randomUUID() + // Its index is read from the blocks the first load kept, and its files from the warm's copies. + expect(await reloadAfterStop(`/counted/basic.h5p?log=${log}&etag=1`, log)).toEqual(['bytes=0-0']) + }) + + it('reads the index again when the host gave none', async () => { + const log = crypto.randomUUID() + // An archive republished at the same URL keeps its package id without a validator, and old + // blocks would send zip.js to the old offsets: the probe, then the index. + const requests = await reloadAfterStop(`/counted/basic.h5p?log=${log}`, log) + expect(requests[0]).toBe('bytes=0-0') + expect(requests.length).toBeGreaterThan(1) + }) +}) diff --git a/packages/player/tests/unit/block-cache.test.ts b/packages/player/tests/unit/block-cache.test.ts index 48f7a6fb..20303290 100644 --- a/packages/player/tests/unit/block-cache.test.ts +++ b/packages/player/tests/unit/block-cache.test.ts @@ -152,3 +152,47 @@ describe('BlockCache', () => { expect(asked).toHaveLength(2) }) }) + +describe('BlockCache, kept beyond the worker (#136)', () => { + it('gives the blocks of a key it holds among those asked for, and holds them again in a new cache', async () => { + const first = new BlockCache(BLOCK, 1024) + const { fetchRange } = source() + await first.read('a', SIZE, { start: 90, end: 99 }, fetchRange) + await first.read('a', SIZE, { start: 0, end: 3 }, fetchRange) + // Block 6 holds 96 to 99 only: the archive ends there. + const kept = first.blocksOf('a', [0, 3, 5, 6]) + expect(kept.map(({ index, bytes }) => [index, bytes.length])).toEqual([ + [0, 16], + [5, 16], + [6, 4] + ]) + + const next = new BlockCache(BLOCK, 1024) + const later = source() + next.seed('a', SIZE, kept) + expect(await next.read('a', SIZE, { start: 92, end: 97 }, later.fetchRange)).toEqual(archive.slice(92, 98)) + expect(await next.read('a', SIZE, { start: 2, end: 9 }, later.fetchRange)).toEqual(archive.slice(2, 10)) + expect(later.asked).toEqual([]) + }) + + it('leaves out a block of the wrong length for its place, and one it already holds', async () => { + const cache = new BlockCache(BLOCK, 1024) + const { asked, fetchRange } = source() + await cache.read('a', SIZE, { start: 0, end: 3 }, fetchRange) + + cache.seed('a', SIZE, [ + // Held already: what was fetched stands. + { index: 0, bytes: new Uint8Array(BLOCK).fill(255) }, + // Short for a whole block, and past the archive's end. + { index: 1, bytes: archive.slice(16, 20) }, + { index: 9, bytes: new Uint8Array(BLOCK) } + ]) + expect(await cache.read('a', SIZE, { start: 0, end: 3 }, fetchRange)).toEqual(archive.slice(0, 4)) + expect(await cache.read('a', SIZE, { start: 16, end: 19 }, fetchRange)).toEqual(archive.slice(16, 20)) + // Block 1 was fetched after all. + expect(asked).toEqual([ + { start: 0, end: 15 }, + { start: 16, end: 31 } + ]) + }) +}) diff --git a/packages/player/tests/unit/chunk-store-write.test.ts b/packages/player/tests/unit/chunk-store-write.test.ts index 75176370..777f00ab 100644 --- a/packages/player/tests/unit/chunk-store-write.test.ts +++ b/packages/player/tests/unit/chunk-store-write.test.ts @@ -237,6 +237,31 @@ describe('ChunkStore writes', () => { expect(heard).toEqual([{ size: null, available: 48, complete: false }]) }) + it('keeps the blocks an index was read from, with what they were read under, and reads them back whole (#136)', async () => { + const { caches } = fakeCaches(false) + vi.stubGlobal('caches', caches) + const store = new ChunkStore('pkg') + expect(await store.getIndexBlocks()).toBeUndefined() + + const kept = { + validator: '"abc123"', + size: 200_000, + blocks: [ + { index: 0, bytes: Uint8Array.from({ length: 65_536 }, (_, i) => i % 251) }, + { index: 3, bytes: Uint8Array.from({ length: 3_392 }, (_, i) => (i * 7) % 256) } + ] + } + await store.setIndexBlocks(kept) + expect(await store.getIndexBlocks()).toEqual(kept) + + // Cut short, the copy reads as none, and the index is read from the network. + const cache = await caches.open(cacheNameFor('pkg')) + const [key] = (await cache.keys()).map((request) => request.url).filter((url) => url.endsWith('/index-blocks')) + const whole = new Uint8Array(await (await cache.match(key))!.arrayBuffer()) + await cache.put(key, new Response(whole.slice(0, whole.length - 1))) + expect(await store.getIndexBlocks()).toBeUndefined() + }) + it('gives up with a QuotaError when nothing is left to evict', async () => { vi.stubGlobal('caches', fakeCaches(true).caches) setEvictionPolicy(async () => null) diff --git a/packages/player/tests/unit/range-http-handle.test.ts b/packages/player/tests/unit/range-http-handle.test.ts index 70774c3a..8aeeca56 100644 --- a/packages/player/tests/unit/range-http-handle.test.ts +++ b/packages/player/tests/unit/range-http-handle.test.ts @@ -1,6 +1,6 @@ import { afterEach, describe, expect, it, vi } from 'vitest' import { BLOCK_CACHE_MAX_READ, BLOCK_SIZE } from '../../src/shared/constants' -import { forgetBlocks, openSource } from '../../src/shared/source' +import { blocksReadBy, forgetBlocks, openSource, seedBlocks } from '../../src/shared/source' import { PackageReader } from '../../src/sw/package-reader' import { collect, manifest, zipOf } from './helpers/archives' @@ -167,4 +167,30 @@ describe('an index read over HTTP', () => { await collect(again.inflate(again.get('content/content.json')!.entry)) expect(asked).toHaveLength(2) }) + + it('costs nothing in a worker started again that put back the blocks the first one read (#136)', async () => { + const bytes = await quiz() + const asked = rangeHost(bytes) + const archiveDescriptor = { type: 'range-http', url: 'https://host.example/quiz.h5p', size: bytes.length, validator: '"v1"' } as const + + const first = await PackageReader.open('pkg', await openSource('pkg', archiveDescriptor)) + await collect(first.inflate(first.get('content/content.json')!.entry)) + const kept = blocksReadBy(first.handle) + expect(asked).toHaveLength(2) + + // A worker started again, its cache empty: the index costs the tail and the head again. + forgetBlocks() + const cold = await PackageReader.open('pkg', await openSource('pkg', archiveDescriptor)) + await collect(cold.inflate(cold.get('content/content.json')!.entry)) + expect(asked).toHaveLength(4) + + // The same, with the first worker's blocks put back: nothing. + forgetBlocks() + seedBlocks(archiveDescriptor, bytes.length, kept) + const warm = await PackageReader.open('pkg', await openSource('pkg', archiveDescriptor)) + await collect(warm.inflate(warm.get('content/content.json')!.entry)) + expect(asked).toHaveLength(4) + // What was kept is what the index and the content read: the head's block and the tail's. + expect(kept.map(({ index }) => index)).toEqual(blocksReadBy(warm.handle).map(({ index }) => index)) + }) }) diff --git a/packages/player/vite.plugins.ts b/packages/player/vite.plugins.ts index 7649e363..4aa650a9 100644 --- a/packages/player/vite.plugins.ts +++ b/packages/player/vite.plugins.ts @@ -1,3 +1,4 @@ +import { createHash } from 'node:crypto' import { readFileSync } from 'node:fs' import { readFile, stat } from 'node:fs/promises' import type { ServerResponse } from 'node:http' @@ -302,6 +303,7 @@ function answerRange(rangeHeader: string | undefined, res: ServerResponse, body: * costs: `/counted/?log=` serves the archive, and `/counted/__log?log=` answers * the `Range` header of every request it took for that id, in the order they came. The id keeps * test cases apart, and makes each load a package of its own, with nothing cached from another. + * `&etag=1` adds an `ETag`, the validator a host gives (#136); without it the host gives none. */ function countingHostHandler(dirs: readonly string[]): Connect.NextHandleFunction { const prefix = '/counted/' @@ -326,6 +328,7 @@ function countingHostHandler(dirs: readonly string[]): Connect.NextHandleFunctio res.setHeader('content-type', 'application/octet-stream') res.setHeader('cache-control', 'no-store') res.setHeader('accept-ranges', 'bytes') + if (url.searchParams.get('etag')) res.setHeader('etag', `"${createHash('sha1').update(body).digest('hex').slice(0, 16)}"`) answerRange(req.headers.range, res, body) }, () => noSuchFixture(res)