diff --git a/AGENTS.md b/AGENTS.md index 46ca79d2..09d1e9b6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -149,7 +149,7 @@ Read it before changing the Service Worker's file server (`src/sw/mount.ts`, `pa - `SourceReader` overrides `createReadable`. - A large span over HTTP is then pulled by several connections at once. -- Small reads over HTTP go through a block cache, and a rebuilt reader finds it. +- Small reads over HTTP go through a block cache, and a rebuilt reader finds it; with a validator, so does a worker started again. - Waiting on the watermark is bounded by a stall, not by a wall clock. - A ranged read survives the link, and a body the server gives up on ends short. - A body that streams holds its fetch event open. diff --git a/design/serving.md b/design/serving.md index 6d657f2f..1c257b87 100644 --- a/design/serving.md +++ b/design/serving.md @@ -36,7 +36,7 @@ How a package's bytes reach the H5P runtime: the Service Worker's virtual file s keeps a request's bytes as they arrive and the consumer follows it live when its turn comes, and the window fills only after the first byte, which costs a per-connection-capped host one round trip and nothing else. Same 40 MB at 4 Mbit/s now: first byte in under two seconds. -- **Small reads over HTTP go through a block cache, and a rebuilt reader finds it.** zip.js reads +- **Small reads over HTTP go through a block cache, and a rebuilt reader finds it; with a validator, so does a worker started again.** zip.js reads an index in pieces, one after another: 65,557 bytes from the end to find the end-of-central- directory record, then the directory — which in a package of a few hundred entries lies inside those same bytes — then each entry's 30-byte local header, its extra field when it has one, and @@ -57,7 +57,8 @@ How a package's bytes reach the H5P runtime: the Service Worker's virtual file s them in pieces (`blocksFor`). The cache is one per worker, made on first use so the element's bundle does not carry it, keyed by the archive's URL, validator and size — the package id's ingredients — so the new handle a rebuilt reader opens finds the blocks - the first one fetched; it dies with the worker, and it is never authority. The quiz now costs + the first one fetched; it dies with the worker, but for the blocks an index was read from, and + it is never authority. The quiz now costs four: the probe, the tail, the head and the warm. Two things left out on purpose, by the owner's decision the same day: a whole-file download below a size, which with the cache saves one request and needs a threshold; and rules per content type for what to fetch first, which would @@ -65,6 +66,25 @@ How a package's bytes reach the H5P runtime: the Service Worker's virtual file s `tests/unit/range-http-handle.test.ts` a real archive's index over a host that honours `Range`, and `tests/browser/range-requests.test.ts` counts what a load costs at `/counted/`, a host that writes down every request it takes. + + The blocks an index was read from outlive the worker (2026-11-07, #136). Chromium stops an idle + worker after about thirty seconds, so the usual revisit found the cache empty and read the tail + and the head again, one after the other, before the worker answered anything. Measured on the + demo's quiz from a host paced to 1 MB/s with 100 ms on every request, the worker stopped + through CDP between the loads: a reload took four requests and was ready at 0.64 s. Now, at the + end of each `index` of a whole reader over HTTP, `keepIndexBlocks` writes the blocks that + reader's small reads went through (`blocksReadBy`: the index, `content.json`, whatever the load + read before the boot; the player names no file) into the package's cache, with the validator + and the size they were read under. `openReader` puts them back (`seedBlocks`) before zip.js + opens the archive, when both still match, and the same reload is the probe alone, ready at + 0.21 s. Only with a validator: without one, an archive republished at the same URL keeps its + package id, and old blocks would send zip.js to the old offsets, so such a host reads the index + again. A copy that is missing, cut short or of the wrong length is read around, never trusted + (`BlockCache.seed` checks each block's length for its place), and it goes with the package's + cache when that is evicted. The bytes are kept rather than a parsed index, so zip.js stays the + only parser, and the price is one Cache API read in place of a round trip. + `tests/browser-emulated/worker-stopped.test.ts` stops the worker and counts the reload's + requests at `/counted/` with an `ETag` and without. - **Waiting on the watermark is bounded by a stall, not by a wall clock.** Inflating a few hundred megabytes over a network legitimately takes minutes, and a request for the *tail* of such an entry cannot be answered until it finishes: an mp4 whose `moov` index sits at the end — common, diff --git a/docs/setup.md b/docs/setup.md index f7a5a80b..569c3d84 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -14,7 +14,10 @@ caching and upgrades. - The page on `https://`, or `localhost` while developing: the player needs a Service Worker. - CORS headers on the package host, so the page can read the `.h5p`. `Range` support is optional: - without it the archive is downloaded once and played from the cache. + without it the archive is downloaded once and played from the cache. On another origin, add + `Access-Control-Expose-Headers: ETag` (or `Last-Modified`): the player can then tell an archive + republished at the same URL from the old one, and a reload after the browser stopped its + worker reads nothing of the archive but one byte. - A browser with Service Workers, the Cache API, IndexedDB, Web Locks and `DecompressionStream`; [Compatibility](compatibility.md) says which are tested. diff --git a/packages/player/src/shared/block-cache.ts b/packages/player/src/shared/block-cache.ts index 5448f725..08ce742d 100644 --- a/packages/player/src/shared/block-cache.ts +++ b/packages/player/src/shared/block-cache.ts @@ -21,7 +21,10 @@ import type { ByteRange } from './range.js' * bytes out from under it. * * Nothing here is authority: the key names the bytes — the archive's URL, validator and size, the - * ingredients of its package id — and the cache dies with the worker that holds it. + * ingredients of its package id — and the cache dies with the worker that holds it. The blocks + * an index was read from are the exception: kept with the package when the archive has a + * validator (`blocksOf`), and put back by the next worker (`seed`), so a worker the browser + * stopped does not read the index again (#136). */ export class BlockCache { /** Held blocks by `blockId`, oldest use first: a use moves a block to the end. */ @@ -94,6 +97,35 @@ export class BlockCache { return out } + /** + * The blocks of `key` held among `indices`, in that order, for keeping beyond this worker + * (#136): a worker the browser stops takes its cache with it, and the next one read the index + * again. One not held is left out. + */ + blocksOf(key: string, indices: Iterable): Array<{ index: number; bytes: Uint8Array }> { + const out: Array<{ index: number; bytes: Uint8Array }> = [] + for (const index of indices) { + const bytes = this.blocks.get(blockId(key, index)) + if (bytes) out.push({ index, bytes }) + } + return out + } + + /** + * Holds blocks of `key`, in an archive of `size` bytes, that an earlier worker read: from then + * on a read finds them as if fetched. A block of the wrong length for its place is left out, + * and so is one already held or on its way, which is as good or newer. + */ + seed(key: string, size: number, blocks: Iterable<{ index: number; bytes: Uint8Array }>): void { + for (const { index, bytes } of blocks) { + const id = blockId(key, index) + const expected = Math.min(this.blockSize, size - index * this.blockSize) + if (!Number.isInteger(index) || index < 0 || expected <= 0 || bytes.length !== expected) continue + if (this.blocks.has(id) || this.pending.has(id)) continue + this.keep(id, bytes) + } + } + /** Forgets every block held. Blocks on their way still arrive, and are kept. */ clear(): void { this.blocks.clear() diff --git a/packages/player/src/shared/chunk-store.ts b/packages/player/src/shared/chunk-store.ts index b214c128..7c642e21 100644 --- a/packages/player/src/shared/chunk-store.ts +++ b/packages/player/src/shared/chunk-store.ts @@ -80,6 +80,59 @@ function forwardIndexKey(pkgId: string): string { return `${CHUNK_KEY_ORIGIN}${pkgId}/chunk/${encodeEntry(ARCHIVE_ENTRY)}${FORWARD_INDEX_MARKER}` } +/** Beside the package's entries, not among them: the blocks are pieces of the archive, not of an entry. */ +function indexBlocksKey(pkgId: string): string { + return `${CHUNK_KEY_ORIGIN}${pkgId}/index-blocks` +} + +/** + * The blocks an archive's index was read from over HTTP, kept with the package (#136), and what + * they were read under: the host's validator and the archive's size. A worker started again puts + * them back in its block cache only when both still match. + */ +export interface IndexBlocks { + validator: string + size: number + blocks: Array<{ index: number; bytes: Uint8Array }> +} + +/** One body: four bytes giving the length of a JSON head, the head, then each block's bytes in turn. */ +function encodeIndexBlocks({ validator, size, blocks }: IndexBlocks): Uint8Array { + const head = new TextEncoder().encode( + JSON.stringify({ validator, size, blocks: blocks.map(({ index, bytes }) => ({ index, length: bytes.length })) }) + ) + const body = new Uint8Array(4 + head.length + blocks.reduce((sum, { bytes }) => sum + bytes.length, 0)) + new DataView(body.buffer).setUint32(0, head.length) + body.set(head, 4) + let at = 4 + head.length + for (const { bytes } of blocks) { + body.set(bytes, at) + at += bytes.length + } + return body +} + +/** `undefined` for a body that is not what `encodeIndexBlocks` writes, whole. */ +function decodeIndexBlocks(body: Uint8Array): IndexBlocks | undefined { + if (body.length < 4) return undefined + const length = new DataView(body.buffer, body.byteOffset, body.byteLength).getUint32(0) + if (4 + length > body.length) return undefined + const head = JSON.parse(new TextDecoder().decode(body.subarray(4, 4 + length))) as { + validator?: unknown + size?: unknown + blocks?: Array<{ index?: unknown; length?: unknown }> + } + if (typeof head.validator !== 'string' || typeof head.size !== 'number' || !Array.isArray(head.blocks)) return undefined + const blocks: IndexBlocks['blocks'] = [] + let at = 4 + length + for (const { index, length: bytes } of head.blocks) { + if (typeof index !== 'number' || typeof bytes !== 'number' || at + bytes > body.length) return undefined + blocks.push({ index, bytes: body.slice(at, at + bytes) }) + at += bytes + } + return at === body.length ? { validator: head.validator, size: head.size, blocks } : undefined +} + /** A write the browser refused for lack of space, in either spelling. */ export function isQuotaError(error: unknown): boolean { return ( @@ -342,6 +395,26 @@ export class ChunkStore { }) } + /** + * The blocks the archive's index was read from over HTTP, as a worker kept them (#136): + * `undefined` when none were kept, or what was kept does not read back whole. + */ + async getIndexBlocks(): Promise { + const cache = await this.cache() + const response = await cache.match(indexBlocksKey(this.pkgId)) + if (!response) return undefined + try { + return decodeIndexBlocks(new Uint8Array(await response.arrayBuffer())) + } catch { + return undefined + } + } + + async setIndexBlocks(kept: IndexBlocks): Promise { + const body = encodeIndexBlocks(kept) + await this.write(indexBlocksKey(this.pkgId), () => new Response(body, { headers: { 'content-type': 'application/octet-stream' } })) + } + /* ---------------------------------------------------------------- writing and eviction */ /** diff --git a/packages/player/src/shared/source.ts b/packages/player/src/shared/source.ts index 060f56d2..51224143 100644 --- a/packages/player/src/shared/source.ts +++ b/packages/player/src/shared/source.ts @@ -203,16 +203,50 @@ export function forgetBlocks(): void { blocks?.clear() } +type RangeHttpDescriptor = Extract + +/** What names an archive's bytes in the block cache: the package id's ingredients, and the size. */ +function blockKeyOf(descriptor: RangeHttpDescriptor, size: number): string { + return `${descriptor.url}\u0000${descriptor.validator ?? ''}\u0000${size}` +} + +/** A block of an archive, by its place: block `index` starts at `index * BLOCK_SIZE`. */ +export interface KeptBlock { + index: number + bytes: Uint8Array +} + +/** + * The blocks `handle`'s small reads went through, held in this worker's cache, for keeping with + * the package (#136): what the index and a load's reads before the boot needed. Only a handle over + * HTTP has any. + */ +export function blocksReadBy(handle: SourceHandle): KeptBlock[] { + return handle instanceof RangeHttpHandle ? handle.blocksRead() : [] +} + +/** Puts back blocks an earlier worker read of the archive at `descriptor`, `size` bytes long. */ +export function seedBlocks(descriptor: RangeHttpDescriptor, size: number, kept: Iterable): void { + blockCache().seed(blockKeyOf(descriptor, size), size, kept) +} + class RangeHttpHandle implements SourceHandle { received = 0 /** What names these bytes in the block cache: the package id's ingredients, and the size. */ private readonly blockKey: string + /** The blocks this handle's small reads went through, fetched or found held. */ + private readonly used = new Set() constructor( - readonly descriptor: Extract, + readonly descriptor: RangeHttpDescriptor, readonly size: number ) { - this.blockKey = `${descriptor.url}\u0000${descriptor.validator ?? ''}\u0000${size}` + this.blockKey = blockKeyOf(descriptor, size) + } + + /** The blocks this handle's small reads went through that the cache still holds, in archive order. */ + blocksRead(): KeptBlock[] { + return blockCache().blocksOf(this.blockKey, [...this.used].sort((a, b) => a - b)) } /** Counts a body's bytes as they pass, without holding any of them. */ @@ -268,6 +302,8 @@ class RangeHttpHandle implements SourceHandle { */ async read(range: ByteRange): Promise { if (range.end - range.start + 1 > BLOCK_CACHE_MAX_READ) return this.fetchBytes(range) + const last = Math.floor(Math.min(range.end, this.size - 1) / BLOCK_SIZE) + for (let index = Math.floor(range.start / BLOCK_SIZE); index <= last; index += 1) this.used.add(index) return blockCache().read(this.blockKey, this.size, range, (missing) => this.fetchBytes(missing)) } diff --git a/packages/player/src/sw/mount.ts b/packages/player/src/sw/mount.ts index 6344d457..13acb904 100644 --- a/packages/player/src/sw/mount.ts +++ b/packages/player/src/sw/mount.ts @@ -39,7 +39,7 @@ import { type WorkerReply } from '../shared/protocol.js' import { contentRange, parseRange, type ByteRange } from '../shared/range.js' -import { openSource } from '../shared/source.js' +import { blocksReadBy, openSource, seedBlocks, type SourceHandle } from '../shared/source.js' import * as db from '../shared/idb.js' import { dependenciesIn } from '../shared/content-libraries.js' import { libraryFolderNames } from '../shared/library-folders.js' @@ -176,6 +176,16 @@ class VirtualServer { * archive without another attempt to cache them. */ private storageFullUntil = 0 + /** + * The blocks kept with each package, or put back from it, in this worker, by their indices: an + * index that read none beyond them writes nothing. + */ + private readonly keptIndexBlocks = new Map>() + /** + * Readers whose frame has booted. What one reads after that is the runtime's, not the index's: + * a second load in a worker still running reuses the reader, and keeps nothing more. + */ + private readonly booted = new WeakSet() constructor( private readonly worker: ServiceWorkerGlobalScope, @@ -183,9 +193,54 @@ class VirtualServer { ) { installEvictionPolicy((pkgId) => { for (const key of this.readers.keys()) if (key.startsWith(pkgId)) this.readers.delete(key) + this.keptIndexBlocks.delete(pkgId) }) } + /** + * Keeps the blocks this reader's index was read from with the package (#136). The block cache + * dies with the worker, and Chromium stops an idle one after about thirty seconds, so a reload + * read the tail, `h5p.json` and `content.json` again before the first answer. Only for an + * archive over HTTP whose host gave a validator: without one, an archive republished at the same + * URL keeps its package id, and old blocks would send zip.js to the old offsets. The blocks are + * whatever this reader's small reads went through before its frame booted, the index and what + * the load read for the plan and the check; the player names no file. Written again only when a + * load read a block beyond those kept. A copy that cannot be written is no copy. + */ + private async keepIndexBlocks(reader: PackageReader): Promise { + const { handle } = reader + const { descriptor } = handle + if (descriptor.type !== 'range-http' || !descriptor.validator || this.booted.has(reader)) return + const blocks = blocksReadBy(handle) + const kept = this.keptIndexBlocks.get(reader.pkgId) + if (blocks.length === 0 || (kept && blocks.every(({ index }) => kept.has(index)))) return + try { + await new ChunkStore(reader.pkgId).setIndexBlocks({ validator: descriptor.validator, size: handle.size, blocks }) + this.keptIndexBlocks.set(reader.pkgId, new Set(blocks.map(({ index }) => index))) + } catch { + // Read from the network next time, as before. + } + } + + /** + * Puts back the blocks an earlier worker kept for this archive, when the validator and the size + * they were read under still match the ones this load was given: zip.js then reads the index + * from memory, as a reader built again after `attach-libraries` does. A copy that is missing or + * cannot be read means a read from the network, as before; it is never the authority. + */ + private async seedIndexBlocks(pkgId: string, handle: SourceHandle): Promise { + const { descriptor } = handle + if (descriptor.type !== 'range-http' || !descriptor.validator) return + try { + const kept = await new ChunkStore(pkgId).getIndexBlocks() + if (!kept || kept.validator !== descriptor.validator || kept.size !== handle.size) return + seedBlocks(descriptor, handle.size, kept.blocks) + this.keptIndexBlocks.set(pkgId, new Set(kept.blocks.map(({ index }) => index))) + } catch { + // The network answers. + } + } + /* ---------------------------------------------------------------- control channel */ async handleMessage(event: ExtendableMessageEvent): Promise { @@ -248,6 +303,8 @@ class VirtualServer { mainLibrary: typeof reader.manifest.mainLibrary === 'string' ? reader.manifest.mainLibrary : undefined, ...(downgraded.length > 0 ? { downgraded } : {}) }) + // After the reply, so the load does not wait on it. + if (!reader.partial) await this.keepIndexBlocks(reader) return } @@ -415,6 +472,7 @@ class VirtualServer { // `let`: a wait on a partial reader hands back the reader to go on with, which is a new one // once the archive is whole, as in `serveEntry`. let reader = await this.refresh(load, await this.reader(load)) + this.booted.add(reader) const manifest = reader.hasFallbacks ? await reader.mergedManifest() : reader.manifest // A download that stops moving is a stall, answered with a retry as an entry's is, and not a // library the package lacks: a lookup that found one ends the walk's waiting. @@ -841,6 +899,7 @@ class VirtualServer { } const handle = await openSource(pkgId, record.source, file) + await this.seedIndexBlocks(pkgId, handle) // Not held to its libraries here: an `index` does that, after any plan from the pack, and a // bundle or a part registered to supply libraries to something else never is. diff --git a/packages/player/tests/browser-emulated/worker-stopped.test.ts b/packages/player/tests/browser-emulated/worker-stopped.test.ts new file mode 100644 index 00000000..7fbd970a --- /dev/null +++ b/packages/player/tests/browser-emulated/worker-stopped.test.ts @@ -0,0 +1,46 @@ +/// +import { cdp } from 'vitest/browser' +import { afterEach, describe, expect, it } from 'vitest' +import '../../src/h5p-offline-player' +import { clearPackageCaches, play } from '../browser/utils' + +/** + * A load after the browser stopped the Service Worker (#136). Chromium stops an idle worker after + * about thirty seconds, and the block cache under small reads lives in its memory, so the next load + * read the archive's index again: the tail and the head, one after the other, before the worker + * answered anything. The blocks the index was read from are kept with the package when the host + * gives a validator. CDP stops the worker here, as DevTools' Stop does; `/counted/` writes down + * every request the host takes, and gives an `ETag` with `&etag=1`. + */ + +const rangesOf = async (log: string): Promise => (await fetch(`/counted/__log?log=${log}`)).json() + +/** Plays `src`, removes the player, stops the worker, and plays it again: the second load's requests. */ +async function reloadAfterStop(src: string, log: string): Promise { + const first = await play(src) + const before = (await rangesOf(log)).length + first.remove() + await cdp().send('ServiceWorker.enable') + await cdp().send('ServiceWorker.stopAllWorkers') + await play(src) + return (await rangesOf(log)).slice(before) +} + +describe('a load after the browser stopped the worker', () => { + afterEach(() => clearPackageCaches()) + + it('asks the host for the probe alone when it gave a validator', async () => { + const log = crypto.randomUUID() + // Its index is read from the blocks the first load kept, and its files from the warm's copies. + expect(await reloadAfterStop(`/counted/basic.h5p?log=${log}&etag=1`, log)).toEqual(['bytes=0-0']) + }) + + it('reads the index again when the host gave none', async () => { + const log = crypto.randomUUID() + // An archive republished at the same URL keeps its package id without a validator, and old + // blocks would send zip.js to the old offsets: the probe, then the index. + const requests = await reloadAfterStop(`/counted/basic.h5p?log=${log}`, log) + expect(requests[0]).toBe('bytes=0-0') + expect(requests.length).toBeGreaterThan(1) + }) +}) diff --git a/packages/player/tests/unit/block-cache.test.ts b/packages/player/tests/unit/block-cache.test.ts index 48f7a6fb..20303290 100644 --- a/packages/player/tests/unit/block-cache.test.ts +++ b/packages/player/tests/unit/block-cache.test.ts @@ -152,3 +152,47 @@ describe('BlockCache', () => { expect(asked).toHaveLength(2) }) }) + +describe('BlockCache, kept beyond the worker (#136)', () => { + it('gives the blocks of a key it holds among those asked for, and holds them again in a new cache', async () => { + const first = new BlockCache(BLOCK, 1024) + const { fetchRange } = source() + await first.read('a', SIZE, { start: 90, end: 99 }, fetchRange) + await first.read('a', SIZE, { start: 0, end: 3 }, fetchRange) + // Block 6 holds 96 to 99 only: the archive ends there. + const kept = first.blocksOf('a', [0, 3, 5, 6]) + expect(kept.map(({ index, bytes }) => [index, bytes.length])).toEqual([ + [0, 16], + [5, 16], + [6, 4] + ]) + + const next = new BlockCache(BLOCK, 1024) + const later = source() + next.seed('a', SIZE, kept) + expect(await next.read('a', SIZE, { start: 92, end: 97 }, later.fetchRange)).toEqual(archive.slice(92, 98)) + expect(await next.read('a', SIZE, { start: 2, end: 9 }, later.fetchRange)).toEqual(archive.slice(2, 10)) + expect(later.asked).toEqual([]) + }) + + it('leaves out a block of the wrong length for its place, and one it already holds', async () => { + const cache = new BlockCache(BLOCK, 1024) + const { asked, fetchRange } = source() + await cache.read('a', SIZE, { start: 0, end: 3 }, fetchRange) + + cache.seed('a', SIZE, [ + // Held already: what was fetched stands. + { index: 0, bytes: new Uint8Array(BLOCK).fill(255) }, + // Short for a whole block, and past the archive's end. + { index: 1, bytes: archive.slice(16, 20) }, + { index: 9, bytes: new Uint8Array(BLOCK) } + ]) + expect(await cache.read('a', SIZE, { start: 0, end: 3 }, fetchRange)).toEqual(archive.slice(0, 4)) + expect(await cache.read('a', SIZE, { start: 16, end: 19 }, fetchRange)).toEqual(archive.slice(16, 20)) + // Block 1 was fetched after all. + expect(asked).toEqual([ + { start: 0, end: 15 }, + { start: 16, end: 31 } + ]) + }) +}) diff --git a/packages/player/tests/unit/chunk-store-write.test.ts b/packages/player/tests/unit/chunk-store-write.test.ts index 75176370..777f00ab 100644 --- a/packages/player/tests/unit/chunk-store-write.test.ts +++ b/packages/player/tests/unit/chunk-store-write.test.ts @@ -237,6 +237,31 @@ describe('ChunkStore writes', () => { expect(heard).toEqual([{ size: null, available: 48, complete: false }]) }) + it('keeps the blocks an index was read from, with what they were read under, and reads them back whole (#136)', async () => { + const { caches } = fakeCaches(false) + vi.stubGlobal('caches', caches) + const store = new ChunkStore('pkg') + expect(await store.getIndexBlocks()).toBeUndefined() + + const kept = { + validator: '"abc123"', + size: 200_000, + blocks: [ + { index: 0, bytes: Uint8Array.from({ length: 65_536 }, (_, i) => i % 251) }, + { index: 3, bytes: Uint8Array.from({ length: 3_392 }, (_, i) => (i * 7) % 256) } + ] + } + await store.setIndexBlocks(kept) + expect(await store.getIndexBlocks()).toEqual(kept) + + // Cut short, the copy reads as none, and the index is read from the network. + const cache = await caches.open(cacheNameFor('pkg')) + const [key] = (await cache.keys()).map((request) => request.url).filter((url) => url.endsWith('/index-blocks')) + const whole = new Uint8Array(await (await cache.match(key))!.arrayBuffer()) + await cache.put(key, new Response(whole.slice(0, whole.length - 1))) + expect(await store.getIndexBlocks()).toBeUndefined() + }) + it('gives up with a QuotaError when nothing is left to evict', async () => { vi.stubGlobal('caches', fakeCaches(true).caches) setEvictionPolicy(async () => null) diff --git a/packages/player/tests/unit/range-http-handle.test.ts b/packages/player/tests/unit/range-http-handle.test.ts index 70774c3a..8aeeca56 100644 --- a/packages/player/tests/unit/range-http-handle.test.ts +++ b/packages/player/tests/unit/range-http-handle.test.ts @@ -1,6 +1,6 @@ import { afterEach, describe, expect, it, vi } from 'vitest' import { BLOCK_CACHE_MAX_READ, BLOCK_SIZE } from '../../src/shared/constants' -import { forgetBlocks, openSource } from '../../src/shared/source' +import { blocksReadBy, forgetBlocks, openSource, seedBlocks } from '../../src/shared/source' import { PackageReader } from '../../src/sw/package-reader' import { collect, manifest, zipOf } from './helpers/archives' @@ -167,4 +167,30 @@ describe('an index read over HTTP', () => { await collect(again.inflate(again.get('content/content.json')!.entry)) expect(asked).toHaveLength(2) }) + + it('costs nothing in a worker started again that put back the blocks the first one read (#136)', async () => { + const bytes = await quiz() + const asked = rangeHost(bytes) + const archiveDescriptor = { type: 'range-http', url: 'https://host.example/quiz.h5p', size: bytes.length, validator: '"v1"' } as const + + const first = await PackageReader.open('pkg', await openSource('pkg', archiveDescriptor)) + await collect(first.inflate(first.get('content/content.json')!.entry)) + const kept = blocksReadBy(first.handle) + expect(asked).toHaveLength(2) + + // A worker started again, its cache empty: the index costs the tail and the head again. + forgetBlocks() + const cold = await PackageReader.open('pkg', await openSource('pkg', archiveDescriptor)) + await collect(cold.inflate(cold.get('content/content.json')!.entry)) + expect(asked).toHaveLength(4) + + // The same, with the first worker's blocks put back: nothing. + forgetBlocks() + seedBlocks(archiveDescriptor, bytes.length, kept) + const warm = await PackageReader.open('pkg', await openSource('pkg', archiveDescriptor)) + await collect(warm.inflate(warm.get('content/content.json')!.entry)) + expect(asked).toHaveLength(4) + // What was kept is what the index and the content read: the head's block and the tail's. + expect(kept.map(({ index }) => index)).toEqual(blocksReadBy(warm.handle).map(({ index }) => index)) + }) }) diff --git a/packages/player/vite.plugins.ts b/packages/player/vite.plugins.ts index 7649e363..4aa650a9 100644 --- a/packages/player/vite.plugins.ts +++ b/packages/player/vite.plugins.ts @@ -1,3 +1,4 @@ +import { createHash } from 'node:crypto' import { readFileSync } from 'node:fs' import { readFile, stat } from 'node:fs/promises' import type { ServerResponse } from 'node:http' @@ -302,6 +303,7 @@ function answerRange(rangeHeader: string | undefined, res: ServerResponse, body: * costs: `/counted/?log=` serves the archive, and `/counted/__log?log=` answers * the `Range` header of every request it took for that id, in the order they came. The id keeps * test cases apart, and makes each load a package of its own, with nothing cached from another. + * `&etag=1` adds an `ETag`, the validator a host gives (#136); without it the host gives none. */ function countingHostHandler(dirs: readonly string[]): Connect.NextHandleFunction { const prefix = '/counted/' @@ -326,6 +328,7 @@ function countingHostHandler(dirs: readonly string[]): Connect.NextHandleFunctio res.setHeader('content-type', 'application/octet-stream') res.setHeader('cache-control', 'no-store') res.setHeader('accept-ranges', 'bytes') + if (url.searchParams.get('etag')) res.setHeader('etag', `"${createHash('sha1').update(body).digest('hex').slice(0, 16)}"`) answerRange(req.headers.range, res, body) }, () => noSuchFixture(res)