diff --git a/crates/Cargo.lock b/crates/Cargo.lock index f7dc13d..47a7f0d 100644 --- a/crates/Cargo.lock +++ b/crates/Cargo.lock @@ -4217,9 +4217,9 @@ dependencies = [ [[package]] name = "rustix" -version = "1.1.5" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +checksum = "e5e76742e5294153d4dde1207dbec5bdd81eef6fabdc0d2118babadabfd858aa" dependencies = [ "bitflags 2.13.2", "errno", diff --git a/crates/desktop/Cargo.lock b/crates/desktop/Cargo.lock index 9d86921..b48a5b1 100644 --- a/crates/desktop/Cargo.lock +++ b/crates/desktop/Cargo.lock @@ -25,7 +25,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ "cfg-if", - "cipher", + "cipher 0.4.4", "cpufeatures 0.2.17", ] @@ -37,7 +37,7 @@ checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" dependencies = [ "aead", "aes", - "cipher", + "cipher 0.4.4", "ctr", "ghash", "subtle", @@ -620,7 +620,7 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" dependencies = [ - "cipher", + "cipher 0.4.4", ] [[package]] @@ -642,7 +642,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ae3c82e4355234767756212c570e29833699ab63e6ffd161887314cc5b43847" dependencies = [ "aead", - "cipher", + "cipher 0.4.4", "ctr", "subtle", ] @@ -693,7 +693,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818" dependencies = [ "cfg-if", - "cipher", + "cipher 0.4.4", "cpufeatures 0.2.17", ] @@ -704,6 +704,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if", + "cipher 0.5.2", "cpufeatures 0.3.1", "rand_core 0.10.1", ] @@ -716,7 +717,7 @@ checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35" dependencies = [ "aead", "chacha20 0.9.1", - "cipher", + "cipher 0.4.4", "poly1305", "zeroize", ] @@ -742,10 +743,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ "crypto-common 0.1.7", - "inout", + "inout 0.1.4", "zeroize", ] +[[package]] +name = "cipher" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c" +dependencies = [ + "block-buffer 0.12.1", + "crypto-common 0.2.2", + "inout 0.2.6", +] + [[package]] name = "cmake" version = "0.1.58" @@ -1029,7 +1041,7 @@ version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" dependencies = [ - "cipher", + "cipher 0.4.4", ] [[package]] @@ -2793,6 +2805,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "inout" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a580c13f0ffc335ef42dd3f6b1a41b726051c5e1656d9a3aa6fddb3c5dfbbf9" +dependencies = [ + "hybrid-array", +] + [[package]] name = "inplace-vec-builder" version = "0.1.1" @@ -3918,6 +3939,7 @@ name = "nfx-node" version = "0.1.1" dependencies = [ "bytes", + "getrandom 0.4.3", "hex", "http-body-util", "hyper", @@ -3930,6 +3952,7 @@ dependencies = [ "nostr-database", "nostr-memory", "nostr-sdk", + "portmapper", "rustls-pki-types", "serde_json", "str0m", @@ -3957,6 +3980,7 @@ name = "nfx-proto" version = "0.1.0" dependencies = [ "base64 0.23.1", + "chacha20 0.10.2", "hex", "k256", "nfx-pay-wire", @@ -5368,9 +5392,9 @@ dependencies = [ [[package]] name = "rustix" -version = "1.1.5" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +checksum = "e5e76742e5294153d4dde1207dbec5bdd81eef6fabdc0d2118babadabfd858aa" dependencies = [ "bitflags 2.13.2", "errno", diff --git a/web/player/README.md b/web/player/README.md index 9b003fe..f7a5d09 100644 --- a/web/player/README.md +++ b/web/player/README.md @@ -17,7 +17,8 @@ const m = await resolveManifest(a, relays); // NFX-02 over Nostr, deletions const stop = watchEndpoints(a, m.namespace, relays, (e) => { // e.origins: https bases, e.trackers: wss trackers, e.stunServers: stun: URLs, // from the newest live beacon per seeder; e.originSources: the origins with the - // renditions each serves; e.availability: how many seeders hold each rendition + // renditions beacons claim each serves (hints); e.availability: how many seeders + // hold each rendition }); const session = await open(videoElement, { manifestJson: m.event, // verified again in WASM @@ -37,16 +38,26 @@ The types are in `src/api.ts`. `open()` also takes a bare `root` (with `video` a bind the hash list) instead of `manifestJson`. A seeder may hold only some renditions of a video (NFX-03 §4), so a session offers only -levels some source it uses can deliver (NFX-10 §1, `src/levels.ts`). It takes every file -from one origin: the first that serves a hash list that verifies and holds an allowed -rendition. Its levels are the allowed renditions (`renditions`, or all) that this origin -serves (an `OriginSource`'s `renditions`, from its beacon's `https` endpoint; a bare URL -serves all). The mesh adds none, since a segment peers do not deliver comes from that -origin. hls.js reads a master playlist cut to those variants, matched by playlist content -name, so it never learns of another level, and ABR picks freely among the rest, starting on -the first of `renditions`. `availability` (who holds which rendition, by distinct seeder key, -newest beacon each, at most 64) lets a page rank them; the test page passes them most-held -first when no seeder holds every rendition. +levels some source it uses can deliver (NFX-10 §1, `src/levels.ts`, `src/probe.ts`). It +takes every file from one origin, and **asks the origin** what it holds (nfx #85): once the +hash list verifies, `open()` fetches each allowed rendition's media playlist (verified) and +sends a `GET` for the init it names, reading at most 64 KiB; in parallel (6 at a time, at +most 32 renditions) and within 8 s per origin in all. A rendition is held when its init +answers `200` (or `206`). The first origin that holds an allowed rendition is used, and its +levels are the allowed renditions (`renditions`, or all) it holds; an origin that holds +none is skipped, with an `origin` event giving each rendition's answer. The mesh adds no +level, since a segment peers do not deliver comes from that origin. hls.js reads a master +playlist cut to those variants, matched by playlist content name, so it never learns of +another level, and ABR picks freely among the rest, starting on the first of `renditions`. + +What beacons say an origin serves (an `OriginSource`'s `renditions`, from its beacon's +`https` endpoint) is **a hint, not a constraint**: beacons come from any key, so a stranger +could otherwise deny an origin by naming it with a list that leaves out what the viewer +needs. The hint orders the work (an origin whose hint names an allowed rendition is tried +first, and the renditions it names are asked for first), and never removes an origin or a +level. `availability` (who holds which rendition, by distinct seeder key, newest beacon +each, at most 64) lets a page rank renditions; the test page shows it and leaves the levels +to `open()`. `signaling` is how mesh peers find each other and exchange offers, answers and ICE candidates, wired into the engine in one place (`src/mesh.ts` `engineSignaling`). Two kinds: @@ -77,8 +88,8 @@ npm ci npm run build # the WASM (wasm-pack) and the test page, out/ npm run build:lib # the library: out-lib/player.js (minified) and its declarations npm run typecheck -npm run unit # sanitisers, the beacon endpoint book and availability, open()'s option checks and levels -npm run e2e # the test page end to end (cargo, ffmpeg, openssl, Chromium), held renditions included +npm run unit # sanitisers, the beacon endpoint book and availability, open()'s option checks, levels and probes +npm run e2e # the test page end to end (cargo, ffmpeg, openssl, Chromium), held renditions and a hostile beacon included npm run e2e:mesh # the mesh, the bridge, the library's lifecycle, trackers from a beacon eval "$(bash e2e-strfry.sh /var/tmp/strfry)" # the stock relay, strfry at a pinned commit npm run e2e:nostr # two browsers over Nostr only (strfry, no tracker), the origin cut diff --git a/web/player/e2e.ts b/web/player/e2e.ts index b79cf49..bbfa838 100644 --- a/web/player/e2e.ts +++ b/web/player/e2e.ts @@ -15,13 +15,22 @@ * * Held renditions (NFX-03 §§4–5, NFX-10 §1): a second video in two renditions, 720p and * 360p, which the node holds in 360p only (`--seed #360p`): its origin answers 404 for - * 720p's files and its beacon's `https` endpoint names `renditions: ["360p"]`. By address, - * the player must offer 360p alone, play it, and never request one 720p file (playlist, - * init or segment), both when it takes its renditions from the beacons' availability and - * when the page asks for 720p first (`&renditions=720p,360p`: the origin's list cuts it). - * By origin and root, `&renditions=` cuts the levels the same way and is refused when it - * names a rendition the video lacks, or is malformed; without it both levels are listed, - * each with its rendition id. + * 720p's init and segments, and its beacon's `https` endpoint names `renditions: ["360p"]`. + * The library asks the origin which renditions it holds (nfx #85): for 720p it reads the + * playlist (verified, to learn the init's content name) and asks for the init once, which + * answers 404. So by address, with no renditions asked for and with 720p asked for first + * (`&renditions=720p,360p`), and by origin and root without `&renditions=`, the player must + * offer 360p alone, play it, and send exactly one request for 720p's playlist and one for + * its init per session, and none for a 720p segment. By origin and root, `&renditions=360p` + * asks for nothing of 720p; `&renditions=` is refused when it names a rendition the video + * lacks (before any probe), or is malformed. + * + * A hostile beacon (nfx #85): a third video in 720p and 360p, served whole by an honest + * origin (a plain NFX-05 §6 server over the store, behind TLS) that no honest beacon names. + * A second nfxd, under a stranger's key, seeds it in 360p only and names that origin in its + * `https` endpoint with `renditions: ["360p"]`, a narrowing claim. By address, the page + * must still offer 720p and 360p (each with its rendition id) and play; asked for 720p alone + * (`&renditions=720p`), it must play 720p from that origin rather than skip it. * * Usage: npm run build && npm run e2e * Needs `cargo` (builds nfxd and nfx-package) and ffmpeg (`NFX_FFMPEG`, else PATH). @@ -132,15 +141,24 @@ async function liar(upstream: string, segments: Set): Promise { return `http://127.0.0.1:${port}`; } -/** An https front for the origin, with a throwaway self-signed certificate (deleted with the - * work dir); the upstream is set once nfxd has printed its origin. */ +/** A throwaway self-signed certificate for 127.0.0.1, made once (deleted with the work dir). */ +let selfSignedPair: { key: Buffer; cert: Buffer } | null = null; +function selfSigned(): { key: Buffer; cert: Buffer } { + if (selfSignedPair === null) { + const key = join(work, 'tls.key'); + const cert = join(work, 'tls.crt'); + run('openssl', ['req', '-x509', '-newkey', 'ec', '-pkeyopt', 'ec_paramgen_curve:prime256v1', '-nodes', '-keyout', key, '-out', cert, '-days', '1', '-subj', '/CN=127.0.0.1', '-addext', 'subjectAltName=IP:127.0.0.1']); + selfSignedPair = { key: readFileSync(key), cert: readFileSync(cert) }; + } + return selfSignedPair; +} + +/** An https front for the origin, with the throwaway certificate; the upstream is set once + * nfxd has printed its origin. */ async function tlsProxy(): Promise<{ url: string; setUpstream: (u: string) => void; requests: string[] }> { - const key = join(work, 'tls.key'); - const cert = join(work, 'tls.crt'); - run('openssl', ['req', '-x509', '-newkey', 'ec', '-pkeyopt', 'ec_paramgen_curve:prime256v1', '-nodes', '-keyout', key, '-out', cert, '-days', '1', '-subj', '/CN=127.0.0.1', '-addext', 'subjectAltName=IP:127.0.0.1']); let upstream = ''; const requests: string[] = []; - const server = createHttpsServer({ key: readFileSync(key), cert: readFileSync(cert) }, (req, res) => { + const server = createHttpsServer(selfSigned(), (req, res) => { requests.push(new URL(req.url ?? '/', 'http://x').pathname); const up = httpRequest(`${upstream}${req.url}`, { method: req.method }, (r) => { res.writeHead(r.statusCode ?? 502, r.headers); @@ -154,6 +172,46 @@ async function tlsProxy(): Promise<{ url: string; setUpstream: (u: string) => vo return { url: `https://127.0.0.1:${port}`, setUpstream: (u) => (upstream = u), requests }; } +/** + * An honest origin (NFX-05 §6) by hand, behind TLS: for each of `roots`, its hash list at + * `/`, its master playlist at `//master.m3u8`, and every file it lists at + * `//.`, from `store`; anything else `404` (`no-store`). It serves every + * rendition, and records each request's path. + */ +async function staticOrigin(store: string, roots: string[]): Promise<{ url: string; requests: string[] }> { + const videos = new Map; master: string }>(); + for (const root of roots) { + const list = JSON.parse(readFileSync(join(store, root), 'utf8')) as HashListFile; + const master = list.files.find((f) => f.role === 'playlist-master')?.sha256; + if (master === undefined) throw new Error(`hash list ${root} has no master playlist`); + videos.set(root, { files: new Set(list.files.map((f) => f.sha256)), master }); + } + const types: Record = { m3u8: 'application/vnd.apple.mpegurl', mp4: 'video/mp4', m4s: 'video/iso.segment' }; + const requests: string[] = []; + const server = createHttpsServer(selfSigned(), (req, res) => { + const path = new URL(req.url ?? '/', 'http://x').pathname; + requests.push(path); + const [, root = '', name, ...more] = path.split('/'); + const video = videos.get(root); + let file: string | undefined; + let type = 'application/json'; + if (video !== undefined && more.length === 0 && (req.method === 'GET' || req.method === 'HEAD')) { + if (name === undefined) file = root; + else if (name === 'master.m3u8') [file, type] = [video.master, types.m3u8!]; + else { + const [sha = '', ext = ''] = name.split('.'); + if (video.files.has(sha)) [file, type] = [sha, types[ext] ?? 'application/octet-stream']; + } + } + const cors = { 'access-control-allow-origin': '*' }; + if (file === undefined) return res.writeHead(404, { ...cors, 'cache-control': 'no-store' }).end(); + res.writeHead(200, { ...cors, 'content-type': type, 'cache-control': 'public, max-age=31536000, immutable' }).end(readFileSync(join(store, file))); + }); + servers.push(server as unknown as Server); + const port = await new Promise((ok) => server.listen(0, '127.0.0.1', () => ok((server.address() as AddressInfo).port))); + return { url: `https://127.0.0.1:${port}`, requests }; +} + /** Wait until the player plays past 2 s or reports an error; return its state. */ async function settle(page: import('playwright').Page, timeout: number): Promise { await page.waitForFunction( @@ -235,12 +293,24 @@ async function main(): Promise { if (list2.renditions.map((r) => r.id).sort().join() !== '360p,720p') throw new Error(`video 2 renditions: ${JSON.stringify(list2.renditions)}`); const files720 = renditionFiles(join(pkg, 'store'), list2, '720p'); const files360 = renditionFiles(join(pkg, 'store'), list2, '360p'); + // A third video in the same two renditions, from a third clip, in a store of its own: an + // honest origin serves it whole, and only a stranger's beacon names that origin. + const clip3 = join(work, 'clip3.mp4'); + run(ffmpeg, ['-hide_banner', '-loglevel', 'error', '-f', 'lavfi', '-i', 'testsrc2=size=1280x720:rate=25', '-f', 'lavfi', '-i', 'sine=frequency=880:sample_rate=48000', '-t', '8', '-c:v', 'libx264', '-preset', 'ultrafast', '-pix_fmt', 'yuv420p', '-c:a', 'aac', clip3]); + const pkg3 = join(work, 'pkg3'); + run(bin('nfx-package'), [clip3, '--video', 'nfx:mainnet:1:player-e2e-hostile', '--out', pkg3, '--renditions', '720p,360p', '--preset', 'veryfast']); + const meta3 = JSON.parse(readFileSync(join(pkg3, 'nfx.json'), 'utf8')) as { root: string; video: string }; + const list3 = JSON.parse(readFileSync(join(pkg3, 'store', meta3.root), 'utf8')) as HashListFile; + if (list3.renditions.map((r) => r.id).sort().join() !== '360p,720p') throw new Error(`video 3 renditions: ${JSON.stringify(list3.renditions)}`); + const files720of3 = renditionFiles(join(pkg3, 'store'), list3, '720p'); const key = join(work, 'node.key'); const pubkey = run(bin('nfxd'), ['key', 'new', key]).split('\n')[0]!; const a = `38504:${pubkey}:${meta.video}`; const a2 = `38504:${pubkey}:${meta2.video}`; + const a3 = `38504:${pubkey}:${meta3.video}`; const tls = await tlsProxy(); + const whole = await staticOrigin(join(pkg3, 'store'), [meta3.root]); const node = await start( bin('nfxd'), ['run', '--key', key, '--store', join(pkg, 'store'), '--seed', a, '--seed', `${a2}#360p`, '--embed-relay', '127.0.0.1:0', '--origin', '127.0.0.1:0', '--https-url', tls.url], @@ -253,6 +323,18 @@ async function main(): Promise { if (published !== a) throw new Error(`published ${published}, expected ${a}`); const published2 = run(bin('nfxd'), ['publish', '--key', key, '--relay', relay, '--package', pkg2, '--title', 'Player e2e, held renditions']).trim(); if (published2 !== a2) throw new Error(`published ${published2}, expected ${a2}`); + const published3 = run(bin('nfxd'), ['publish', '--key', key, '--relay', relay, '--package', pkg3, '--title', 'Player e2e, a hostile beacon']).trim(); + if (published3 !== a3) throw new Error(`published ${published3}, expected ${a3}`); + // The stranger: another key, its own copy of the store and state, the creator's relay. It + // seeds video 3 in 360p only and names the honest origin, which it does not run, as + // serving 360p alone. + const strangerKey = join(work, 'stranger.key'); + run(bin('nfxd'), ['key', 'new', strangerKey]); + const strangerStore = join(work, 'stranger-store'); + cpSync(join(pkg3, 'store'), strangerStore, { recursive: true }); + const seeding3 = new RegExp(`${a3.replace(/[^0-9A-Za-z]/g, (c) => `\\${c}`)}: Seeding`); + const stranger = start(bin('nfxd'), ['run', '--key', strangerKey, '--store', strangerStore, '--relay', relay, '--seed', `${a3}#360p`, '--https-url', whole.url], [seeding3]); + stranger.catch(() => {}); // awaited below; a failure before then must not go unhandled await new Promise((ok, fail) => { const timer = setTimeout(() => fail(new Error('never seeding')), 60_000); let text = ''; @@ -276,6 +358,7 @@ async function main(): Promise { if (served.playlist720 !== 200 || served.init720 !== 404 || served.segment720 !== 404 || served.segment360 !== 200) { throw new Error(`node origin for video 2: ${JSON.stringify(served)}`); } + await stranger; const list = JSON.parse(readFileSync(join(pkg, 'store', meta.root), 'utf8')) as { files: { role: string; sha256: string }[] }; const segments = new Set(list.files.filter((f) => f.role === 'segment').map((f) => f.sha256)); @@ -350,44 +433,77 @@ async function main(): Promise { } await tlsContext.close(); - // Held renditions: video 2, which the node holds in 360p only. By address, the origin - // comes from the beacon with the renditions it serves; once with the renditions the - // page takes from the beacons' availability, once asking for 720p first. - const heldContext = await browser.newContext({ ignoreHTTPSErrors: true }); // self-signed, test only + // Held renditions: video 2, which the node holds in 360p only, and the hostile beacon: + // video 3, which only the stranger's beacon names, on the honest origin. By address, + // each page in a context of its own (no shared HTTP cache, so each request count is one + // session's); all four wait for the next beacons together. + const fresh = async (): Promise => (await browser.newContext({ ignoreHTTPSErrors: true })).newPage(); // self-signed, test only const q2 = (extra: string): string => `${player}/?a=${encodeURIComponent(a2)}&relay=${encodeURIComponent(relay)}${extra}`; - const fromBeacons = await heldContext.newPage(); - const askedFirst = await heldContext.newPage(); - await Promise.all([fromBeacons.goto(q2('')), askedFirst.goto(q2('&renditions=720p,360p'))]); - const [beaconHeld, originCut] = await Promise.all([settle(fromBeacons, 150_000), settle(askedFirst, 150_000)]); + const q3 = (extra: string): string => `${player}/?a=${encodeURIComponent(a3)}&relay=${encodeURIComponent(relay)}${extra}`; + const [fromBeacons, askedFirst, hostileAll, hostile720] = await Promise.all([fresh(), fresh(), fresh(), fresh()]); + await Promise.all([fromBeacons.goto(q2('')), askedFirst.goto(q2('&renditions=720p,360p')), hostileAll.goto(q3('')), hostile720.goto(q3('&renditions=720p'))]); + const [beaconHeld, originCut, narrowed, wanted720] = await Promise.all([ + settle(fromBeacons, 150_000), + settle(askedFirst, 150_000), + settle(hostileAll, 150_000), + settle(hostile720, 150_000), + ]); results.heldFromBeacons = beaconHeld; results.heldAskedFirst = originCut; + results.hostileBeacon = narrowed; + results.hostileBeacon720 = wanted720; const only360 = (s: { levels: { id: string; height: number }[] }): boolean => s.levels.length === 1 && s.levels[0]?.id === '360p' && s.levels[0]?.height === 360; - const healthy = (s: { resolved: { root: string } | null; origin: string; playing: boolean; rejected: number; errors: string[] }): boolean => - s.resolved?.root === meta2.root && s.origin === tls.url && s.playing && s.rejected === 0 && s.errors.length === 0; + const levelIds = (s: { levels: { id: string; height: number }[] }): string => s.levels.map((l) => `${l.id}@${l.height}`).join(); + type Settled = { resolved: { root: string } | null; origin: string; playing: boolean; rejected: number; errors: string[] }; + const healthy = (s: Settled, root: string, from: string): boolean => + s.resolved?.root === root && s.origin === from && s.playing && s.rejected === 0 && s.errors.length === 0; const wantAvailability = { seeders: 1, all: 0, renditions: [{ id: '360p', seeders: 1 }] }; - const wantSources = [{ url: tls.url, renditions: ['360p'] }]; if ( - !healthy(beaconHeld) || + !healthy(beaconHeld, meta2.root, tls.url) || !only360(beaconHeld) || - JSON.stringify(beaconHeld.renditions) !== '["360p"]' || + beaconHeld.renditions !== null || JSON.stringify(beaconHeld.availability) !== JSON.stringify(wantAvailability) || - JSON.stringify(beaconHeld.originSources) !== JSON.stringify(wantSources) + JSON.stringify(beaconHeld.originSources) !== JSON.stringify([{ url: tls.url, renditions: ['360p'] }]) ) { throw new Error(`held renditions, from beacons: ${JSON.stringify(beaconHeld)}`); } - if (!healthy(originCut) || !only360(originCut) || JSON.stringify(originCut.renditions) !== '["720p","360p"]') { + if (!healthy(originCut, meta2.root, tls.url) || !only360(originCut) || JSON.stringify(originCut.renditions) !== '["720p","360p"]') { throw new Error(`held renditions, 720p asked for first: ${JSON.stringify(originCut)}`); } + // 720p of video 2 through the TLS front: per session, its playlist once and its init + // once (the probe, answered 404), and never a segment. + const count = (log: string[], root: string, names: string[]): number[] => names.map((n) => log.filter((p) => p === `/${root}/${n}`).length); + const [playlist720 = '', init720 = '', ...segments720] = files720.names; const tls720 = asked(tls.requests, meta2.root, files720.shas); const tls360 = asked(tls.requests, meta2.root, files360.shas); - results.heldRequests = { r720: tls720.length, r360: tls360.length }; - if (tls720.length !== 0 || tls360.length < 3) throw new Error(`held renditions, requests: 720p ${JSON.stringify(tls720)}, 360p ${tls360.length}`); - await heldContext.close(); + results.heldRequests = { r720: tls720, r360: tls360.length }; + const [tlsPlaylist, tlsInit] = count(tls.requests, meta2.root, [playlist720, init720]); + if (tlsPlaylist !== 2 || tlsInit !== 2 || tls720.length !== 4 || count(tls.requests, meta2.root, segments720).some((n) => n !== 0) || tls360.length < 3) { + throw new Error(`held renditions, requests: 720p ${JSON.stringify(tls720)}, 360p ${tls360.length}`); + } + // The hostile beacon: its narrowing claim reached the library as the origin's hint, and + // was only a hint: both levels offered, and 720p played when asked for. + const wantHostileSources = JSON.stringify([{ url: whole.url, renditions: ['360p'] }]); + if ( + !healthy(narrowed, meta3.root, whole.url) || + levelIds(narrowed) !== '360p@360,720p@720' || + narrowed.renditions !== null || + JSON.stringify(narrowed.originSources) !== wantHostileSources || + JSON.stringify(narrowed.availability) !== JSON.stringify(wantAvailability) + ) { + throw new Error(`a hostile beacon, every rendition: ${JSON.stringify(narrowed)}`); + } + const honest720 = asked(whole.requests, meta3.root, files720of3.shas).filter((p) => p.endsWith('.m4s')); + results.hostileRequests = { r720segments: honest720.length }; + if (!healthy(wanted720, meta3.root, whole.url) || levelIds(wanted720) !== '720p@720' || JSON.stringify(wanted720.originSources) !== wantHostileSources || honest720.length === 0) { + throw new Error(`a hostile beacon, 720p asked for: ${JSON.stringify(wanted720)}, 720p segments ${honest720.length}`); + } + for (const p of [fromBeacons, askedFirst, hostileAll, hostile720]) await p.context().close(); - // By origin and root (no beacon): `&renditions=` cuts the levels the same way. + // By origin and root (no beacon): `&renditions=360p` asks for nothing of 720p. const rec = await recorder(origin); - const byRoot = (o: string, extra: string): string => `${player}/?origin=${encodeURIComponent(o)}&root=${meta2.root}${extra}`; + const byRoot = (o: string, extra: string, r = meta2.root): string => `${player}/?origin=${encodeURIComponent(o)}&root=${r}${extra}`; const cutPage = await context.newPage(); await cutPage.goto(byRoot(rec.url, '&renditions=360p')); const cut = await settle(cutPage, 60_000); @@ -396,14 +512,28 @@ async function main(): Promise { if (!cut.playing || !only360(cut) || cut.errors.length !== 0 || cut.rejected !== 0 || rec720.length !== 0) { throw new Error(`renditions by root: ${JSON.stringify(cut)}, 720p requests ${JSON.stringify(rec720)}`); } - // Without it, both levels are listed, each with its rendition id (by playlist content - // name, in hls.js's order); level 0, the 360p one, plays. - const both = await context.newPage(); - await both.goto(byRoot(origin, '&level=0')); + // Without it, the library asks: 360p alone is offered, after one request for 720p's + // playlist and one for its init, and no 720p segment is requested. + const rec2 = await recorder(origin); + const askedPage = await (await browser.newContext()).newPage(); + await askedPage.goto(byRoot(rec2.url, '')); + const probed = await settle(askedPage, 60_000); + const rec2of720 = asked(rec2.requests, meta2.root, files720.shas); + results.heldByRootAsked = { levels: probed.levels, r720: rec2of720 }; + const [recPlaylist, recInit] = count(rec2.requests, meta2.root, [playlist720, init720]); + if (!probed.playing || !only360(probed) || probed.errors.length !== 0 || recPlaylist !== 1 || recInit !== 1 || rec2of720.length !== 2) { + throw new Error(`held renditions by root, asked: ${JSON.stringify(probed)}, 720p requests ${JSON.stringify(rec2of720)}`); + } + await askedPage.context().close(); + // An origin that holds both: both levels are listed, each with its rendition id (by + // playlist content name, in hls.js's order); level 0, the 360p one, plays. + const bothContext = await browser.newContext({ ignoreHTTPSErrors: true }); // self-signed, test only + const both = await bothContext.newPage(); + await both.goto(byRoot(whole.url, '&level=0', meta3.root)); const listed = await settle(both, 60_000); - results.heldBothListed = listed.levels; - const ids = listed.levels.map((l: { id: string; height: number }) => `${l.id}@${l.height}`).join(); - if (!listed.playing || listed.errors.length !== 0 || ids !== '360p@360,720p@720') throw new Error(`both levels: ${JSON.stringify(listed)}`); + results.bothListed = listed.levels; + if (!listed.playing || listed.errors.length !== 0 || levelIds(listed) !== '360p@360,720p@720') throw new Error(`both levels: ${JSON.stringify(listed)}`); + await bothContext.close(); // Refused: a rendition the video lacks (once the hash list has verified, and before // anything plays), and malformed lists (before anything is fetched). const refusals: Record = {}; diff --git a/web/player/src/api.ts b/web/player/src/api.ts index af61d99..fe7fab6 100644 --- a/web/player/src/api.ts +++ b/web/player/src/api.ts @@ -41,16 +41,22 @@ export interface ResolveOptions { } /** - * An origin `open()` may play from, and the renditions it serves (NFX-03 §4). + * An origin `open()` may play from, and the renditions it is said to serve (NFX-03 §4). * `watchEndpoints` reports them (`Endpoints.originSources`); a page may also write its own. */ export interface OriginSource { /** `http(s)://host[:port][/prefix]`, as `OpenCommon.origins` takes a bare string. */ url: string; /** - * The ids of the renditions whose `init` and `segment` files it serves (the hash list's - * `renditions[].id`): its `https` endpoint's `renditions`. Absent: every rendition. Empty: - * none, so `open()` skips it. Ids the hash list does not name are ignored. At most 32. + * **A hint, never a constraint** (it was a constraint until nfx #85): the ids of the + * renditions whose `init` and `segment` files it is said to serve (the hash list's + * `renditions[].id`), as its `https` endpoint's `renditions` claim them. Beacons come from + * any key, so `open()` uses this only to order: an origin whose hint names an allowed + * rendition is tried before one whose hint does not, and the renditions it names are asked + * for first. It never skips an origin, and never adds or removes a level: which levels a + * session offers is what the origin itself answers for (`OpenCommon`). Absent: no claim + * (every rendition). Empty: a claim of none, so the origin is tried after the others. + * Ids the hash list does not name are ignored. At most 32. */ renditions?: string[]; } @@ -88,16 +94,18 @@ export interface RenditionAvailability { export interface Endpoints { /** * `https://host[:port][/prefix]` origin bases (NFX-05 §6), at most 8, most-named first. - * Give `open()` `originSources` instead: a bare URL is taken to serve every rendition, so - * a session could offer a level that origin answers with `404`. + * `originSources` is the same list with what beacons claim each origin serves; give + * `open()` either (it asks each origin what it holds). */ origins: string[]; /** - * `origins` again, in the same order, each with the renditions it serves, as `open()`'s - * `origins` takes them. When several seeders name one origin, it serves only what every - * one of them says it does: a beacon can narrow what an origin is taken to serve, never - * widen it, so the worst a lying beacon does is have `open()` skip that origin, rather - * than offer a level whose segments it answers with `404`. + * `origins` again, in the same order, each with the renditions beacons claim it serves, + * as `open()`'s `origins` takes them. When several seeders name one origin, the claim is + * only what every one of them says it serves. That is a hint (`OriginSource.renditions`): + * since nfx #85 `open()` asks the origin itself which renditions it holds, so a beacon + * from any key, narrowing or lying, can change the order in which origins are tried but + * can neither make `open()` skip an origin that serves the allowed renditions nor make + * it offer a level the origin does not serve. */ originSources: OriginSource[]; /** `wss://` trackers from `webrtc` endpoints (NFX-10 §2), at most 4, most-named first. */ @@ -189,8 +197,9 @@ export interface NostrSignaling { export type PlayerEvent = | { type: 'log'; message: string } /** - * An origin served a hash list that verified and is used (`ok`), or was not (`why`: no - * hash list that verifies, or none of the allowed renditions held). + * An origin served a hash list that verified and answered for at least one allowed + * rendition, and is used (`ok`); or it was not (`why`: no hash list that verifies, or no + * allowed rendition's init answered, with each rendition's answer). */ | { type: 'origin'; origin: string; ok: boolean; why?: string } | { type: 'verified'; sha256: string; url: string } @@ -203,22 +212,36 @@ export type PlayerEvent = /** * What `open()` plays, from where, and how. * - * **Which levels a session offers** (NFX-10 §1, NFX-03 §5). The session takes every file - * from one origin: the first of `origins`, in order, that serves a hash list that verifies - * and holds at least one allowed rendition (an origin that cannot is skipped, with an - * `origin` event saying why). The session's levels are then the allowed renditions - * (`renditions`, or every rendition of the hash list) that this origin serves (its - * `OriginSource.renditions`, or every one). The mesh never adds a level: a segment peers do - * not deliver in time comes over HTTP from that origin, so a rendition it does not serve - * would only fail. hls.js is told of no other level: the master playlist it reads is cut to - * those renditions' variants, matched by playlist content name (NFX-10 §2), and a level - * outside them stops the session with an `error` event before any segment is fetched. + * **Which levels a session offers** (NFX-10 §1, NFX-03 §5; nfx #85). The session takes + * every file from one origin. `open()` tries `origins` in order, except that an origin + * whose `OriginSource.renditions` hint names none of the allowed renditions is tried after + * the others. On each, once its hash list has verified, it **asks the origin** which of + * the allowed renditions (`renditions`, or every rendition of the hash list; at most 32, + * those the hint names first) it holds: for each, it fetches the rendition's media + * playlist, verifies it, and sends a `GET` for the init it names (NFX-05 §3), reading at + * most 64 KiB of the answer. These run in parallel, 6 at a time, and all of one origin's + * together get 8 s. A rendition is held when its init answers `200` (or `206`); anything + * else, or no answer in time, and it is not. The first origin that holds at least one + * allowed rendition is used (one that holds none, or whose hash list does not verify, is + * skipped, with an `origin` event saying why), and the session's levels are the allowed + * renditions it holds. So the cost is a playlist and an init per allowed rendition per + * session, both of which hls.js would fetch anyway for the level it plays (they are + * immutable, so the HTTP cache usually answers the second time). + * + * Beacon claims never remove an origin or a level: a beacon from any key can only change + * the order above. And a level is never offered whose segments no source can deliver: the + * mesh never adds a level, since a segment peers do not deliver in time comes over HTTP + * from that origin, so a rendition it does not hold would only fail. hls.js is told of no + * other level: the master playlist it reads is cut to those renditions' variants, matched + * by playlist content name (NFX-10 §2), and a level outside them stops the session with an + * `error` event before any segment is fetched. */ export interface OpenCommon { /** * Origin bases (NFX-05 §6) to try in order (above): `http(s)://host[:port][/prefix]`, no - * query, fragment or credentials. A bare string serves every rendition; an - * `OriginSource` names those it serves (`Endpoints.originSources` are such). + * query, fragment or credentials. A bare string makes no claim; an `OriginSource` names + * the renditions it is said to serve, as a hint (`Endpoints.originSources` are such). + * Either way `open()` asks the origin what it holds. */ origins: (string | OriginSource)[]; /** @@ -227,9 +250,10 @@ export interface OpenCommon { * Default: every rendition. `open()` refuses an empty or malformed list with a * `TypeError` before it fetches anything, and an id the hash list lacks with a * `RangeError` once the hash list has verified, leaving nothing running either way. - * Levels are cut to these (and to what the origin serves, above); hls.js's ABR then picks - * freely among them, starting on the first of them the origin serves unless `level` says - * otherwise. + * Levels are cut to these (and to what the origin answers that it holds, above); hls.js's + * ABR then picks freely among them, starting on the first of them the origin holds unless + * `level` says otherwise. Only these are asked for, so naming one rendition costs one + * probe. */ renditions?: string[]; /** @@ -251,9 +275,9 @@ export interface OpenCommon { autoplay?: boolean; /** * Pin this level once the master playlist is parsed (test knob): an index into the - * session's levels (`Session.levels`, after `renditions` and the origin have cut them, in - * hls.js's order), not into the master playlist. It turns ABR off and overrides the start - * `renditions` would give; past the last level it is ignored. + * session's levels (`Session.levels`, after `renditions` and the origin's answers have + * cut them, in hls.js's order), not into the master playlist. It turns ABR off and + * overrides the start `renditions` would give; past the last level it is ignored. */ level?: number; /** Seconds ahead of the playhead fetched over HTTP; the rest may come from peers (test knob). */ @@ -292,8 +316,8 @@ export interface Stats { /** One video playing in one element. */ export interface Session { /** - * The session's levels (`OpenCommon`: only renditions allowed and served by its origin), - * once hls.js has parsed the master playlist (empty before). + * The session's levels (`OpenCommon`: only renditions allowed and held by its origin, by + * its own answers), once hls.js has parsed the master playlist (empty before). */ readonly levels: Level[]; /** Play `index` from now on, or -1 to let hls.js pick. */ diff --git a/web/player/src/levels.ts b/web/player/src/levels.ts index 94d2042..afaf203 100644 --- a/web/player/src/levels.ts +++ b/web/player/src/levels.ts @@ -2,6 +2,11 @@ * Which levels a session offers (NFX-10 §1, NFX-03 §§4–5), as plain data: no network, no * hls.js, no WASM. `open()` (./lib.ts) applies it; unit.ts tests it. * + * What beacons say an origin serves (`OriginSource.renditions`) is a hint, never a + * constraint (nfx #85): it orders which origin is tried first (`originOrder`) and which + * renditions are asked for first (`probeOrder`). Which levels a session offers is what the + * origin itself answers for (./probe.ts), cut by the caller's `renditions`. + * * A level is matched to its rendition by its playlist's content name (NFX-10 §2), never by * bitrate or size: the last element of the playlist URL's path is `.`, and the * hash list's `renditions[].playlist` names the playlist file with that sha256. This is @@ -27,10 +32,12 @@ function show(id: unknown): string { return JSON.stringify(typeof id === 'string' && id.length > 64 ? `${id.slice(0, 64)}…` : id) ?? String(id); } -/** One rendition of a verified hash list: its id, and the sha256 of its playlist. */ +/** One rendition of a verified hash list: its id, and the sha256 and size of its playlist. */ export interface HashListRendition { id: string; playlist: string; + /** The playlist's listed size in bytes, or null if the list gives none that is usable. */ + playlistSize: number | null; } /** What `open()` reads from a verified hash list. */ @@ -48,19 +55,20 @@ export interface HashListLevels { */ export function hashListLevels(json: string): HashListLevels { const list = JSON.parse(json) as { files?: unknown; renditions?: unknown }; - const playlists = new Map(); + const playlists = new Map(); let master: string | null = null; for (const f of Array.isArray(list.files) ? (list.files as unknown[]) : []) { - const { name, role, sha256 } = (f ?? {}) as { name?: unknown; role?: unknown; sha256?: unknown }; + const { name, role, sha256, size } = (f ?? {}) as { name?: unknown; role?: unknown; sha256?: unknown; size?: unknown }; if (typeof name !== 'string' || typeof sha256 !== 'string') continue; - if (role === 'playlist' && !playlists.has(name)) playlists.set(name, sha256); + const bytes = typeof size === 'number' && Number.isSafeInteger(size) && size >= 0 ? size : null; + if (role === 'playlist' && !playlists.has(name)) playlists.set(name, { sha256, size: bytes }); if (role === 'playlist-master' && master === null) master = sha256; } const renditions: HashListRendition[] = []; for (const r of Array.isArray(list.renditions) ? (list.renditions as unknown[]) : []) { const { id, playlist } = (r ?? {}) as { id?: unknown; playlist?: unknown }; - const sha = typeof playlist === 'string' ? playlists.get(playlist) : undefined; - if (typeof id === 'string' && sha !== undefined) renditions.push({ id, playlist: sha }); + const file = typeof playlist === 'string' ? playlists.get(playlist) : undefined; + if (typeof id === 'string' && file !== undefined) renditions.push({ id, playlist: file.sha256, playlistSize: file.size }); } return { renditions, master }; } @@ -105,32 +113,55 @@ export function checkRenditions(list: unknown): string[] | null { } /** - * Whether an origin that serves `serves` (null: every rendition) can serve any of `wanted` - * (null: every rendition of the hash list), as far as is known before the hash list is - * read. `open()` skips one that cannot without fetching from it. + * Whether an origin's hint (`hint`: what beacons say it serves; null: no claim, every + * rendition) names any of `wanted` (null: every rendition of the hash list). A hint only: + * `originOrder` tries an origin whose hint does not after those whose hint does, never + * skips it. + */ +export function mayServe(wanted: readonly string[] | null, hint: readonly string[] | null): boolean { + if (hint === null) return true; + return wanted === null ? hint.length > 0 : wanted.some((id) => hint.includes(id)); +} + +/** + * `origins` in the order `open()` tries them: those whose hint names an allowed rendition + * (or makes no claim) first, then the rest, each group in the given order. Every origin is + * kept: a beacon from any key can demote an origin this way, never remove it, and an + * origin tried later is still asked what it holds. + */ +export function originOrder(origins: readonly T[], wanted: readonly string[] | null): T[] { + return [...origins.filter((o) => mayServe(wanted, o.renditions)), ...origins.filter((o) => !mayServe(wanted, o.renditions))]; +} + +/** + * The renditions to ask an origin for, in order: those of `ids` (the allowed ones, in the + * caller's order) that its hint names first, then the rest, at most `MAX_RENDITIONS`. A hint + * orders, never removes; a rendition past the cap is not asked for, so it is not offered. */ -export function mayServe(wanted: readonly string[] | null, serves: readonly string[] | null): boolean { - if (serves === null) return true; - return wanted === null ? serves.length > 0 : wanted.some((id) => serves.includes(id)); +export function probeOrder(ids: readonly string[], hint: readonly string[] | null): string[] { + const named = hint === null ? [...ids] : ids.filter((id) => hint.includes(id)); + const rest = hint === null ? [] : ids.filter((id) => !hint.includes(id)); + return [...named, ...rest].slice(0, MAX_RENDITIONS); } /** * A session's renditions on one origin: the allowed ones (`wanted`, or every id of the hash - * list in its order) that the origin serves (`serves`, or every one), in `wanted`'s order. - * Empty when it serves none of them. Throws a `RangeError` for an id of `wanted` the hash - * list (`ids`) does not have: a caller's mistake, the same on every origin. + * list in its order) that the origin answered for (`held`, from its probes; null before + * they ran: every allowed one), in `wanted`'s order. Empty when it holds none of them. + * Throws a `RangeError` for an id of `wanted` the hash list (`ids`) does not have: a + * caller's mistake, the same on every origin, so `open()` checks it before any probe. */ export function sessionRenditions( ids: readonly string[], wanted: readonly string[] | null, - serves: readonly string[] | null, + held: readonly string[] | null, ): string[] { const missing = wanted?.find((id) => !ids.includes(id)); if (missing !== undefined) { const has = ids.slice(0, 16).map(show).join(', ') + (ids.length > 16 ? ', …' : ''); throw new RangeError(`nfx: renditions: ${show(missing)} is not a rendition of this video (it has ${has})`); } - return (wanted ?? ids).filter((id) => serves === null || serves.includes(id)); + return (wanted ?? ids).filter((id) => held === null || held.includes(id)); } /** diff --git a/web/player/src/lib.ts b/web/player/src/lib.ts index 5af4082..7892b07 100644 --- a/web/player/src/lib.ts +++ b/web/player/src/lib.ts @@ -8,11 +8,12 @@ * - `watchEndpoints(a, namespace, relays, onChange)`: origins and trackers named by * seeders' live beacons (NFX-03), validated and capped; * - `open(videoElement, options)`: verify the manifest again, fetch the hash list from the - * first origin whose copy verifies, and play with hls.js behind a loader that checks - * every playlist, init and segment against it before hls.js sees a byte. With a - * `signaling` (WebTorrent trackers, or Nostr relays with no tracker: NFX-10 §6), the - * NFX-10 browser mesh (p2p-media-loader) joins in, its segments checked the same way. - * `destroy()` stops all of it. + * first origin whose copy verifies and that answers for an allowed rendition's init (the + * origin is asked; beacons' claims only order the asking), and play with hls.js behind a + * loader that checks every playlist, init and segment against it before hls.js sees a + * byte. With a `signaling` (WebTorrent trackers, or Nostr relays with no tracker: NFX-10 + * §6), the NFX-10 browser mesh (p2p-media-loader) joins in, its segments checked the same + * way. `destroy()` stops all of it. * * Nothing here reads the page's URL or DOM beyond the element it is given, and nothing * here keeps state between sessions but the loaded WASM module. @@ -50,7 +51,7 @@ import { checkSignaling as checkSignalingOption, originBase as beaconOrigin, } from './endpoints'; -import { MAX_RENDITIONS as LEVELS_MAX_RENDITIONS, checkRenditions, cutMaster, mayServe, sessionRenditions } from './levels'; +import { MAX_RENDITIONS as LEVELS_MAX_RENDITIONS, checkRenditions, cutMaster, originOrder } from './levels'; import { MAX_RELAYS as MESH_MAX_RELAYS, MAX_TRACKERS as MESH_MAX_TRACKERS, @@ -58,9 +59,10 @@ import { relayUrl as meshRelayUrl, trackerUrl as meshTrackerUrl, } from './mesh'; +import { type Probe, type Selection, describe as describeProbes, initUrl, probeStatus, selectRenditions } from './probe'; import { NostrSignaler } from './signal-nostr'; import { resolveManifest as resolveOverRelays, watchEndpoints as watchBeacons } from './resolve'; -import { Anchor, type ManifestBinding, loadWasm } from './verify'; +import { Anchor, MAX_HASH_LIST_BYTES, type ManifestBinding, loadWasm } from './verify'; // Everything public is declared here or in ./api.ts, so the declaration file built from // this one needs nothing but api.d.ts beside it. @@ -127,8 +129,9 @@ export function resolveManifest(a: string, relays: string[], options: ResolveOpt * Report the origins, trackers and STUN servers seeders' live beacons for `a` name, and * which renditions they hold, each time that changes (the newest unexpired beacon per * seeder counts, at most 64 seeders; `https://` origins only, at most 8, with the - * renditions each serves; `wss://` trackers only, at most 4; global unicast STUN addresses - * only, at most 4; `availability`). Returns `stop`, which closes the subscriptions. + * renditions beacons claim each serves, a hint for `open()`; `wss://` trackers only, at + * most 4; global unicast STUN addresses only, at most 4; `availability`). Returns `stop`, + * which closes the subscriptions. */ export function watchEndpoints( a: string, @@ -148,12 +151,13 @@ function abortError(signal: AbortSignal): unknown { } /** - * Play a video in `el`. Resolves once a hash list has verified and hls.js has started; - * levels follow (`levels` event). Rejects if the manifest fails NFX-02, if no origin - * serves a hash list that verifies and holds an allowed rendition (`OpenCommon` has the - * rule), if `renditions` names one the hash list lacks, if MediaSource is missing (native - * HLS would play unverified bytes), or if `signal` aborts first. A rejected `open()` leaves - * nothing running. + * Play a video in `el`. Resolves once a hash list has verified, the origin has answered + * which allowed renditions it holds, and hls.js has started; levels follow (`levels` + * event). Rejects if the manifest fails NFX-02, if no origin serves a hash list that + * verifies and answers for an allowed rendition's init (`OpenCommon` has the rule), if + * `renditions` names one the hash list lacks, if MediaSource is missing (native HLS would + * play unverified bytes), or if `signal` aborts first. A rejected `open()` leaves nothing + * running. */ export async function open(el: HTMLVideoElement, options: OpenOptions): Promise { const { signal } = options; @@ -203,12 +207,10 @@ export async function open(el: HTMLVideoElement, options: OpenOptions): Promise< failures.push(`${origin}: ${why}`); emit({ type: 'origin', origin, ok: false, why }); }; - for (const { url: origin, renditions: serves } of origins) { - // Known before any fetch: it serves none of the renditions asked for. - if (!mayServe(wanted, serves)) { - skip(origin, 'it serves none of the allowed renditions'); - continue; - } + // What beacons say an origin serves (`hint`) only orders: origins whose hint names an + // allowed rendition are tried first, and those renditions are asked for first. Which + // renditions the session offers is what the origin answers for (nfx #85). + for (const { url: origin, renditions: hint } of originOrder(origins, wanted)) { let loaded: Anchor; try { loaded = await Anchor.load(origin, root, binding, signal); @@ -217,21 +219,33 @@ export async function open(el: HTMLVideoElement, options: OpenOptions): Promise< skip(origin, message(e)); continue; } - // A RangeError (an id the hash list lacks) ends open(): every origin serves the same list. - renditions = sessionRenditions( - loaded.renditions.map((r) => r.id), - wanted, - serves, - ); + let picked: Selection; + try { + // A RangeError (an id the hash list lacks) ends open(), before any probe: every + // origin serves the same list. + picked = await selectRenditions( + loaded.renditions.map((r) => r.id), + wanted, + hint, + initProbe(loaded), + { signal }, + ); + } catch (e) { + if (signal?.aborted) throw abortError(signal); + throw e; + } + renditions = picked.renditions; + const summary = describeProbes(picked.outcomes) + (picked.unasked > 0 ? `; ${picked.unasked} more not asked for (at most ${MAX_RENDITIONS})` : ''); if (renditions.length === 0) { - skip(origin, 'it serves none of the allowed renditions'); + skip(origin, `it holds none of the allowed renditions (${summary})`); continue; } + log(`${origin} holds ${renditions.join(', ')} of the allowed renditions (${summary})`); anchor = loaded; emit({ type: 'origin', origin, ok: true }); break; } - if (anchor === null) throw new Error(`no origin served a hash list that verifies and an allowed rendition (${failures.join('; ')})`); + if (anchor === null) throw new Error(`no origin served a hash list that verifies and held an allowed rendition (${failures.join('; ')})`); if (signal?.aborted) throw abortError(signal); log(`hash list verified: ${anchor.size} files of ${anchor.video}${binding ? ' (bound to the manifest)' : ''}`); const levelRule = { renditions, preferStart: wanted !== null }; @@ -240,6 +254,38 @@ export async function open(el: HTMLVideoElement, options: OpenOptions): Promise< }); } +/** + * How to ask the origin `anchor` came from whether it holds a rendition (./probe.ts): its + * media playlist, fetched and verified against the hash list, names its init (NFX-05 §3), + * and a GET of that init must answer 200 or 206. Every URL is one the hash list names, + * under `//`: the playlist's content name, and the init's, read from bytes + * that verified. + */ +/** `.`: a content name (NFX-05 §3), with no path, query or fragment. */ +const CONTENT_NAME = /^[0-9a-f]{64}\.[0-9A-Za-z]{1,8}$/; + +function initProbe(anchor: Anchor): Probe { + const base = `${anchor.origin}/${anchor.root}/`; + return async (id, signal) => { + const r = anchor.renditions.find((x) => x.id === id); + if (r === undefined) throw new Error('not a rendition of the hash list'); + const playlistUrl = `${base}${r.playlist}.m3u8`; + let text: string; + try { + const bytes = await anchor.fetchChecked(playlistUrl, Math.min(r.playlistSize ?? MAX_HASH_LIST_BYTES, MAX_HASH_LIST_BYTES), signal); + text = new TextDecoder('utf-8', { fatal: true }).decode(bytes); + } catch (e) { + throw new Error(`playlist: ${message(e)}`); + } + // A content name right under `//` (NFX-05 §3), and one the hash list names: + // nothing else is ever requested, whatever the playlist holds. + const init = initUrl(text, playlistUrl); + if (init === null || !init.startsWith(base) || !CONTENT_NAME.test(init.slice(base.length))) throw new Error('its playlist names no init'); + anchor.expected(init); // throws for a file the hash list does not name + return probeStatus((url, req) => fetch(url, req), init, signal); + }; +} + interface StartOptions { /** The session's renditions, in order of preference (`OpenCommon` has the rule). */ renditions: string[]; diff --git a/web/player/src/player.ts b/web/player/src/player.ts index 6f3959e..bf29aa6 100644 --- a/web/player/src/player.ts +++ b/web/player/src/player.ts @@ -23,12 +23,11 @@ * `&level=` (pin a level). * * Which renditions it offers (NFX-03 §5, NFX-10 §1): by address, each beacon-named origin - * goes to the library with the renditions its `https` endpoint names, and the library offers - * only what the origin it plays from serves. When every seeder that holds any rendition - * names which (no endpoint holds them all), the page also passes those renditions, the - * most-held first, so playback starts on the one most seeders hold and ABR picks among the - * rest; if a beacon named one the video lacks, it plays again without that preference. - * `&renditions=[,…]` (test knob) passes that list instead, as given. + * goes to the library with the renditions its `https` endpoint names, as a hint, and the + * library offers what the origin it plays from answers that it holds (nfx #85). The page + * never turns what beacons say (`availability`, `originSources`) into a list of allowed + * renditions: beacons come from any key, and such a list would let one cut the levels. + * `&renditions=[,…]` (test knob) passes a list, as given. * * `window.__nfx` exposes state for automation. */ @@ -182,15 +181,6 @@ const common = { level: whole(params.get('level'), 16), } satisfies Partial; -/** - * The renditions to offer, from what seeders hold (NFX-03 §5), the most-held first: only - * when no seeder holds every rendition, since those ids are the hash list's, which this page - * does not read. Otherwise none: the library offers every rendition the origin serves. - */ -function fromAvailability(av: RenditionAvailability): string[] | undefined { - return av.all === 0 && av.renditions.length > 0 ? av.renditions.map((r) => r.id) : undefined; -} - /** Open a session, and show it in `__nfx`. */ async function play(opts: Parameters[1]): Promise { const s = await open($('v'), opts); @@ -253,26 +243,16 @@ async function startByAddress(a: string, relays: string[], hint: string | null): const trackers = [...new Set([...urlTrackers, ...heard.trackers])].slice(0, MAX_TRACKERS); // Likewise its own STUN servers, then those seeders named (NFX-10 §1). const stun = [...iceServers, ...heard.stunServers.map((urls) => ({ urls }))].slice(0, MAX_ICE_SERVERS); - // Each beacon-named origin with the renditions it serves; the hint serves them all. + // Each beacon-named origin with the renditions beacons claim it serves (a hint the + // library only orders by); the page's own hint makes no claim. const origins = fresh.map((o): string | OriginSource => heard.originSources.find((s) => s.url === o) ?? o); - const preferred = urlRenditions === undefined ? fromAvailability(heard.availability) : undefined; - const opts = { ...common, iceServers: stun, manifestJson: m.event, origins, signaling: trackerSignaling(trackers) }; - try { - await play({ ...opts, renditions: urlRenditions ?? preferred }); - return; - } catch (e) { - log(e instanceof Error ? e.message : String(e)); - // A beacon named a rendition this video lacks: play without the preference. Any - // other refusal of the page's own options holds for every origin: give up. - if (e instanceof RangeError && preferred !== undefined) log('again, without the renditions beacons named'); - else if (e instanceof RangeError || e instanceof TypeError) throw e; - else continue; - } try { - await play(opts); + await play({ ...common, iceServers: stun, manifestJson: m.event, origins, renditions: urlRenditions, signaling: trackerSignaling(trackers) }); return; } catch (e) { log(e instanceof Error ? e.message : String(e)); + // A refusal of the page's own options holds for every origin: give up. + if (e instanceof RangeError || e instanceof TypeError) throw e; continue; } } diff --git a/web/player/src/probe.ts b/web/player/src/probe.ts new file mode 100644 index 0000000..0e59ed4 --- /dev/null +++ b/web/player/src/probe.ts @@ -0,0 +1,201 @@ +/** + * Which renditions an origin holds, by asking it (nfx #85). Beacons come from any key + * (NFX-03), so what one says an origin serves is a hint; the origin itself is + * authoritative, and answers `404` for a file it does not hold (NFX-05 §6, NFX-03 §4). + * After the hash list verifies, `open()` (./lib.ts) asks the origin for each allowed + * rendition's init and offers only the renditions it answers for. + * + * This file is plain logic over an injected `fetch`-shaped probe, so unit.ts runs it without + * a browser: the runner (bounded in parallel and in time, aborted with the session), the + * byte-capped GET, and reading a verified media playlist for its init's URI. + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +import { probeOrder, sessionRenditions } from './levels'; + +/** How long all of one origin's probes may take together, in milliseconds. */ +export const PROBE_TIMEOUT_MS = 8_000; +/** How many renditions are probed at once on one origin. */ +export const PROBE_CONCURRENCY = 6; +/** + * Most body bytes a probe reads. An init (NFX-05 §1) is a few KiB, so a whole one is read, + * and lands in the HTTP cache for hls.js (content addresses are immutable, NFX-05 §6.1); + * a larger or endless body is cut off here. + */ +export const PROBE_MAX_BYTES = 64 * 1024; + +/** What one probe found: the origin holds the rendition (`status` 200 or 206), or why not. */ +export type ProbeOutcome = { held: true; status: number } | { held: false; why: string }; + +/** What a probe of one rendition resolves to: the HTTP status its init answered with. */ +export type Probe = (id: string, signal: AbortSignal) => Promise; + +export interface ProbeOptions { + /** All probes together, in milliseconds. Default `PROBE_TIMEOUT_MS`. */ + timeoutMs?: number; + /** Probes at once. Default `PROBE_CONCURRENCY`. */ + concurrency?: number; + /** Aborting it rejects `probeAll` with its reason and cancels every probe. */ + signal?: AbortSignal; +} + +function message(e: unknown): string { + return e instanceof Error ? e.message : String(e); +} + +function abortReason(signal: AbortSignal): unknown { + return signal.reason ?? new DOMException('aborted', 'AbortError'); +} + +/** + * Run `probe` for each of `ids`, in that order, at most `concurrency` at once, all under one + * deadline. Resolves with every id's outcome, in `ids`' order: held when the init answered + * 200 or 206; otherwise why not (another status, a failure, no answer by the deadline, or + * not reached by then). A probe still running at the deadline is aborted and counted as not + * held: a level is offered only on an answer. Rejects only when `signal` aborts. + */ +export async function probeAll(ids: readonly string[], probe: Probe, o: ProbeOptions = {}): Promise> { + const timeoutMs = o.timeoutMs ?? PROBE_TIMEOUT_MS; + const asked = o.concurrency ?? PROBE_CONCURRENCY; + const concurrency = Number.isFinite(asked) ? Math.max(1, Math.floor(asked)) : PROBE_CONCURRENCY; + if (o.signal?.aborted) throw abortReason(o.signal); + const ctl = new AbortController(); + // Every probe races this, so the deadline holds even for a probe that ignores its signal. + const stopped = new Promise((_, reject) => { + ctl.signal.addEventListener('abort', () => reject(abortReason(ctl.signal)), { once: true }); + }); + stopped.catch(() => {}); + let timedOut = false; + const onAbort = (): void => ctl.abort(o.signal ? abortReason(o.signal) : undefined); + o.signal?.addEventListener('abort', onAbort, { once: true }); + const timer = setTimeout(() => { + timedOut = true; + ctl.abort(new DOMException('probe timed out', 'TimeoutError')); + }, timeoutMs); + const out = new Map(); + let next = 0; + const worker = async (): Promise => { + while (next < ids.length && !ctl.signal.aborted) { + const id = ids[next++]!; + try { + const running = probe(id, ctl.signal); + // Settled later, after an abort, it must not surface as an unhandled rejection. + running.catch(() => {}); + const status = await Promise.race([running, stopped]); + out.set(id, status === 200 || status === 206 ? { held: true, status } : { held: false, why: `HTTP ${status}` }); + } catch (e) { + const why = !ctl.signal.aborted ? message(e) : timedOut ? `no answer within ${timeoutMs} ms` : 'aborted'; + out.set(id, { held: false, why }); + } + } + }; + try { + await Promise.all(Array.from({ length: Math.min(concurrency, ids.length) }, worker)); + } finally { + clearTimeout(timer); + o.signal?.removeEventListener('abort', onAbort); + // Nothing should still be running; anything that is, is cut loose. + ctl.abort(); + } + if (o.signal?.aborted) throw abortReason(o.signal); + return new Map(ids.map((id): [string, ProbeOutcome] => [id, out.get(id) ?? { held: false, why: `not probed within ${timeoutMs} ms` }])); +} + +/** + * GET `url` and resolve with its status, having read at most `maxBytes` of the body (and + * none of an error's). No `Range` header: it is not CORS-safelisted in every browser, and a + * preflight would cost a round trip and depend on the origin answering `OPTIONS`, which + * NFX-05 §6 does not require. `cache: 'no-cache'`: the origin is asked, never only the + * HTTP cache, so a copy cached in an earlier session cannot vouch for an origin that no + * longer holds the file (the answer is cached again, for hls.js). Rejects on a network + * failure, or when `signal` aborts. + */ +export async function probeStatus( + fetchFn: (url: string, init: RequestInit) => Promise, + url: string, + signal: AbortSignal, + maxBytes: number = PROBE_MAX_BYTES, +): Promise { + const res = await fetchFn(url, { signal, credentials: 'omit', cache: 'no-cache' }); + const body = res.body; + if (body === null) return res.status; + const reader = body.getReader(); + if (!res.ok) { + void reader.cancel().catch(() => {}); + return res.status; + } + let read = 0; + for (;;) { + const { done, value } = await reader.read(); + if (done) return res.status; + read += value.byteLength; + if (read >= maxBytes) { + void reader.cancel().catch(() => {}); + return res.status; + } + } +} + +/** `#EXT-X-MAP` with its `URI` (NFX-05 §3: a tag has at most one `URI` attribute). */ +const MAP_URI = /#EXT-X-MAP:[^\r\n]*?\bURI="([^"\r\n]*)"/; + +/** + * The URL of a media playlist's init (its `#EXT-X-MAP` URI), resolved against the + * playlist's own URL; without one, its first media segment's, which is then what tells + * whether the origin holds the rendition. Null when it names neither. `text` must be a + * playlist that verified against the hash list (NFX-05 §3 makes its URIs content names). + */ +export function initUrl(text: string, playlistUrl: string): string | null { + const map = MAP_URI.exec(text)?.[1]; + const first = text + .split(/\r\n|\r|\n/) + .map((l) => l.trim()) + .find((l) => l !== '' && !l.startsWith('#')); + const uri = map ?? first; + if (uri === undefined || uri === '') return null; + try { + return new URL(uri, playlistUrl).href; + } catch { + return null; + } +} + +/** + * The outcomes in a short line for a log or an `origin` event: at most `max` renditions, + * each id cut to 64 characters (ids come from the creator, statuses from the origin). + */ +export function describe(outcomes: ReadonlyMap, max = 8): string { + const parts = [...outcomes].map(([id, o]) => `${id.length > 64 ? `${id.slice(0, 64)}…` : id}: ${o.held ? `HTTP ${o.status}` : o.why.slice(0, 120)}`); + return parts.slice(0, max).join('; ') + (parts.length > max ? `; and ${parts.length - max} more` : ''); +} + +/** What `selectRenditions` found on one origin. */ +export interface Selection { + /** The session's renditions there: allowed, and held by the origin's own answers, in the caller's order. */ + renditions: string[]; + /** Each rendition asked for, in the order asked, with its outcome. */ + outcomes: Map; + /** Allowed renditions past `MAX_RENDITIONS`, not asked for (so not offered). */ + unasked: number; +} + +/** + * One origin's part of `open()`'s rule (`OpenCommon`): of the hash list's renditions + * (`ids`), the allowed ones (`wanted`, or all) that the origin holds, by asking it (`probe` + * per rendition, run by `probeAll`). `hint` (what beacons claim it serves) orders the + * asking and nothing else. Throws a `RangeError`, before any probe, for an id of `wanted` + * the hash list lacks; rejects when `o.signal` aborts. + */ +export async function selectRenditions( + ids: readonly string[], + wanted: readonly string[] | null, + hint: readonly string[] | null, + probe: Probe, + o: ProbeOptions = {}, +): Promise { + const allowed = sessionRenditions(ids, wanted, null); + const asked = probeOrder(allowed, hint); + const outcomes = await probeAll(asked, probe, o); + const held = asked.filter((id) => outcomes.get(id)?.held === true); + return { renditions: sessionRenditions(ids, wanted, held), outcomes, unasked: allowed.length - asked.length }; +} diff --git a/web/player/src/resolve.ts b/web/player/src/resolve.ts index 7a599ac..1bb9074 100644 --- a/web/player/src/resolve.ts +++ b/web/player/src/resolve.ts @@ -72,7 +72,7 @@ export async function resolveManifest(a: string, relays: string[], timeoutMs = 8 /** * Watch seeders' live beacons for `a` (in `namespace`) and report what they name together - * (`Endpoints`) each time that changes: `https` origins (with the renditions each serves), + * (`Endpoints`) each time that changes: `https` origins (with the renditions claimed for each), * `webrtc` trackers, `stun` servers, and which renditions the seeders hold, from the newest * unexpired beacon of each seeder (./endpoints.ts). Beacons are ephemeral, so a * fresh subscription hears each seeder at its next republish (≤ TTL/2). Returns `stop`. diff --git a/web/player/src/verify.ts b/web/player/src/verify.ts index 3ee9be0..3679e94 100644 --- a/web/player/src/verify.ts +++ b/web/player/src/verify.ts @@ -36,10 +36,10 @@ export function loadWasm(url?: string | URL): Promise { return ready; } -/** `res`'s body, refused past `max` bytes without reading the rest. */ -async function cappedBody(res: Response, max: number): Promise { +/** `res`'s body, refused past `max` bytes without reading the rest (`what` names it in the error). */ +async function cappedBody(res: Response, max: number, what: string): Promise { const declared = Number(res.headers.get('content-length')); - if (Number.isFinite(declared) && declared > max) throw new Error(`hash list: over ${max} bytes`); + if (Number.isFinite(declared) && declared > max) throw new Error(`${what}: over ${max} bytes`); if (!res.body) return new Uint8Array(await res.arrayBuffer()); const reader = res.body.getReader(); const parts: Uint8Array[] = []; @@ -50,7 +50,7 @@ async function cappedBody(res: Response, max: number): Promise { size += value.byteLength; if (size > max) { await reader.cancel(); - throw new Error(`hash list: over ${max} bytes`); + throw new Error(`${what}: over ${max} bytes`); } parts.push(value); } @@ -91,7 +91,7 @@ export class Anchor { await loadWasm(); const res = await fetch(`${origin}/${root}`, { signal, credentials: 'omit' }); if (!res.ok) throw new Error(`hash list: HTTP ${res.status}`); - const bytes = await cappedBody(res, MAX_HASH_LIST_BYTES); + const bytes = await cappedBody(res, MAX_HASH_LIST_BYTES, 'hash list'); const list = manifest ? new VerifiedHashList(bytes, root, manifest.video, manifest.segs) : VerifiedHashList.fromRoot(bytes, root); @@ -125,7 +125,22 @@ export class Anchor { } /** The verified sha256 of `bytes` fetched from `url`; throws on any mismatch. */ - check(url: string, bytes: ArrayBuffer): string { - return this.list.check(new URL(url, location.href).pathname, new Uint8Array(bytes)); + check(url: string, bytes: ArrayBuffer | Uint8Array): string { + return this.list.check(new URL(url, location.href).pathname, bytes instanceof Uint8Array ? bytes : new Uint8Array(bytes)); + } + + /** + * The file at `url` (one the hash list names), fetched and verified: at most `max` bytes + * are read. Throws on an HTTP error, a longer body, any mismatch, or when `signal` aborts. + */ + async fetchChecked(url: string, max: number, signal: AbortSignal): Promise { + const res = await fetch(url, { signal, credentials: 'omit' }); + if (!res.ok) { + void res.body?.cancel().catch(() => {}); + throw new Error(`HTTP ${res.status}`); + } + const bytes = await cappedBody(res, max, 'file'); + this.check(url, bytes); + return bytes; } } diff --git a/web/player/unit.ts b/web/player/unit.ts index 5a7e105..e61eebe 100644 --- a/web/player/unit.ts +++ b/web/player/unit.ts @@ -1,8 +1,10 @@ /** * Unit checks for the player's pure parts (`npm run unit`, after `npm run wasm`): the URL * sanitisers, the beacon endpoint book behind `watchEndpoints` (rendition availability and - * what each origin serves included), `open()`'s option checks and the levels it offers - * (`renditions`, origin sources, the master playlist cut), and the Nostr signaler's + * what beacons claim each origin serves included), `open()`'s option checks and the levels it + * offers (`renditions`, origin hints, asking the origin what it holds: the probe runner, its + * bounds, timeouts and aborts, the byte-capped GET, a hostile hint ignored; the master + * playlist cut), and the Nostr signaler's * bookkeeping (presence, glare, freshness, bounds). The spec's signed beacon vectors * (`beacon.json`, `beacon-renditions.json`) run through nfx-wasm into the book, so the shape * `verifyBeacon` returns is the shape the book reads, and the hash list vector through @@ -38,9 +40,22 @@ import { cutMaster, hashListLevels, mayServe, + originOrder, + probeOrder, sessionRenditions, } from './src/levels'; import { MAX_RELAYS, relayUrl } from './src/mesh'; +import { + PROBE_CONCURRENCY, + PROBE_MAX_BYTES, + PROBE_TIMEOUT_MS, + type Probe, + describe as describeProbes, + initUrl, + probeAll, + probeStatus, + selectRenditions, +} from './src/probe'; import { LIMITS, OfferBackoff, @@ -424,18 +439,36 @@ refuses('a null origin', () => checkOriginSources([null as never]), TypeError, / refuses('origins not a list', () => checkOriginSources('https://a.example' as never), TypeError, /must be a list/); refuses('too many origins', () => checkOriginSources(Array.from({ length: 17 }, (_, i) => `https://o${i}.example`)), TypeError, /at most 16/); -eq('mayServe: an origin serving all', mayServe(['720p'], null), true); +// Hints (what beacons claim an origin serves): they order, never remove (nfx #85). +eq('mayServe: no claim', mayServe(['720p'], null), true); eq('mayServe: overlap', mayServe(['720p', '360p'], ['360p']), true); eq('mayServe: disjoint', mayServe(['720p'], ['360p']), false); -eq('mayServe: serves none', mayServe(null, []), false); +eq('mayServe: a claim of none', mayServe(null, []), false); eq('mayServe: some, every rendition asked for', mayServe(null, ['360p']), true); - +{ + const o = (url: string, renditions: string[] | null) => ({ url, renditions }); + const list = [o('a', ['360p']), o('b', null), o('c', ['720p']), o('d', [])]; + eq('originOrder: a hint naming none of the allowed goes last, kept', originOrder(list, ['720p']).map((x) => x.url), ['b', 'c', 'a', 'd']); + eq('originOrder: every rendition allowed', originOrder(list, null).map((x) => x.url), ['a', 'b', 'c', 'd']); + eq('originOrder: nothing removed', originOrder(list, ['4320p']).length, 4); + eq('originOrder: stable within each group', originOrder([o('x', ['1']), o('y', ['2']), o('z', ['1'])], ['2']).map((x) => x.url), ['y', 'x', 'z']); +} const ladder = ['1080p', '720p', '360p']; -eq('every rendition, every one served', sessionRenditions(ladder, null, null), ladder); +eq('probeOrder: no hint', probeOrder(ladder, null), ladder); +eq('probeOrder: named first, the rest kept', probeOrder(ladder, ['360p']), ['360p', '1080p', '720p']); +eq('probeOrder: a claim of none removes nothing', probeOrder(ladder, []), ladder); +eq('probeOrder: ids the list lacks are ignored', probeOrder(['720p'], ['4320p', '720p']), ['720p']); +{ + const many = Array.from({ length: 40 }, (_, i) => `r${i}`); + eq('probeOrder: at most MAX_RENDITIONS', probeOrder(many, null).length, MAX_RENDITIONS); + eq('probeOrder: the hinted ones survive the cap, in the caller order', probeOrder(many, ['r39', 'r38']).slice(0, 3), ['r38', 'r39', 'r0']); +} + +eq('every rendition, every one held', sessionRenditions(ladder, null, null), ladder); eq('the caller order', sessionRenditions(ladder, ['360p', '1080p'], null), ['360p', '1080p']); -eq('cut to what the origin serves', sessionRenditions(ladder, ['360p', '1080p'], ['1080p', '720p']), ['1080p']); -eq('an origin naming ids the list lacks', sessionRenditions(ladder, null, ['720p', '4320p']), ['720p']); -eq('none of the allowed served', sessionRenditions(ladder, ['720p'], ['360p']), []); +eq('cut to what the origin holds', sessionRenditions(ladder, ['360p', '1080p'], ['1080p', '720p']), ['1080p']); +eq('held ids the list lacks', sessionRenditions(ladder, null, ['720p', '4320p']), ['720p']); +eq('none of the allowed held', sessionRenditions(ladder, ['720p'], ['360p']), []); refuses('an id the hash list lacks', () => sessionRenditions(ladder, ['360p', '2160p'], null), RangeError, /"2160p" is not a rendition of this video \(it has "1080p", "720p", "360p"\)/); refuses('checked even when the origin serves none', () => sessionRenditions(ladder, ['2160p'], []), RangeError, /"2160p"/); @@ -468,12 +501,14 @@ eq('content hash: not a URL', contentHash('http://[::1'), null); }; eq('hash list levels', hashListLevels(JSON.stringify(list)), { renditions: [ - { id: '1080p', playlist: H(1) }, - { id: '720p', playlist: H(2) }, - { id: '360p', playlist: H(3) }, + { id: '1080p', playlist: H(1), playlistSize: 1 }, + { id: '720p', playlist: H(2), playlistSize: 1 }, + { id: '360p', playlist: H(3), playlistSize: 1 }, ], master: H(9), }); + const sized = { ...list, files: list.files.map((f) => (f.name === 'r720.m3u8' ? { ...f, size: -1 } : f.name === 'r360.m3u8' ? { ...f, size: '7' } : f)) }; + eq('an unusable size is null', hashListLevels(JSON.stringify(sized)).renditions.map((r) => r.playlistSize), [1, null, null]); } { const base = `https://o.example/nfx/${'a'.repeat(64)}/master.m3u8`; @@ -505,6 +540,225 @@ eq('content hash: not a URL', contentHash('http://[::1'), null); eq('I-frame variants untouched', cutMaster(iframes, base, new Set([H(2)])).text.endsWith(`URI="${H(7)}.m3u8"\n`), true); } +// ---- Asking the origin what it holds (nfx #85): the probe runner, the GET, the init URI --- +const unhandled: unknown[] = []; +process.on('unhandledRejection', (e) => void unhandled.push(e)); +const sleep = (ms: number): Promise => new Promise((r) => setTimeout(r, ms)); +/** A probe answering from `table` (id → status, or an Error to throw), after `ms`. */ +const answers = (table: Record, ms = 1, log: string[] = []): Probe => async (id) => { + log.push(id); + await sleep(ms); + const a = table[id]; + if (a instanceof Error) throw a; + return a ?? 404; +}; +const outcomes = (m: Map) => Object.fromEntries(m); +eq('probe defaults', [PROBE_TIMEOUT_MS, PROBE_CONCURRENCY, PROBE_MAX_BYTES], [8000, 6, 65536]); +{ + const m = await probeAll(['720p', '360p', '1080p', '240p', 'x'], answers({ '720p': 200, '360p': 206, '1080p': 404, '240p': 500, x: new Error('network down') })); + eq('probeAll: 200 and 206 hold, nothing else does', outcomes(m), { + '720p': { held: true, status: 200 }, + '360p': { held: true, status: 206 }, + '1080p': { held: false, why: 'HTTP 404' }, + '240p': { held: false, why: 'HTTP 500' }, + x: { held: false, why: 'network down' }, + }); + eq('probeAll: in the order asked', [...m.keys()], ['720p', '360p', '1080p', '240p', 'x']); + eq('probeAll: nothing to ask', (await probeAll([], answers({}))).size, 0); + eq('probeAll: a concurrency that is no number', outcomes(await probeAll(['a'], answers({ a: 200 }), { concurrency: Number.NaN })), { a: { held: true, status: 200 } }); + eq('probeAll: a deadline of 0 offers nothing', [...(await probeAll(['a'], answers({ a: 200 }, 5), { timeoutMs: 0 })).values()].some((o) => o.held), false); +} +{ + // At most `concurrency` at once, started in the order given. + let now = 0; + let most = 0; + const started: string[] = []; + const probe: Probe = async (id) => { + started.push(id); + now += 1; + most = Math.max(most, now); + await sleep(5); + now -= 1; + return 200; + }; + const ids = Array.from({ length: 9 }, (_, i) => `r${i}`); + const m = await probeAll(ids, probe, { concurrency: 2 }); + eq('probeAll: bounded in parallel', [most, started, [...m.values()].every((o) => o.held)], [2, ids, true]); + most = 0; + await probeAll(ids, probe); + eq('probeAll: 6 at once by default', most, 6); +} +{ + // One deadline for all: a probe that never answers, and ignores its signal, is cut off; + // the ids never reached are not held either, and probeAll still resolves on time. + const seen: AbortSignal[] = []; + const hang: Probe = (id, signal) => { + seen.push(signal); + return id === 'fast' ? Promise.resolve(200) : new Promise(() => {}); + }; + const t0 = Date.now(); + const m = await probeAll(['fast', 'stuck', 'never'], hang, { timeoutMs: 60, concurrency: 1 }); + const took = Date.now() - t0; + eq('probeAll: the deadline', outcomes(m), { + fast: { held: true, status: 200 }, + stuck: { held: false, why: 'no answer within 60 ms' }, + never: { held: false, why: 'not probed within 60 ms' }, + }); + eq('probeAll: resolves at the deadline', took >= 55 && took < 2000, true); + eq('probeAll: the probes are told to stop', seen.every((s) => s.aborted), true); + // A probe that rejects after its deadline is not an unhandled rejection. + const late: Probe = (_, signal) => new Promise((_, reject) => signal.addEventListener('abort', () => setTimeout(() => reject(new Error('late')), 5))); + eq('probeAll: a late rejection', outcomes(await probeAll(['a'], late, { timeoutMs: 20 })), { a: { held: false, why: 'no answer within 20 ms' } }); +} +{ + // The caller's signal: aborting rejects with its reason and stops every probe. + const ctl = new AbortController(); + const seen: AbortSignal[] = []; + const p = probeAll(['a', 'b'], (_, signal) => { + seen.push(signal); + return new Promise(() => {}); + }, { signal: ctl.signal, timeoutMs: 60_000 }); + await sleep(5); + ctl.abort(new Error('viewer left')); + let got = 'resolved'; + try { + await p; + } catch (e) { + got = (e as Error).message; + } + eq('probeAll: aborted', [got, seen.length, seen.every((s) => s.aborted)], ['viewer left', 2, true]); + let called = 0; + let pre = 'resolved'; + try { + await probeAll(['a'], async () => (called += 1), { signal: AbortSignal.abort(new Error('already')) }); + } catch (e) { + pre = (e as Error).message; + } + eq('probeAll: aborted before it starts', [pre, called], ['already', 0]); + // Its timer and listeners go with it: no timeout left behind. + const timers = (): number => process.getActiveResourcesInfo().filter((r) => r === 'Timeout').length; + const before = timers(); + await probeAll(['a', 'b'], answers({ a: 200, b: 200 }), { timeoutMs: 60_000, signal: new AbortController().signal }); + await sleep(5); + eq('probeAll: leaves no timer', timers(), before); +} +{ + // The GET: no Range header, credentials omitted, at most PROBE_MAX_BYTES of the body read, + // none of an error's; an origin without Range support answers 200 with the whole file. + const calls: { url: string; init: RequestInit }[] = []; + let pulled = 0; + let cancelled = 0; + const stream = (chunk: number, total: number): ReadableStream => { + let sent = 0; + return new ReadableStream({ + pull(c) { + pulled += 1; + if (sent >= total) return c.close(); + const n = Math.min(chunk, total - sent); + sent += n; + c.enqueue(new Uint8Array(n)); + }, + cancel() { + cancelled += 1; + }, + }); + }; + const serve = (status: number, body: () => ReadableStream | null) => async (url: string, init: RequestInit) => { + calls.push({ url, init }); + return new Response(body(), { status }); + }; + const sig = new AbortController().signal; + eq('probeStatus: a small init, read whole', await probeStatus(serve(200, () => stream(1000, 3000)), 'https://o.example/r/i.mp4', sig), 200); + eq('probeStatus: read to the end, nothing cancelled', [cancelled, calls[0]?.init.headers, calls[0]?.init.credentials, calls[0]?.init.cache, calls[0]?.init.signal === sig], [0, undefined, 'omit', 'no-cache', true]); + pulled = 0; + eq('probeStatus: an endless 200 (no Range)', await probeStatus(serve(200, () => stream(16 * 1024, Number.MAX_SAFE_INTEGER)), 'https://o.example/r/i.mp4', sig), 200); + await sleep(1); + eq('probeStatus: cut at the cap', [pulled <= PROBE_MAX_BYTES / (16 * 1024) + 2, cancelled], [true, 1]); + pulled = 0; + eq('probeStatus: a 404', await probeStatus(serve(404, () => stream(1000, 1_000_000)), 'https://o.example/r/i.mp4', sig), 404); + await sleep(1); + eq('probeStatus: an error body is not read', [pulled <= 1, cancelled], [true, 2]); + eq('probeStatus: a 206', await probeStatus(serve(206, () => stream(10, 10)), 'https://o.example/r/i.mp4', sig), 206); + eq('probeStatus: no body', await probeStatus(serve(200, () => null), 'https://o.example/r/i.mp4', sig), 200); + eq('probeStatus: a smaller cap', await probeStatus(serve(200, () => stream(100, 1_000_000)), 'https://o.example/r/i.mp4', sig, 250), 200); + let failed = ''; + try { + await probeStatus(async () => { + throw new TypeError('Failed to fetch'); + }, 'https://o.example/r/i.mp4', sig); + } catch (e) { + failed = (e as Error).message; + } + eq('probeStatus: a network failure rejects', failed, 'Failed to fetch'); + // Aborted mid-body: the read rejects, so probeAll counts it as no answer. + const ctl = new AbortController(); + const slow = async (_: string, init: RequestInit): Promise => { + const body = new ReadableStream({ + start(c) { + c.enqueue(new Uint8Array(10)); + init.signal?.addEventListener('abort', () => c.error(new DOMException('aborted', 'AbortError'))); + }, + }); + return new Response(body, { status: 200 }); + }; + const pending = probeStatus(slow, 'https://o.example/r/i.mp4', ctl.signal); + await sleep(5); + ctl.abort(); + let aborted = ''; + try { + await pending; + } catch (e) { + aborted = (e as Error).name; + } + eq('probeStatus: aborted mid-body', aborted, 'AbortError'); +} +{ + const pl = 'https://o.example/nfx/root/aaaa.m3u8'; + const seg = (n: number) => `${String(n).repeat(64)}.m4s`; + eq('initUrl: EXT-X-MAP, resolved against the playlist', initUrl(`#EXTM3U\n#EXT-X-MAP:URI="${'b'.repeat(64)}.mp4"\n#EXTINF:2,\n${seg(1)}\n`, pl), `https://o.example/nfx/root/${'b'.repeat(64)}.mp4`); + eq('initUrl: other attributes first, CRLF', initUrl(`#EXTM3U\r\n#EXT-X-MAP:BYTERANGE="9@0",URI="init.mp4"\r\n`, pl), 'https://o.example/nfx/root/init.mp4'); + eq('initUrl: no map, the first segment', initUrl(`#EXTM3U\n# a comment\n\n#EXTINF:2,\n ${seg(2)} \n#EXTINF:2,\n${seg(3)}\n`, pl), `https://o.example/nfx/root/${seg(2)}`); + eq('initUrl: neither', initUrl('#EXTM3U\n#EXT-X-ENDLIST\n', pl), null); + eq('initUrl: an empty URI', initUrl('#EXTM3U\n#EXT-X-MAP:URI=""\n', pl), null); +} +{ + const many = new Map(Array.from({ length: 10 }, (_, i): [string, { held: false; why: string }] => [i === 0 ? 'x'.repeat(100) : `r${i}`, { held: false, why: 'HTTP 404' }])); + const line = describeProbes(many); + eq('describe: capped', [line.startsWith(`${'x'.repeat(64)}…: HTTP 404; r1: HTTP 404`), line.endsWith('; and 2 more')], [true, true]); + eq('describe: held', describeProbes(new Map([['720p', { held: true, status: 200 }]])), '720p: HTTP 200'); +} +{ + // One origin's part of open(): a hint orders the asking and nothing else. + const ids = ['720p', '360p']; + const asked: string[] = []; + // A hostile beacon narrows an origin that holds both to 360p: both are still offered. + const both = await selectRenditions(ids, null, ['360p'], answers({ '720p': 200, '360p': 200 }, 1, asked)); + eq('a hostile narrowing hint is ignored', [both.renditions, asked, [...both.outcomes.keys()], both.unasked], [['720p', '360p'], ['360p', '720p'], ['360p', '720p'], 0]); + // The viewer wants 720p, which the hint leaves out: the origin is still asked, and used. + asked.length = 0; + const wanted = await selectRenditions(ids, ['720p'], ['360p'], answers({ '720p': 200, '360p': 200 }, 1, asked)); + eq('a hint cannot take away a wanted rendition', [wanted.renditions, asked], [['720p'], ['720p']]); + // A hint widening what the origin holds adds nothing: 720p's init answers 404. + const lying = await selectRenditions(ids, null, ['720p', '360p'], answers({ '720p': 404, '360p': 200 })); + eq('a hint cannot add a level', [lying.renditions, lying.outcomes.get('720p')], [['360p'], { held: false, why: 'HTTP 404' }]); + eq('a claim of none: still asked', (await selectRenditions(ids, null, [], answers({ '720p': 200, '360p': 200 }))).renditions, ['720p', '360p']); + eq('it holds none', (await selectRenditions(ids, ['720p'], null, answers({}))).renditions, []); + eq('the caller order, not the asking order', (await selectRenditions(ids, ['360p', '720p'], ['720p'], answers({ '720p': 200, '360p': 200 }))).renditions, ['360p', '720p']); + let range = ''; + asked.length = 0; + try { + await selectRenditions(ids, ['4320p'], null, answers({}, 1, asked)); + } catch (e) { + range = e instanceof RangeError ? 'RangeError' : String(e); + } + eq('an id the hash list lacks: refused before any probe', [range, asked.length], ['RangeError', 0]); + const forty = Array.from({ length: 40 }, (_, i) => `r${i}`); + const capped = await selectRenditions(forty, null, ['r39'], answers(Object.fromEntries(forty.map((id) => [id, 200])))); + eq('at most 32 asked; the rest are not offered', [capped.outcomes.size, capped.unasked, capped.renditions.length, capped.renditions.includes('r39'), capped.renditions.includes('r38')], [32, 8, 32, true, false]); +} +await sleep(20); +eq('no unhandled rejection from any probe', unhandled.map(String), []); + // ---- Nostr signaling: the option, and the signaler's bookkeeping -------------------------- const relayCases: [string, string | null][] = [ ['wss://relay.example', 'wss://relay.example'], @@ -662,7 +916,7 @@ throws('vector beacon past its expiration', () => verifyBeacon(JSON.stringify(ve // The hash list vector through nfx-wasm: what open() reads of it, and its master cut. const v = JSON.parse(readFileSync(join(here, '../../spec/test-vectors/hashlist.json'), 'utf8')) as { rendered_bytes_sha256: string; - hashlist: { files: { name: string; sha256: string }[] }; + hashlist: { files: { name: string; sha256: string; size: number }[] }; playlists: Record; }; const bytes = new TextEncoder().encode(`${JSON.stringify(v.hashlist, null, 2)}\n`); @@ -670,7 +924,14 @@ throws('vector beacon past its expiration', () => verifyBeacon(JSON.stringify(ve const list = VerifiedHashList.fromRoot(bytes, v.rendered_bytes_sha256); const levels = hashListLevels(list.toJSON()); const sha = (name: string): string | undefined => v.hashlist.files.find((f) => f.name === name)?.sha256; - eq('hash list vector levels', levels, { renditions: [{ id: '720p', playlist: sha('r720.m3u8') }], master: sha('master.m3u8') }); + const size = (name: string): number | undefined => v.hashlist.files.find((f) => f.name === name)?.size; + eq('hash list vector levels', levels, { renditions: [{ id: '720p', playlist: sha('r720.m3u8'), playlistSize: size('r720.m3u8') }], master: sha('master.m3u8') }); + // The vector's own playlist names its init by content name (NFX-05 §3). + const r720 = v.playlists['r720.m3u8'] ?? ''; + const under = `https://seed.example/nfx/${v.rendered_bytes_sha256}/`; + eq('vector playlist: its init', initUrl(r720, `${under}${sha('r720.m3u8')}.m3u8`), `${under}${sha('init-720.mp4')}.mp4`); + // And the playlist verifies under that URL, so open() may read it. + eq('vector playlist verifies', list.check(`/nfx/${v.rendered_bytes_sha256}/${sha('r720.m3u8')}.m3u8`, new TextEncoder().encode(r720)), sha('r720.m3u8')); const master = v.playlists['master.m3u8'] ?? ''; const base = `https://seed.example/nfx/${v.rendered_bytes_sha256}/master.m3u8`; eq('vector master, its rendition kept', cutMaster(master, base, new Set(levels.renditions.map((r) => r.playlist))), { text: master, kept: 1 });