Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions apps/sandbox/templates/moq-relay-interop/index.html
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@
<div id="player-col">
<simple-moq-video id="player" playsinline></simple-moq-video>
<div id="controls">
<label for="src-input">Source</label>
<input id="src-input" type="text" spellcheck="false" />
<button id="load-btn" type="button">Load</button>
<button id="play-btn" type="button">Play</button>
Expand Down
17 changes: 16 additions & 1 deletion packages/html/src/media/simple-moq-video/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ function isValidPreload(value: string | null): value is MoqMediaProps['preload']
* `playbackRate` (live-only), and `textTracks` (no text renderer yet).
*/
class SimpleMoqMediaImpl extends MoqMediaBase {
static readonly observedAttributes = ['autoplay', 'src', 'preload', 'target-latency', 'muted'];
static readonly observedAttributes = ['autoplay', 'src', 'preload', 'target-latency', 'adaptive-latency', 'muted'];
static shadowRootOptions: ShadowRootInit = { mode: 'open' };

readonly #canvas: HTMLCanvasElement;
Expand Down Expand Up @@ -106,6 +106,14 @@ class SimpleMoqMediaImpl extends MoqMediaBase {
const root = this.shadowRoot!;
this.#canvas = root.querySelector('canvas') ?? document.createElement('canvas');
if (!this.#canvas.isConnected) root.append(this.#canvas);
// A `<canvas>` defaults to 300×150 until something draws to it; native
// `<video>` reports 0×0 before the first frame. `#lastWidth`/`#lastHeight`
// start at 0 to match, so without this the first poll tick in
// `#connectEventBridge` sees the HTML default as a "change" and fires a
// `resize` exposing 300×150 as the stream's dimensions before the
// renderer has presented anything.
this.#canvas.width = 0;
this.#canvas.height = 0;
applyShadowStyles(root, [SHADOW_STYLE_SHEET]);
}

Expand Down Expand Up @@ -155,6 +163,13 @@ class SimpleMoqMediaImpl extends MoqMediaBase {
// false, so latency control silently stops).
const parsed = isNull(newValue) ? Number.NaN : Number(newValue);
this.targetLatency = Number.isFinite(parsed) && parsed > 0 ? parsed : undefined;
} else if (name === 'adaptive-latency') {
// Presence-based, like `autoplay`: this is a *request*, not a
// number, so it routes to its own slot rather than widening
// `targetLatency`, and an element that never carries the attribute
// never writes the slot at all — leaving the engine config's own
// default (off) in charge. `target-latency` still wins over it.
this.adaptiveLatency = !isNull(newValue);
} else if (name === 'muted') {
// `muted` is the *default* muted state per spec: removing the
// attribute must not unmute an element the user muted.
Expand Down
118 changes: 107 additions & 11 deletions packages/html/src/media/simple-moq-video/tests/index.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -81,20 +81,51 @@ describe('SimpleMoqVideo', () => {
// A declarative-shadow-DOM host already has a root at upgrade time; a bare
// `attachShadow` throws NotSupportedError there and bricks the element.
it('reuses an existing shadow root instead of attaching a second one', () => {
const tag = defineElement();
const host = document.createElement('div');
host.innerHTML = `<${tag}></${tag}>`;
const el = host.firstElementChild as SimpleMoqVideo;
// Simulate the upgrade order declarative shadow DOM produces.
const preAttached = document.createElement(tag) as SimpleMoqVideo;
// happy-dom neither parses declarative shadow DOM (`<template
// shadowrootmode>`) nor implements `customElements.upgrade()`, so there
// is no way to reproduce, through real parsing/upgrade timing, an
// element whose constructor observes a shadow root that already
// exists. Stand in for it: override `shadowRoot` on a throwaway
// subclass so the base constructor's `if (!this.shadowRoot)` guard
// sees a pre-existing root exactly as a declarative-shadow-DOM host
// would produce. Built before the spy below so its own `attachShadow`
// call isn't mistaken for one made by the element under test.
const existingRoot = document.createElement('div').attachShadow({ mode: 'open' });
const existingCanvas = document.createElement('canvas');
existingRoot.append(existingCanvas);

const attachShadowSpy = vi.spyOn(Element.prototype, 'attachShadow');

// Baseline: proves the spy actually observes real calls — a normal
// element with no pre-existing root attaches exactly one.
const baseline = document.createElement(defineElement()) as SimpleMoqVideo;
expect(attachShadowSpy).toHaveBeenCalledTimes(1);
attachShadowSpy.mockClear();

const tag = `test-simple-moq-video-${++tagCounter}`;
customElements.define(
tag,
class extends SimpleMoqVideo {
constructor() {
super({ createAudioContext: () => createFakeAudioContext('suspended') });
}
override get shadowRoot() {
return existingRoot;
}
}
);
const el = document.createElement(tag) as SimpleMoqVideo;

expect(el.shadowRoot?.querySelector('canvas')).not.toBeNull();
expect(preAttached.shadowRoot?.querySelectorAll('canvas')).toHaveLength(1);
expect(attachShadowSpy).not.toHaveBeenCalled();
expect(el.shadowRoot).toBe(existingRoot);
expect(existingRoot.querySelectorAll('canvas')).toHaveLength(1);
expect(existingRoot.querySelector('canvas')).toBe(existingCanvas);

// Never connected, so nothing disconnects them — release both engines
// explicitly rather than leaving live compositions behind.
// Neither ever connected, so nothing disconnects them — release both
// engines explicitly rather than leaving live compositions behind.
attachShadowSpy.mockRestore();
baseline.destroy();
el.destroy();
preAttached.destroy();
});

it('keeps the engine alive across disconnect when keep-alive is set', async () => {
Expand Down Expand Up @@ -195,6 +226,36 @@ describe('SimpleMoqVideo', () => {
expect(el.targetLatency).toBe(2);
});

// A *request* rather than a number, so it routes to its own slot
// instead of widening the numeric `targetLatency` — and an element that
// never carries the attribute never writes the slot, leaving the engine
// config's own default in charge.
it('routes the adaptive-latency attribute to its own engine slot', () => {
const el = createConnectedElement();

expect((el as any).engine.state.adaptiveLatencyEnabled.get()).toBeUndefined();
expect(el.targetLatency).toBeUndefined();

el.toggleAttribute('adaptive-latency', true);
expect((el as any).engine.state.adaptiveLatencyEnabled.get()).toBe(true);
expect(el.targetLatency).toBeUndefined();

el.toggleAttribute('adaptive-latency', false);
expect((el as any).engine.state.adaptiveLatencyEnabled.get()).toBe(false);
});

// Precedence has to survive the element boundary: setting both is the
// A/B's "pin it here and tell me what adaptation would have chosen".
it('keeps an explicit target-latency alongside an adaptive request', () => {
const el = createConnectedElement();

el.toggleAttribute('adaptive-latency', true);
el.setAttribute('target-latency', '1.5');

expect((el as any).engine.state.adaptiveLatencyEnabled.get()).toBe(true);
expect((el as any).engine.state.targetLatency.get()).toBe(1.5);
});

it('falls back to the empty preload for values outside the enumeration', () => {
const el = createConnectedElement();

Expand Down Expand Up @@ -297,6 +358,41 @@ describe('SimpleMoqVideo', () => {
expect(events).toEqual([]);
});

// A bare canvas defaults to 300×150 until something draws to it; native
// <video> reports 0×0 before the first frame, so the element must not
// expose (or announce) the canvas default as real stream dimensions.
it('does not fire resize before the renderer presents a frame', async () => {
vi.useFakeTimers();
const el = createConnectedElement();
const events = recordEvents(el, ['resize']);

expect(el.videoWidth).toBe(0);
expect(el.videoHeight).toBe(0);

await vi.advanceTimersByTimeAsync(500);

expect(events).toEqual([]);
expect(el.videoWidth).toBe(0);
expect(el.videoHeight).toBe(0);
});

it('fires resize once the renderer presents its first frame', async () => {
vi.useFakeTimers();
const el = createConnectedElement();
const events = recordEvents(el, ['resize']);

// Simulates what the moq video-renderer actor does on first decode:
// it sets the canvas's backing-store dimensions to the frame size.
const canvas = el.shadowRoot!.querySelector('canvas')!;
canvas.width = 640;
canvas.height = 360;
await vi.advanceTimersByTimeAsync(250);

expect(events).toEqual(['resize']);
expect(el.videoWidth).toBe(640);
expect(el.videoHeight).toBe(360);
});

it('dispatches volumechange when volume or muted change', () => {
const el = createConnectedElement();
const events = recordEvents(el, ['volumechange']);
Expand Down
33 changes: 29 additions & 4 deletions packages/spf/src/media/moq/codec-mapping.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
*
* The WebCodecs types come from the WebWorker lib — this stays DOM-free.
*/
import type { MoqAudioTrack, MoqVideoTrack } from './parse-catalog';
import { type MoqAudioTrack, type MoqVideoTrack, parseChannelConfig } from './parse-catalog';

/**
* Decoder config for a catalog video track, or `null` when the catalog
Expand Down Expand Up @@ -37,17 +37,34 @@ export function toVideoDecoderConfig(track: MoqVideoTrack): VideoDecoderConfig |
* there because the shared type requires a number, and configuring an
* AAC decoder at 48 kHz for a 44.1 kHz stream corrupts every frame rather
* than failing loudly. Opus is the one exception: RFC 6716 decoders always
* operate at 48 kHz, so its rate needs no declaration.
* operate at 48 kHz, so its rate needs no declaration. A declared rate is
* still validated — a malformed or hostile catalog could otherwise produce
* a non-integer, negative, or out-of-range `AudioDecoderConfig.sampleRate`
* (WebIDL `unsigned long`) — and rejected rather than handed to the decoder.
*
* Channel count follows the catalog's `channelConfig` (§5.2.21) the same
* way, via `parseChannelConfig`, rather than the projection's
* `AudioTrack.channels` substitute: an omitted `channelConfig` defaults to
* stereo (the common convention for simple streams), but a *declared*
* value this parser can't resolve — an unrecognized surround/object-audio
* layout string, or a malformed one — rejects the track instead of
* silently decoding it as stereo.
*/
export function toAudioDecoderConfig(track: MoqAudioTrack): AudioDecoderConfig | null {
const codec = track.codecs[0];
if (!codec) return null;

const sampleRate = track.moq.samplerate ?? (isOpus(codec) ? OPUS_SAMPLE_RATE : undefined);
if (sampleRate === undefined) return null;
if (sampleRate === undefined || !isValidSampleRate(sampleRate)) return null;

const numberOfChannels =
track.moq.channelConfig === undefined ? DEFAULT_CHANNELS : parseChannelConfig(track.moq.channelConfig);
if (numberOfChannels === undefined) return null;

const config: AudioDecoderConfig = {
codec,
sampleRate,
numberOfChannels: track.channels,
numberOfChannels,
};
if (track.moq.initData !== undefined) {
config.description = track.moq.initData.slice().buffer;
Expand All @@ -56,6 +73,14 @@ export function toAudioDecoderConfig(track: MoqAudioTrack): AudioDecoderConfig |
}

const OPUS_SAMPLE_RATE = 48_000;
const DEFAULT_CHANNELS = 2;

/** `AudioDecoderConfig.sampleRate` is a WebIDL `unsigned long`: a positive integer up to 2^32 - 1. */
const MAX_SAMPLE_RATE = 0xff_ff_ff_ff;

function isValidSampleRate(value: number): boolean {
return Number.isSafeInteger(value) && value > 0 && value <= MAX_SAMPLE_RATE;
}

function isOpus(codec: string): boolean {
return codec.toLowerCase().startsWith('opus');
Expand Down
39 changes: 30 additions & 9 deletions packages/spf/src/media/moq/parse-catalog.ts
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,13 @@ export interface MoqCatalogTrack {
renderGroup?: number;
altGroup?: number;
targetLatency?: number;
/**
* Publisher-declared minimum buffer in milliseconds (msf-01 §5.2.9) —
* the packaging/encode jitter a receiver has to absorb even on a perfect
* path. Read by the adaptive latency controller as one additive term of
* its margin; unused by the fixed-setpoint chain.
*/
jitter?: number;
buffers?: { target?: number; min?: number; max?: number };
maxGopDuration?: number;
maxGroupDuration?: number;
Expand Down Expand Up @@ -88,6 +95,8 @@ export interface MoqTrackFields {
renderGroup?: number;
altGroup?: number;
targetLatency?: number;
/** Publisher-declared minimum buffer in milliseconds (msf-01 §5.2.9). */
jitter?: number;
buffers?: { target?: number; min?: number; max?: number };
maxGopDuration?: number;
maxGroupDuration?: number;
Expand Down Expand Up @@ -223,6 +232,7 @@ function parseCatalogTrackFields(
fields.renderGroup = number(raw.renderGroup);
fields.altGroup = number(raw.altGroup);
fields.targetLatency = number(raw.targetLatency);
fields.jitter = number(raw.jitter);
fields.maxGopDuration = number(raw.maxGopDuration);
fields.maxGroupDuration = number(raw.maxGroupDuration);
fields.temporalId = number(raw.temporalId);
Expand Down Expand Up @@ -416,6 +426,7 @@ function moqFieldsOf(track: MoqCatalogTrack): MoqTrackFields {
if (track.renderGroup !== undefined) fields.renderGroup = track.renderGroup;
if (track.altGroup !== undefined) fields.altGroup = track.altGroup;
if (track.targetLatency !== undefined) fields.targetLatency = track.targetLatency;
if (track.jitter !== undefined) fields.jitter = track.jitter;
if (track.buffers !== undefined) fields.buffers = track.buffers;
if (track.maxGopDuration !== undefined) fields.maxGopDuration = track.maxGopDuration;
if (track.maxGroupDuration !== undefined) fields.maxGroupDuration = track.maxGroupDuration;
Expand All @@ -431,18 +442,28 @@ function trackUrl(sessionUri: string, track: MoqCatalogTrack): string {
return `${sessionUri}#msf:${encodeNamespaceName(track.namespace, track.name)}`;
}

/** Sane upper bound on a decoded channel count (WebCodecs practical ceiling). */
const MAX_CHANNELS = 255;

/**
* Channel count from a catalog `channelConfig` (§5.2.21). Accepts a plain
* count (`'2'`) and the dotted surround form (`'5.1'` → 6, `'7.1.4'` → 12);
* `parseInt` alone would read `'5.1'` as 5 and silently drop the LFE.
* Unrecognized values fall back to stereo.
* Channel count from a catalog `channelConfig` (§5.2.21), or `undefined` if
* it doesn't resolve to one. Accepts a plain count (`'2'`) and the dotted
* surround form (`'5.1'` → 6, `'7.1.4'` → 12); `parseInt` alone would read
* `'5.1'` as 5 and silently drop the LFE channel.
*
* `channelConfig` is intentionally flexible (codec-specific layout strings
* are allowed alongside numeric ones), so anything this parser doesn't
* recognize — and any numeric total that isn't a sane positive channel
* count, guarding against overflowed or hostile input producing `Infinity`
* — returns `undefined` rather than guessing stereo. Callers decide how to
* substitute (see `moqCatalogToPresentation` and `toAudioDecoderConfig`).
*/
export function parseChannelConfig(channelConfig: string | undefined): number {
if (!channelConfig) return 2;
export function parseChannelConfig(channelConfig: string | undefined): number | undefined {
if (!channelConfig) return undefined;
const trimmed = channelConfig.trim();
if (!/^\d+(\.\d+)*$/.test(trimmed)) return 2;
if (!/^\d+(\.\d+)*$/.test(trimmed)) return undefined;
const total = trimmed.split('.').reduce((sum, part) => sum + Number(part), 0);
return total > 0 ? total : 2;
return Number.isSafeInteger(total) && total > 0 && total <= MAX_CHANNELS ? total : undefined;
}

/**
Expand Down Expand Up @@ -497,7 +518,7 @@ export function moqCatalogToPresentation(
// carry the raw values — `toAudioDecoderConfig` reads those rather
// than trusting these for decoder configuration.
sampleRate: track.samplerate ?? 48_000,
channels: parseChannelConfig(track.channelConfig),
channels: parseChannelConfig(track.channelConfig) ?? 2,
});
} else {
text.push({
Expand Down
37 changes: 37 additions & 0 deletions packages/spf/src/media/moq/tests/codec-mapping.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -89,4 +89,41 @@ describe('toAudioDecoderConfig', () => {
it('pins Opus to 48 kHz without a declared samplerate (RFC 6716 decode rate)', () => {
expect(toAudioDecoderConfig(audioTrack({ codecs: ['Opus'] }))).toMatchObject({ sampleRate: 48_000 });
});

// A malformed or hostile catalog `samplerate` must not reach
// `AudioDecoderConfig.sampleRate` (a WebIDL `unsigned long`) — reject the
// track instead of configuring the decoder with an invalid rate.
it.each([-44_100, 0, 44_100.5, Infinity, 0x1_00_00_00_00])(
'returns null for a malformed samplerate (%s)',
(samplerate) => {
const track = audioTrack({
codecs: ['mp4a.40.2'],
moq: { namespace: ['ns'], name: 'audio', packaging: 'loc', isLive: true, samplerate },
});
expect(toAudioDecoderConfig(track)).toBeNull();
}
);

it('derives channel count from the declared channelConfig, including surround layouts', () => {
const track = audioTrack({
channels: 2, // projection substitute — must not be trusted over channelConfig
moq: { namespace: ['ns'], name: 'audio', packaging: 'loc', isLive: true, channelConfig: '5.1' },
});
expect(toAudioDecoderConfig(track)).toMatchObject({ numberOfChannels: 6 });
});

it('defaults to stereo when channelConfig is omitted', () => {
expect(toAudioDecoderConfig(audioTrack())).toMatchObject({ numberOfChannels: 2 });
});

// An omitted channelConfig defaults to stereo (common convention for
// simple streams), but a *declared* value this parser can't resolve to a
// channel count must not silently decode as stereo either.
it('returns null when a declared channelConfig cannot be resolved', () => {
const track = audioTrack({
channels: 2,
moq: { namespace: ['ns'], name: 'audio', packaging: 'loc', isLive: true, channelConfig: 'JOC' },
});
expect(toAudioDecoderConfig(track)).toBeNull();
});
});
Loading