From 7f056cbe2206e5801860e22c28863cd89ab5a304 Mon Sep 17 00:00:00 2001 From: Matt McClure Date: Tue, 18 Aug 2026 05:36:58 +0000 Subject: [PATCH] feat(spf): add an audio codec preference to the publish encoder ladder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit probeEncoderSupport built its audio ladder from a hardcoded Opus codec, so every publish session was Opus unconditionally — making RTMP/FLV and LL-HLS egress video-only for every browser publish even where the browser ships an AAC encoder. Mirror the camera/screen codec seam: `audio.codec` on the engine config prepends a preferred WebCodecs codec string to the Opus default rather than replacing it, codec-major across the sample-rate ladder, so `audio: { codec: 'mp4a.40.2' }` publishes AAC where the browser can encode it and degrades to Opus where it can't. The catalog side already describes whatever the encoder emits (mp4a init-data gating, non-Opus sample-rate passthrough), so this is encoder-ladder work only. Closes #36 --- .../behaviors/dom/probe-encoder-support.ts | 46 ++++++++++----- .../dom/tests/probe-encoder-support.test.ts | 56 +++++++++++++++++++ .../spf/src/publish/engines/moq/engine.ts | 5 +- packages/spf/src/publish/engines/moq/index.ts | 1 + 4 files changed, 94 insertions(+), 14 deletions(-) diff --git a/packages/spf/src/publish/behaviors/dom/probe-encoder-support.ts b/packages/spf/src/publish/behaviors/dom/probe-encoder-support.ts index fa22da2a09..90e7ed4c74 100644 --- a/packages/spf/src/publish/behaviors/dom/probe-encoder-support.ts +++ b/packages/spf/src/publish/behaviors/dom/probe-encoder-support.ts @@ -14,11 +14,14 @@ * bitstream format so the avcC extradata exists to publish out-of-band — * see `videoCandidates`) with a VP8 fallback, at the track * resolution/framerate; Opus at the track sample rate with a 48 kHz - * fallback. A `config.{camera,screen}.codec` prepends itself to the - * ladder rather than replacing it. The screen ladder defaults to a lower - * framerate than the camera's (static degrade-screen-first tuning, per - * the multi-source design record's "Encoder budget" decision — no dynamic - * policy in v1). + * fallback. A `config.{camera,screen,audio}.codec` prepends itself to the + * ladder rather than replacing it — for audio the preferred codec probes + * at every ladder sample rate before Opus does (codec preference + * outranks rate preference), so `audio: { codec: 'mp4a.40.2' }` publishes + * AAC where the browser can encode it and falls back to Opus where it + * can't. The screen ladder defaults to a lower framerate than the + * camera's (static degrade-screen-first tuning, per the multi-source + * design record's "Encoder budget" decision — no dynamic policy in v1). * * Each probe is a single-positive-state reactor mirroring the acquire * behaviors: the probe runs in the positive state's `effects:` so a @@ -78,11 +81,22 @@ export interface VideoEncodeTuning { codec?: string; } +export interface AudioEncodeTuning { + bitrate?: number; + /** + * Preferred WebCodecs audio codec string (e.g. `'mp4a.40.2'` for + * AAC-LC). Prepended to the Opus default rather than replacing it, so + * an unsupported preference degrades to Opus instead of failing the + * kind. + */ + codec?: string; +} + /** Per-kind encode tuning (mirrors `MoqPublishEngineConfig`). */ export interface ProbeEncoderSupportConfig { camera?: VideoEncodeTuning; screen?: VideoEncodeTuning; - audio?: { bitrate?: number }; + audio?: AudioEncodeTuning; selectEncoderConfig?: SelectEncoderConfig; } @@ -147,14 +161,20 @@ function screenCandidates(track: CaptureTrackFacts, tuning: VideoEncodeTuning | }); } -function audioCandidates(track: CaptureTrackFacts, audio: ProbeEncoderSupportConfig['audio']): AudioEncoderConfig[] { +function audioCandidates(track: CaptureTrackFacts, audio: AudioEncodeTuning | undefined): AudioEncoderConfig[] { + const codecs = [...new Set([...(audio?.codec ? [audio.codec] : []), DEFAULT_AUDIO_CODEC])]; const sampleRates = [...new Set([track.sampleRate ?? OPUS_SAMPLE_RATE, OPUS_SAMPLE_RATE])]; - return sampleRates.map((sampleRate) => ({ - codec: DEFAULT_AUDIO_CODEC, - sampleRate, - numberOfChannels: track.channelCount ?? 1, - bitrate: audio?.bitrate ?? DEFAULT_AUDIO_BITRATE, - })); + // Codec-major: the preferred codec exhausts its sample rates before the + // default gets a turn — the preference exists for egress compatibility, + // which a rate tweak can't buy back. + return codecs.flatMap((codec) => + sampleRates.map((sampleRate) => ({ + codec, + sampleRate, + numberOfChannels: track.channelCount ?? 1, + bitrate: audio?.bitrate ?? DEFAULT_AUDIO_BITRATE, + })) + ); } /** diff --git a/packages/spf/src/publish/behaviors/dom/tests/probe-encoder-support.test.ts b/packages/spf/src/publish/behaviors/dom/tests/probe-encoder-support.test.ts index 3175da839b..cd51862348 100644 --- a/packages/spf/src/publish/behaviors/dom/tests/probe-encoder-support.test.ts +++ b/packages/spf/src/publish/behaviors/dom/tests/probe-encoder-support.test.ts @@ -34,6 +34,18 @@ function mockVideoEncoderSupport(supported: () => boolean): void { })); } +/** + * Force the audio ladder's verdict per candidate. Needed for any AAC arm: + * headless Chromium ships no AAC encoder, so the preference walk would + * always fall back to Opus under the real `isConfigSupported`. + */ +function mockAudioEncoderSupport(supported: (config: AudioEncoderConfig) => boolean): void { + vi.spyOn(AudioEncoder, 'isConfigSupported').mockImplementation(async (config) => ({ + supported: supported(config), + config, + })); +} + const CAMERA_TRACK = { width: 320, height: 240, frameRate: 30 }; const SCREEN_TRACK = { width: 1920, height: 1080, frameRate: 30 }; const MIC_TRACK = { sampleRate: 48_000, channelCount: 1 }; @@ -129,6 +141,50 @@ describe('probeEncoderSupport', () => { expect(state.activeEncodings.get()!.screen).toMatchObject({ framerate: 15, bitrate: 1_500_000 }); }); + it('walks the audio codec preference before the Opus default (codec-major ladder)', async () => { + mockAudioEncoderSupport(() => true); + const { state } = setupProbe({ audio: { codec: 'mp4a.40.2' } }); + + state.micTracks.set({ sampleRate: 44_100, channelCount: 2 }); + + await vi.waitFor(() => { + expect(state.encoderSupport.get()?.audio).toBeDefined(); + }); + // The preferred codec exhausts its sample rates before Opus gets a turn. + expect(state.encoderSupport.get()!.audio!.map(({ codec, sampleRate }) => ({ codec, sampleRate }))).toEqual([ + { codec: 'mp4a.40.2', sampleRate: 44_100 }, + { codec: 'mp4a.40.2', sampleRate: 48_000 }, + { codec: 'opus', sampleRate: 44_100 }, + { codec: 'opus', sampleRate: 48_000 }, + ]); + expect(state.activeEncodings.get()!.audio).toMatchObject({ codec: 'mp4a.40.2', sampleRate: 44_100 }); + }); + + it('falls back to Opus when the preferred audio codec is unsupported', async () => { + mockAudioEncoderSupport((config) => config.codec === 'opus'); + const { state } = setupProbe({ audio: { codec: 'mp4a.40.2' } }); + + state.micTracks.set(MIC_TRACK); + + await vi.waitFor(() => { + expect(state.activeEncodings.get()?.audio).toBeDefined(); + }); + expect(state.activeEncodings.get()!.audio).toMatchObject({ codec: 'opus', sampleRate: 48_000 }); + expect(state.publishError.get()).toBeUndefined(); + }); + + it('does not duplicate the ladder when the preferred audio codec is the default', async () => { + mockAudioEncoderSupport(() => true); + const { state } = setupProbe({ audio: { codec: 'opus' } }); + + state.micTracks.set(MIC_TRACK); + + await vi.waitFor(() => { + expect(state.encoderSupport.get()?.audio).toBeDefined(); + }); + expect(state.encoderSupport.get()!.audio).toHaveLength(1); + }); + it('resolves the active encodings through the selectEncoderConfig seam', async () => { const chosen: ActiveEncodingsFacts = {}; const { state } = setupProbe({ diff --git a/packages/spf/src/publish/engines/moq/engine.ts b/packages/spf/src/publish/engines/moq/engine.ts index 60440d08da..8f155d4a96 100644 --- a/packages/spf/src/publish/engines/moq/engine.ts +++ b/packages/spf/src/publish/engines/moq/engine.ts @@ -24,6 +24,7 @@ import type { CaptureDeviceFacts } from '../../behaviors/dom/enumerate-capture-d import { enumerateCaptureDevices } from '../../behaviors/dom/enumerate-capture-devices'; import type { ActiveEncodingsFacts, + AudioEncodeTuning, EncoderSupportFacts, SelectEncoderConfig, VideoEncodeTuning, @@ -48,6 +49,7 @@ import type { ConnectPublishTransport, PublishEndpoint, PublishSessionActor } fr export type { ActiveEncodingsFacts, AudioEncoderActor, + AudioEncodeTuning, CaptureDeviceFacts, CaptureSourceKind, CaptureStatus, @@ -174,7 +176,8 @@ export interface MoqPublishEngineConfig extends ShareSignalsConfig