From 577403367e11440d4861a92028e3739e36b965c2 Mon Sep 17 00:00:00 2001 From: Vance Ingalls Date: Fri, 19 Jun 2026 15:35:58 -0700 Subject: [PATCH] fix(slideshow): harden media controls in present decks --- packages/core/src/runtime/bridge.test.ts | 34 ++ packages/core/src/runtime/bridge.ts | 6 + packages/core/src/runtime/init.test.ts | 51 ++ packages/core/src/runtime/init.ts | 82 +++- packages/core/src/runtime/state.test.ts | 2 + packages/core/src/runtime/state.ts | 16 + packages/core/src/runtime/types.ts | 3 + .../player/src/hyperframes-player.test.ts | 80 +++ packages/player/src/hyperframes-player.ts | 11 + .../src/runtime-message-handler.test.ts | 34 ++ .../player/src/runtime-message-handler.ts | 5 + .../slideshow/hyperframes-slideshow.test.ts | 433 +++++++++++++++++ .../src/slideshow/hyperframes-slideshow.ts | 460 ++++++++++++++++-- .../src/slideshow/slideshowPresenter.ts | 101 +++- skills/slideshow/SKILL.md | 105 +++- .../references/standalone-harness.md | 371 +++++++++++++- 16 files changed, 1713 insertions(+), 81 deletions(-) diff --git a/packages/core/src/runtime/bridge.test.ts b/packages/core/src/runtime/bridge.test.ts index f590006987..4ab0f9053c 100644 --- a/packages/core/src/runtime/bridge.test.ts +++ b/packages/core/src/runtime/bridge.test.ts @@ -11,6 +11,8 @@ function createMockDeps() { onSetMuted: vi.fn(), onSetVolume: vi.fn(), onSetMediaOutputMuted: vi.fn(), + onSetNativeMediaSyncDisabled: vi.fn(), + onSetWebAudioMediaDisabled: vi.fn(), onSetPlaybackRate: vi.fn(), onSetColorGrading: vi.fn(), onSetColorGradingCompare: vi.fn(), @@ -107,6 +109,38 @@ describe("installRuntimeControlBridge", () => { expect(deps.onSetMediaOutputMuted).toHaveBeenCalledWith(false); }); + it("dispatches set-native-media-sync-disabled command", () => { + const deps = createMockDeps(); + const handler = installRuntimeControlBridge(deps); + handler(makeControlMessage("set-native-media-sync-disabled", { disabled: true })); + expect(deps.onSetNativeMediaSyncDisabled).toHaveBeenCalledWith(true); + handler(makeControlMessage("set-native-media-sync-disabled", { disabled: false })); + expect(deps.onSetNativeMediaSyncDisabled).toHaveBeenCalledWith(false); + }); + + it("set-native-media-sync-disabled coerces absent flag to false", () => { + const deps = createMockDeps(); + const handler = installRuntimeControlBridge(deps); + handler(makeControlMessage("set-native-media-sync-disabled")); + expect(deps.onSetNativeMediaSyncDisabled).toHaveBeenCalledWith(false); + }); + + it("dispatches set-web-audio-media-disabled command", () => { + const deps = createMockDeps(); + const handler = installRuntimeControlBridge(deps); + handler(makeControlMessage("set-web-audio-media-disabled", { disabled: true })); + expect(deps.onSetWebAudioMediaDisabled).toHaveBeenCalledWith(true); + handler(makeControlMessage("set-web-audio-media-disabled", { disabled: false })); + expect(deps.onSetWebAudioMediaDisabled).toHaveBeenCalledWith(false); + }); + + it("set-web-audio-media-disabled coerces absent flag to false", () => { + const deps = createMockDeps(); + const handler = installRuntimeControlBridge(deps); + handler(makeControlMessage("set-web-audio-media-disabled")); + expect(deps.onSetWebAudioMediaDisabled).toHaveBeenCalledWith(false); + }); + it("dispatches set-playback-rate command", () => { const deps = createMockDeps(); const handler = installRuntimeControlBridge(deps); diff --git a/packages/core/src/runtime/bridge.ts b/packages/core/src/runtime/bridge.ts index 7665a38089..6430cdeb9a 100644 --- a/packages/core/src/runtime/bridge.ts +++ b/packages/core/src/runtime/bridge.ts @@ -11,6 +11,8 @@ type BridgeDeps = { onSetMuted: (muted: boolean) => void; onSetVolume: (volume: number) => void; onSetMediaOutputMuted: (muted: boolean) => void; + onSetNativeMediaSyncDisabled: (disabled: boolean) => void; + onSetWebAudioMediaDisabled: (disabled: boolean) => void; onSetPlaybackRate: (rate: number) => void; onSetColorGrading: (target: HfColorGradingTarget | string | null, grading: unknown) => void; onSetColorGradingCompare: ( @@ -46,6 +48,10 @@ const CONTROL_HANDLERS: Record = { "set-volume": (data, deps) => deps.onSetVolume(Math.max(0, Math.min(1, Number(data.volume ?? 1)))), "set-media-output-muted": (data, deps) => deps.onSetMediaOutputMuted(Boolean(data.muted)), + "set-native-media-sync-disabled": (data, deps) => + deps.onSetNativeMediaSyncDisabled(Boolean(data.disabled)), + "set-web-audio-media-disabled": (data, deps) => + deps.onSetWebAudioMediaDisabled(Boolean(data.disabled)), "set-playback-rate": (data, deps) => deps.onSetPlaybackRate(Number(data.playbackRate ?? 1)), "set-color-grading": (data, deps) => deps.onSetColorGrading(data.target ?? null, data.grading ?? null), diff --git a/packages/core/src/runtime/init.test.ts b/packages/core/src/runtime/init.test.ts index 2549c21bb9..f15b2a22a7 100644 --- a/packages/core/src/runtime/init.test.ts +++ b/packages/core/src/runtime/init.test.ts @@ -1147,6 +1147,57 @@ describe("initSandboxRuntimeModular", () => { expect(audio.muted).toBe(false); }); + it("native media sync opt-out leaves user-started media playing while timeline is paused", () => { + const root = document.createElement("div"); + root.setAttribute("data-composition-id", "root"); + root.setAttribute("data-root", "true"); + root.setAttribute("data-start", "0"); + root.setAttribute("data-duration", "10"); + root.setAttribute("data-width", "1920"); + root.setAttribute("data-height", "1080"); + document.body.appendChild(root); + + const audio = document.createElement("audio"); + audio.setAttribute("data-start", "0"); + audio.setAttribute("data-duration", "10"); + audio.setAttribute("src", "voiceover.mp3"); + Object.defineProperty(audio, "duration", { value: 10, configurable: true }); + Object.defineProperty(audio, "readyState", { + value: HTMLMediaElement.HAVE_FUTURE_DATA, + configurable: true, + }); + Object.defineProperty(audio, "currentTime", { value: 0, writable: true, configurable: true }); + Object.defineProperty(audio, "paused", { value: true, writable: true, configurable: true }); + audio.pause = vi.fn(() => { + Object.defineProperty(audio, "paused", { + value: true, + writable: true, + configurable: true, + }); + }); + root.appendChild(audio); + + window.__timelines = { root: createMockTimeline(10) }; + initSandboxRuntimeModular(); + + window.dispatchEvent( + new MessageEvent("message", { + data: { + source: "hf-parent", + type: "control", + action: "set-native-media-sync-disabled", + disabled: true, + }, + }), + ); + Object.defineProperty(audio, "paused", { value: false, writable: true, configurable: true }); + vi.mocked(audio.pause).mockClear(); + + window.__player?.renderSeek(5); + + expect(audio.pause).not.toHaveBeenCalled(); + }); + it("skips the per-frame transport re-seek while a Studio manual-edit gesture is active", () => { const raf = createManualRaf(); vi.spyOn(performance, "now").mockImplementation(() => raf.now()); diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts index 684c6db7bd..ef83d4349a 100644 --- a/packages/core/src/runtime/init.ts +++ b/packages/core/src/runtime/init.ts @@ -1501,23 +1501,27 @@ export function initSandboxRuntimeModular(): void { const forceSync = state.mediaForceSyncNextTick; if (forceSync) state.mediaForceSyncNextTick = false; - syncRuntimeMedia({ - clips: cache.mediaClips, - timeSeconds: state.currentTime, - playing: state.isPlaying, - playbackRate: state.playbackRate, - outputMuted: state.mediaOutputMuted, - userMuted: state.bridgeMuted, - userVolume: state.bridgeVolume, - forceSync, - onElementVolume: (el, volume) => webAudio.setElementVolume(el, volume), - isWebAudioOwned: (el) => webAudio.ownsElement(el), - onAutoplayBlocked: () => { - if (state.mediaAutoplayBlockedPosted) return; - state.mediaAutoplayBlockedPosted = true; - postRuntimeMessage({ source: "hf-preview", type: "media-autoplay-blocked" }); - }, - }); + if (!state.nativeMediaSyncDisabled) { + syncRuntimeMedia({ + clips: cache.mediaClips, + timeSeconds: state.currentTime, + playing: state.isPlaying, + playbackRate: state.playbackRate, + outputMuted: + state.mediaOutputMuted || + (!state.webAudioMediaDisabled && !state.nativeMediaSyncDisabled && webAudio.isActive()), + userMuted: state.bridgeMuted, + userVolume: state.bridgeVolume, + forceSync, + onElementVolume: (el, volume) => webAudio.setElementVolume(el, volume), + isWebAudioOwned: (el) => webAudio.ownsElement(el), + onAutoplayBlocked: () => { + if (state.mediaAutoplayBlockedPosted) return; + state.mediaAutoplayBlockedPosted = true; + postRuntimeMessage({ source: "hf-preview", type: "media-autoplay-blocked" }); + }, + }); + } const visibilityNodes = Array.from(document.querySelectorAll("[data-start]")); const rootComp = resolveRootCompositionElement(); for (const rawNode of visibilityNodes) { @@ -1882,6 +1886,29 @@ export function initSandboxRuntimeModular(): void { el.muted = effective || el.defaultMuted; } }, + onSetNativeMediaSyncDisabled: (disabled) => { + if (state.nativeMediaSyncDisabled === disabled) return; + state.nativeMediaSyncDisabled = disabled; + state.mediaForceSyncNextTick = true; + if (disabled) { + webAudio.stopAll(); + clock.detachAudioSource(); + } else { + syncMediaForCurrentState(); + } + }, + onSetWebAudioMediaDisabled: (disabled) => { + if (state.webAudioMediaDisabled === disabled) return; + state.webAudioMediaDisabled = disabled; + state.mediaForceSyncNextTick = true; + if (disabled) { + webAudio.stopAll(); + clock.detachAudioSource(); + syncMediaForCurrentState(); + } else { + syncMediaForCurrentState(); + } + }, onSetPlaybackRate: (rate) => { applyPlaybackRate(rate); if (state.transportClock) state.transportClock.setRate(state.playbackRate); @@ -2201,7 +2228,12 @@ export function initSandboxRuntimeModular(): void { // 2. HTMLMediaElement (audio.currentTime): ~33ms, frame-accurate // 3. Monotonic (performance.now()): ~1ms, no audio coupling if (clock.isPlaying() && !state.mediaOutputMuted) { - if (webAudio.isActive() && webAudio.context) { + if ( + !state.nativeMediaSyncDisabled && + !state.webAudioMediaDisabled && + webAudio.isActive() && + webAudio.context + ) { const webAudioTime = webAudio.getTime(); if (webAudioTime >= 0) { clock.attachAudioSource({ currentTimeSeconds: webAudioTime }); @@ -2313,6 +2345,7 @@ export function initSandboxRuntimeModular(): void { // same edge as the HTMLMedia path. Reused by play() and by the rate-change // handler (a rate change can't rescale a bounded source in place). const scheduleWebAudioForActiveClips = () => { + if (state.nativeMediaSyncDisabled || state.webAudioMediaDisabled) return; const gen = webAudio.startGeneration(); const audioEls = document.querySelectorAll("audio[data-start]"); for (const rawEl of audioEls) { @@ -2362,7 +2395,14 @@ export function initSandboxRuntimeModular(): void { // stopAll()+reschedule at the new rate to keep trimmed clips ending on time. const applyWebAudioRate = () => { const changed = webAudio.setRate(state.playbackRate); - if (changed && webAudioReady && clock.isPlaying() && webAudio.hasBoundedActiveSources()) { + if ( + changed && + !state.nativeMediaSyncDisabled && + !state.webAudioMediaDisabled && + webAudioReady && + clock.isPlaying() && + webAudio.hasBoundedActiveSources() + ) { webAudio.stopAll(); scheduleWebAudioForActiveClips(); } @@ -2392,7 +2432,9 @@ export function initSandboxRuntimeModular(): void { // Schedule audio through WebAudio for sample-accurate timing. // Falls back to HTMLMediaElement playback if WebAudio isn't ready // or decoding fails (the syncRuntimeMedia path handles that). - if (webAudioReady) scheduleWebAudioForActiveClips(); + if (webAudioReady && !state.nativeMediaSyncDisabled && !state.webAudioMediaDisabled) { + scheduleWebAudioForActiveClips(); + } runAdapters("play"); syncMediaForCurrentState(); colorGrading.redraw(); diff --git a/packages/core/src/runtime/state.test.ts b/packages/core/src/runtime/state.test.ts index f5d788f177..8639148d59 100644 --- a/packages/core/src/runtime/state.test.ts +++ b/packages/core/src/runtime/state.test.ts @@ -9,6 +9,8 @@ describe("createRuntimeState", () => { expect(state.canonicalFps).toBe(30); expect(state.playbackRate).toBe(1); expect(state.bridgeMuted).toBe(false); + expect(state.nativeMediaSyncDisabled).toBe(false); + expect(state.webAudioMediaDisabled).toBe(false); expect(state.capturedTimeline).toBeNull(); expect(state.tornDown).toBe(false); }); diff --git a/packages/core/src/runtime/state.ts b/packages/core/src/runtime/state.ts index b5f02002b2..9a9164e386 100644 --- a/packages/core/src/runtime/state.ts +++ b/packages/core/src/runtime/state.ts @@ -19,6 +19,20 @@ export type RuntimeState = { * accuracy but produces no audio of its own. */ mediaOutputMuted: boolean; + /** + * Disable runtime ownership of native media elements. Slideshow presenter + * mode keeps the slide timeline paused while users interact with embedded + * media, so the runtime must not auto-play, auto-pause, seek, or volume-sync + * those native elements on every transport tick. + */ + nativeMediaSyncDisabled: boolean; + /** + * Disable the runtime's WebAudio replacement for native