diff --git a/packages/freecut-editor/package.json b/packages/freecut-editor/package.json index c0b4384ff..10f2ce165 100644 --- a/packages/freecut-editor/package.json +++ b/packages/freecut-editor/package.json @@ -1,6 +1,6 @@ { "name": "@quantfive/freecut-editor-surface", - "version": "0.3.16", + "version": "0.3.17", "description": "The host-backed FreeCut browser editor surface.", "license": "MIT", "repository": { diff --git a/src/runtime/composition-runtime/components/video-content.test.tsx b/src/runtime/composition-runtime/components/video-content.test.tsx index 6adc801be..6b8626c10 100644 --- a/src/runtime/composition-runtime/components/video-content.test.tsx +++ b/src/runtime/composition-runtime/components/video-content.test.tsx @@ -1,5 +1,5 @@ import { act, render, waitFor } from '@testing-library/react' -import { beforeEach, describe, expect, it, vi } from 'vite-plus/test' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vite-plus/test' import { shouldIssueCoalescedReverseVideoSeek } from '../utils/video-sync-plan' import { getAudioTargetTimeSeconds } from '../utils/video-timing' import { VideoContent } from './video-content' @@ -15,6 +15,7 @@ const testState = vi.hoisted(() => ({ acquireForClip: ReturnType releaseClip: ReturnType } | null, + clock: { currentFrame: 0, onFrameChange: () => () => {} }, renderedIsPlaying: null as boolean | null, playbackState: { currentFrame: 0, @@ -103,10 +104,7 @@ function createMockVideoElement(): HTMLVideoElement { vi.mock('@/runtime/composition-runtime/deps/player', () => ({ useSequenceContext: () => ({ localFrame: 0, from: 0, durationInFrames: 120, parentFrom: 0 }), useVideoSourcePool: () => testState.pool!, - useClock: () => ({ - currentFrame: 0, - onFrameChange: () => () => {}, - }), + useClock: () => testState.clock, useClockPlaybackRate: () => 1, interpolate: () => 0, isVideoPoolAbortError: () => false, @@ -147,7 +145,9 @@ vi.mock('./video-audio-context', () => ({ })) describe('VideoContent pooled handoff', () => { + afterEach(() => vi.restoreAllMocks()) beforeEach(() => { + vi.spyOn(HTMLCanvasElement.prototype, 'getContext').mockReturnValue(null) preloadSourceMock.mockClear() acquireForClipMock.mockClear() releaseClipMock.mockClear() @@ -195,6 +195,52 @@ describe('VideoContent pooled handoff', () => { }, ) + it('gates recycled pixels on acquisition and same-lane cut until the decoded target frame', async () => { + const video = createMockVideoElement() + const callbacks: VideoFrameRequestCallback[] = [] + video.requestVideoFrameCallback = vi.fn((cb) => { + callbacks.push(cb) + return callbacks.length + }) + video.cancelVideoFrameCallback = vi.fn() + acquireForClipMock.mockReturnValue(video) + const view = (id: string, trim: number) => ( + + ) + const { rerender, unmount } = render(view('first', 300)) + expect(video.currentTime).toBeCloseTo(10) + expect(video.style.opacity).toBe('0') + act(() => callbacks.at(-1)?.(0, { mediaTime: 0 } as VideoFrameCallbackMetadata)) + expect(video.style.opacity).toBe('0') + act(() => callbacks.at(-1)?.(0, { mediaTime: 10 } as VideoFrameCallbackMetadata)) + expect(video.style.opacity).toBe('') + rerender(view('second', 600)) + expect(acquireForClipMock).toHaveBeenCalledTimes(1) + expect(video.currentTime).toBeCloseTo(20) + expect(video.style.opacity).toBe('0') + act(() => callbacks.at(-1)?.(0, { mediaTime: 10 } as VideoFrameCallbackMetadata)) + expect(video.style.opacity).toBe('0') + act(() => callbacks.at(-1)?.(0, { mediaTime: 20 } as VideoFrameCallbackMetadata)) + expect(video.style.opacity).toBe('') + unmount() + }) + it('keeps the acquired pool element when only itemId changes on the same pool lane', async () => { const pooledElement = createMockVideoElement() acquireForClipMock.mockReturnValue(pooledElement) diff --git a/src/runtime/composition-runtime/components/video-content.tsx b/src/runtime/composition-runtime/components/video-content.tsx index be7dc16b9..3166d6887 100644 --- a/src/runtime/composition-runtime/components/video-content.tsx +++ b/src/runtime/composition-runtime/components/video-content.tsx @@ -40,6 +40,8 @@ import { } from './video-audio-context' import { getBrowserMediaPlaybackRate } from '@/shared/state/playback/shuttle' +import { createVideoFrameHandoff } from '../utils/video-frame-handoff' + const videoLog = createLogger('NativePreviewVideo') const contentLog = createLogger('VideoContent') @@ -89,6 +91,7 @@ function isRecoverableVideoLoadError(message: string): boolean { */ const NativePreviewVideo: React.FC<{ poolClipId: string + trackId: string itemId: string src: string safeTrimBefore: number @@ -106,6 +109,7 @@ const NativePreviewVideo: React.FC<{ sharedTransitionSync?: boolean }> = ({ poolClipId, + trackId, itemId, src, safeTrimBefore, @@ -134,6 +138,12 @@ const NativePreviewVideo: React.FC<{ const itemIdRef = useRef(itemId) itemIdRef.current = itemId + const handoffRef = useRef | null>(null) + const seekVideo = useCallback((video: HTMLVideoElement, time: number) => { + if (Math.abs(video.currentTime - time) > 0.001) handoffRef.current?.prepare(time) + video.currentTime = time + }, []) + // Brief muted play/pause that fills the decode buffer and re-acquires the // browser's media pipeline, so a subsequent play() starts in ~2 frames // instead of stalling 200-300ms on pipeline re-init. Debounced so repeated @@ -189,7 +199,7 @@ const NativePreviewVideo: React.FC<{ Math.abs(v.currentTime - warmStartTime) > 0.001 ) { try { - v.currentTime = warmStartTime + seekVideo(v, warmStartTime) } catch { // The pooled element may be settling or have been released. } @@ -200,7 +210,7 @@ const NativePreviewVideo: React.FC<{ }) } }, 50) - }, []) + }, [seekVideo]) const audioVolumeRef = useRef(audioVolume) const audioEqStagesRef = useRef(audioEqStages) const onErrorRef = useRef(onError) @@ -389,9 +399,25 @@ const NativePreviewVideo: React.FC<{ ) const clampedInitial = Math.min(initialTargetTime, (element.duration || Infinity) - 0.1) const currentlyPlaying = usePlaybackStore.getState().isPlaying - const isNearTarget = Math.abs(element.currentTime - clampedInitial) < 0.2 + const isNearTarget = Math.abs(element.currentTime - clampedInitial) < 1 / initialFps const isContinuousPlayback = - !initialRequiresVisualSeek && currentlyPlaying && isNearTarget && element.readyState >= 2 + !initialRequiresVisualSeek && + currentlyPlaying && + !element.paused && + !element.seeking && + isNearTarget && + element.readyState >= 2 + + if (containerRef.current) { + handoffRef.current = createVideoFrameHandoff( + element, + containerRef.current, + clock, + trackId, + Math.max(0.05, 2 / initialFps), + ) + handoffRef.current.prepare(clampedInitial) + } elementRef.current = element syncRegisteredVideoElement(itemIdRef.current, element) @@ -400,7 +426,7 @@ const NativePreviewVideo: React.FC<{ if (initialRequiresVisualSeek) { element.pause() element.playbackRate = 1 - element.currentTime = clampedInitial + seekVideo(element, clampedInitial) needsInitialSyncRef.current = false } else if (isContinuousPlayback) { // Split boundary during playback: element was just paused by cleanup @@ -415,7 +441,7 @@ const NativePreviewVideo: React.FC<{ // instead of pausing and waiting for the sync effect next frame. // This eliminates ~16-50ms of React scheduling + readyState gate delay. element.playbackRate = initialMediaPlaybackRate - element.currentTime = clampedInitial + seekVideo(element, clampedInitial) if (element.readyState >= 2) { element.play().catch(() => {}) } @@ -443,7 +469,7 @@ const NativePreviewVideo: React.FC<{ ) if (Math.abs(element.currentTime - clampedLiveTargetTime) <= 0.016) return try { - element.currentTime = clampedLiveTargetTime + seekVideo(element, clampedLiveTargetTime) } catch { // Seek failed - element may still be stabilizing. } @@ -479,7 +505,7 @@ const NativePreviewVideo: React.FC<{ Math.abs(element.currentTime - latestPausedTarget) > 0.001 ) { try { - element.currentTime = latestPausedTarget + seekVideo(element, latestPausedTarget) pausedSeekInFlightRef.current = true } catch { // The latest skim target remains queued for the next sync pass. @@ -496,7 +522,7 @@ const NativePreviewVideo: React.FC<{ const handleEnded = () => { videoLog.debug(`[${shortId}] ended, seeking to last frame`) if (element.duration && element.duration > 0.1) { - element.currentTime = element.duration - 0.05 + seekVideo(element, element.duration - 0.05) } } @@ -548,7 +574,7 @@ const NativePreviewVideo: React.FC<{ 'seekPastEnd:', initialTargetTime > element.duration, ) - element.currentTime = clampedInitial + seekVideo(element, clampedInitial) } else { videoLog.debug( `[${shortId}] continuous playback, skipping seek (drift: ${(element.currentTime - clampedInitial).toFixed(3)}s)`, @@ -589,6 +615,9 @@ const NativePreviewVideo: React.FC<{ element.removeEventListener('error', handleError) element.removeEventListener('ended', handleEnded) + handoffRef.current?.dispose() + handoffRef.current = null + // Pause and remove from DOM element.pause() if (preWarmTimerRef.current !== null) { @@ -621,8 +650,11 @@ const NativePreviewVideo: React.FC<{ // the element across split-boundary transitions. }, [ poolClipId, + trackId, + clock, src, pool, + seekVideo, containerRef, shortId, syncRegisteredVideoElement, @@ -699,11 +731,11 @@ const NativePreviewVideo: React.FC<{ targetTime: seekTo, }) ) { - video.currentTime = seekTo + seekVideo(video, seekTo) pausedSeekInFlightRef.current = true } } else { - video.currentTime = seekTo + seekVideo(video, seekTo) } if (video.currentTime === seekTo || isPlaying) { lastSyncTimeRef.current = Date.now() @@ -718,6 +750,7 @@ const NativePreviewVideo: React.FC<{ if (layoutPlan.seekTo !== null) applyLayoutSeek(layoutPlan.seekTo) }, [ frame, + seekVideo, isPlaying, isReversed, mediaPlaybackRate, @@ -798,7 +831,7 @@ const NativePreviewVideo: React.FC<{ }) ) { try { - video.currentTime = latestReverseSeekTargetRef.current + seekVideo(video, latestReverseSeekTargetRef.current) reverseSeekInFlightRef.current = true lastSyncTimeRef.current = Date.now() needsInitialSyncRef.current = false @@ -831,7 +864,7 @@ const NativePreviewVideo: React.FC<{ video.pause() } if (premountPlan.seekTo !== null) { - video.currentTime = premountPlan.seekTo + seekVideo(video, premountPlan.seekTo) } return } @@ -853,7 +886,7 @@ const NativePreviewVideo: React.FC<{ }) if (initialSyncPlan.seekTo !== null) { try { - video.currentTime = initialSyncPlan.seekTo + seekVideo(video, initialSyncPlan.seekTo) } catch { // Seek failed - video may not be ready yet } @@ -879,7 +912,7 @@ const NativePreviewVideo: React.FC<{ }) if (driftCorrectionPlan.seekTo !== null) { try { - video.currentTime = driftCorrectionPlan.seekTo + seekVideo(video, driftCorrectionPlan.seekTo) lastSyncTimeRef.current = Date.now() } catch { // Seek failed - video may not be ready yet @@ -927,7 +960,7 @@ const NativePreviewVideo: React.FC<{ targetTime: pausedSyncPlan.seekTo, }) ) { - video.currentTime = pausedSyncPlan.seekTo + seekVideo(video, pausedSyncPlan.seekTo) pausedSeekInFlightRef.current = true } } catch { @@ -945,6 +978,7 @@ const NativePreviewVideo: React.FC<{ }, [ frame, fps, + seekVideo, isPlaying, isReversed, isReverseShuttle, @@ -1049,7 +1083,7 @@ const NativePreviewVideo: React.FC<{ if (correctionPlan.kind === 'seek') { try { - v.currentTime = correctionPlan.seekTo + seekVideo(v, correctionPlan.seekTo) if (correctionPlan.shouldUpdateLastSyncTime) { lastSyncTimeRef.current = Date.now() } @@ -1072,7 +1106,7 @@ const NativePreviewVideo: React.FC<{ elementRef.current.playbackRate = mediaPlaybackRateRef.current } } - }, [clock, isPlaying, isReversed, isReverseShuttle, poolClipId, sharedTransitionSync]) + }, [clock, isPlaying, isReversed, isReverseShuttle, poolClipId, sharedTransitionSync, seekVideo]) // Keep volume/gain in sync for pooled element. useEffect(() => { @@ -1279,6 +1313,7 @@ export const VideoContent: React.FC<{ return ( () + video.requestVideoFrameCallback = vi.fn((cb) => { + callbacks.set(++next, cb) + return next + }) + video.cancelVideoFrameCallback = vi.fn() + const guard = createVideoFrameHandoff(video, container, pool, track, 0.067) + const frame = (time: number, id = next) => { + callbacks.get(id)?.(0, { mediaTime: time } as VideoFrameCallbackMetadata) + } + return { video, container, guard, frame } +} + +beforeEach(() => { + vi.spyOn(HTMLCanvasElement.prototype, 'getContext').mockReturnValue(null) +}) + +afterEach(() => { + vi.useRealTimers() + vi.restoreAllMocks() + document.body.replaceChildren() +}) + +describe('video frame handoff', () => { + it('does not reveal recycled pixels based on currentTime, seeked, or an old decoded frame', () => { + const { video, guard, frame } = setup() + guard.prepare(10) + video.currentTime = 10 + video.dispatchEvent(new Event('seeked')) + frame(0) + expect(video.style.opacity).toBe('0') + frame(10) + expect(video.style.opacity).toBe('') + guard.dispose() + }) + + it('rejects the callback from an earlier seek generation and ignores callbacks after disposal', () => { + const { video, guard, frame } = setup() + guard.prepare(10) + guard.prepare(20) + frame(10, 1) + expect(video.style.opacity).toBe('0') + video.currentTime = 20 + frame(20) + expect(video.style.opacity).toBe('') + guard.prepare(30) + guard.dispose() + frame(30) + expect(video.style.opacity).toBe('') + }) + + it('accepts an advanced first frame and rechecks a paused presented frame after seeked', () => { + const { video, guard, frame } = setup() + Object.defineProperty(video, 'paused', { value: false, configurable: true }) + guard.prepare(10) + video.currentTime = 10.2 + frame(10.2) + expect(video.style.opacity).toBe('') + guard.prepare(20) + video.currentTime = 20 + Object.defineProperty(video, 'seeking', { value: true, configurable: true }) + frame(20) + expect(video.style.opacity).toBe('0') + Object.defineProperty(video, 'seeking', { value: false, configurable: true }) + video.dispatchEvent(new Event('seeked')) + expect(video.style.opacity).toBe('') + guard.dispose() + }) + + it('waits for seek completion on engines without video frame callbacks', () => { + vi.useFakeTimers() + const { video, guard } = setup() + Reflect.deleteProperty(video, 'requestVideoFrameCallback') + guard.prepare(10) + video.currentTime = 10 + Object.defineProperty(video, 'seeking', { configurable: true, value: true }) + vi.advanceTimersByTime(20) + expect(video.style.opacity).toBe('0') + Object.defineProperty(video, 'seeking', { configurable: true, value: false }) + video.dispatchEvent(new Event('seeked')) + vi.advanceTimersByTime(20) + expect(video.style.opacity).toBe('') + guard.dispose() + expect(vi.getTimerCount()).toBe(0) + }) + + it('holds the outgoing presented image across lane remounts, scoped to the Player and track', () => { + const draw = vi.fn() + vi.spyOn(HTMLCanvasElement.prototype, 'getContext').mockReturnValue({ + drawImage: draw, + } as unknown as ReturnType) + const pool = {} + const outgoing = setup(pool) + outgoing.guard.prepare(3) + outgoing.video.currentTime = 3 + outgoing.frame(3) + outgoing.guard.dispose() + const incoming = setup(pool) + incoming.guard.prepare(12) + expect(incoming.container.querySelector('canvas')).not.toBeNull() + expect(draw).toHaveBeenCalledWith(outgoing.video, 0, 0, 640, 360) + incoming.frame(0) + expect(incoming.container.querySelector('canvas')).not.toBeNull() + incoming.video.currentTime = 12 + incoming.frame(12) + expect(incoming.container.querySelector('canvas')).toBeNull() + const otherTrack = setup(pool, 'other') + otherTrack.guard.prepare(1) + expect(otherTrack.container.querySelector('canvas')).toBeNull() + const otherPlayer = setup() + otherPlayer.guard.prepare(1) + expect(otherPlayer.container.querySelector('canvas')).toBeNull() + incoming.guard.dispose() + otherTrack.guard.dispose() + otherPlayer.guard.dispose() + }) +}) diff --git a/src/runtime/composition-runtime/utils/video-frame-handoff.ts b/src/runtime/composition-runtime/utils/video-frame-handoff.ts new file mode 100644 index 000000000..8784395be --- /dev/null +++ b/src/runtime/composition-runtime/utils/video-frame-handoff.ts @@ -0,0 +1,132 @@ +// A Clock belongs to one Player. Keep only its last presented image per track, +// never the arbitrary old bitmap on an incoming recycled video element. +const frames = new WeakMap>() + +export function createVideoFrameHandoff( + video: HTMLVideoElement, + container: HTMLElement, + owner: object, + trackId: string, + tolerance: number, +) { + let visible = false + let disposed = false + let lastPresented: number | null = null + let target = 0 + let generation = 0 + let callback: number | null = null + let raf: number | null = null + let overlay: HTMLCanvasElement | null = null + const originalOpacity = video.style.opacity + const cache = frames.get(owner) ?? new Map() + frames.set(owner, cache) + + const capture = () => { + if (!visible || video.seeking || video.readyState < 2 || !video.videoWidth) return + if (getComputedStyle(video).visibility === 'hidden') return + const canvas = cache.get(trackId) ?? document.createElement('canvas') + canvas.width = Math.min(video.videoWidth, 1280) + canvas.height = Math.max(1, Math.round((canvas.width * video.videoHeight) / video.videoWidth)) + try { + const context = canvas.getContext('2d') + if (!context) return + context.drawImage(video, 0, 0, canvas.width, canvas.height) + cache.set(trackId, canvas) + } catch { + // A protected/undecodable source cannot supply a held image. + } + } + const removeOverlay = () => { + overlay?.remove() + overlay = null + } + const cancel = () => { + if (callback !== null) video.cancelVideoFrameCallback(callback) + if (raf !== null) cancelAnimationFrame(raf) + callback = null + raf = null + } + const reveal = (mediaTime: number, epoch: number) => { + if (disposed || epoch !== generation || video.seeking || video.readyState < 2) return false + // currentTime alone is not presentation proof; compare it with the actual + // decoded timestamp. Allow frames advanced past the seek target during play. + if (Math.abs(mediaTime - video.currentTime) > tolerance) return false + if (Math.abs(mediaTime - target) > tolerance && (video.paused || mediaTime < target)) + return false + visible = true + video.style.opacity = originalOpacity + removeOverlay() + return true + } + const awaitFrame = () => { + const epoch = generation + if (typeof video.requestVideoFrameCallback === 'function') { + callback = video.requestVideoFrameCallback((_now, metadata) => { + callback = null + if (disposed || epoch !== generation) return + lastPresented = metadata.mediaTime + reveal(metadata.mediaTime, epoch) + awaitFrame() + }) + } + } + // Older engines without rVFC only release after a settled seek and a paint. + const settled = () => { + if (visible || video.seeking) return + if (typeof video.requestVideoFrameCallback === 'function') { + if (lastPresented !== null) reveal(lastPresented, generation) + return + } + const epoch = generation + if (raf !== null) cancelAnimationFrame(raf) + raf = requestAnimationFrame(() => { + raf = null + reveal(video.currentTime, epoch) + }) + } + video.addEventListener('seeked', settled) + video.addEventListener('loadeddata', settled) + + return { + prepare(time: number) { + if (disposed) return + capture() + lastPresented = null + target = time + generation++ + cancel() + visible = false + video.style.opacity = '0' + removeOverlay() + const held = cache.get(trackId) + if (held) { + overlay = document.createElement('canvas') + overlay.width = held.width + overlay.height = held.height + overlay.getContext('2d')?.drawImage(held, 0, 0) + Object.assign(overlay.style, { + position: 'absolute', + inset: '0', + width: '100%', + height: '100%', + objectFit: video.style.objectFit || 'contain', + pointerEvents: 'none', + }) + overlay.setAttribute('aria-hidden', 'true') + container.appendChild(overlay) + } + awaitFrame() + settled() + }, + dispose() { + capture() + disposed = true + generation++ + cancel() + removeOverlay() + video.style.opacity = originalOpacity + video.removeEventListener('seeked', settled) + video.removeEventListener('loadeddata', settled) + }, + } +}