From 70f571436b038677bd46033d64af9d2764402aba Mon Sep 17 00:00:00 2001 From: webadderall <131426131+webadderall@users.noreply.github.com> Date: Tue, 2 Jun 2026 19:12:11 +1000 Subject: [PATCH 1/2] fix(webcam): stabilize frame sync --- src/components/video-editor/VideoPlayback.tsx | 18 +++++--- .../videoPlayback/webcamSync.test.ts | 42 +++++++++++++++++++ .../video-editor/videoPlayback/webcamSync.ts | 26 ++++++++++++ src/lib/exporter/frameRenderer.test.ts | 32 ++++++++++++++ src/lib/exporter/frameRenderer.ts | 10 ++++- src/lib/exporter/modernFrameRenderer.test.ts | 10 ++--- 6 files changed, 125 insertions(+), 13 deletions(-) diff --git a/src/components/video-editor/VideoPlayback.tsx b/src/components/video-editor/VideoPlayback.tsx index c1e51c3aa..488bc3a9d 100644 --- a/src/components/video-editor/VideoPlayback.tsx +++ b/src/components/video-editor/VideoPlayback.tsx @@ -169,7 +169,10 @@ import { clampFocusToStage as clampFocusToStageUtil } from "./videoPlayback/focu import { layoutVideoContent as layoutVideoContentUtil } from "./videoPlayback/layoutUtils"; import { updateOverlayIndicator } from "./videoPlayback/overlayUtils"; import { createVideoEventHandlers } from "./videoPlayback/videoEventHandlers"; -import { getWebcamMediaTargetTimeSeconds } from "./videoPlayback/webcamSync"; +import { + getWebcamMediaTargetTimeSeconds, + shouldSeekWebcamMedia, +} from "./videoPlayback/webcamSync"; import { findDominantRegion } from "./videoPlayback/zoomRegionUtils"; import { applyZoomTransform, @@ -1836,12 +1839,15 @@ const VideoPlayback = forwardRef( } const previousTimelineTime = lastWebcamSyncTimeRef.current; - const timelineJumped = - previousTimelineTime === null || Math.abs(targetTime - previousTimelineTime) > 0.25; - const driftThreshold = isPlaying ? 0.35 : 0.01; if ( - timelineJumped || - Math.abs(webcamVideo.currentTime - mediaTargetTime) > driftThreshold + shouldSeekWebcamMedia({ + desiredTime: mediaTargetTime, + isPlaying, + isSeeking: webcamVideo.seeking, + previousTimelineTime, + timelineTime: targetTime, + webcamCurrentTime: webcamVideo.currentTime, + }) ) { try { webcamVideo.currentTime = mediaTargetTime; diff --git a/src/components/video-editor/videoPlayback/webcamSync.test.ts b/src/components/video-editor/videoPlayback/webcamSync.test.ts index d8d1f5ac4..636a6488f 100644 --- a/src/components/video-editor/videoPlayback/webcamSync.test.ts +++ b/src/components/video-editor/videoPlayback/webcamSync.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest"; import { getWebcamMediaTargetTimeSeconds, getWebcamPreviewTargetTimeSeconds, + shouldSeekWebcamMedia, } from "./webcamSync"; describe("getWebcamPreviewTargetTimeSeconds", () => { @@ -57,3 +58,44 @@ describe("getWebcamMediaTargetTimeSeconds", () => { ).toBe(0); }); }); + +describe("shouldSeekWebcamMedia", () => { + it("does not issue another corrective seek while the webcam element is already seeking", () => { + expect( + shouldSeekWebcamMedia({ + desiredTime: 10.5, + isPlaying: true, + isSeeking: true, + previousTimelineTime: 10, + timelineTime: 10.5, + webcamCurrentTime: 9.8, + }), + ).toBe(false); + }); + + it("seeks when playback drift grows beyond the active threshold", () => { + expect( + shouldSeekWebcamMedia({ + desiredTime: 10.5, + isPlaying: true, + isSeeking: false, + previousTimelineTime: 10, + timelineTime: 10.5, + webcamCurrentTime: 9.8, + }), + ).toBe(true); + }); + + it("does not seek when the clamped media target is already correct", () => { + expect( + shouldSeekWebcamMedia({ + desiredTime: 1 / 60, + isPlaying: false, + isSeeking: false, + previousTimelineTime: 0, + timelineTime: 0, + webcamCurrentTime: 1 / 60, + }), + ).toBe(false); + }); +}); diff --git a/src/components/video-editor/videoPlayback/webcamSync.ts b/src/components/video-editor/videoPlayback/webcamSync.ts index cdcd9df46..27d310a5a 100644 --- a/src/components/video-editor/videoPlayback/webcamSync.ts +++ b/src/components/video-editor/videoPlayback/webcamSync.ts @@ -15,3 +15,29 @@ export function getWebcamMediaTargetTimeSeconds({ } export const getWebcamPreviewTargetTimeSeconds = getWebcamMediaTargetTimeSeconds; + +export function shouldSeekWebcamMedia({ + desiredTime, + isPlaying, + isSeeking, + previousTimelineTime, + timelineTime, + webcamCurrentTime, +}: { + desiredTime: number; + isPlaying: boolean; + isSeeking: boolean; + previousTimelineTime: number | null; + timelineTime: number; + webcamCurrentTime: number; +}): boolean { + if (isSeeking) { + return false; + } + + const timelineJumped = + previousTimelineTime === null || Math.abs(timelineTime - previousTimelineTime) > 0.25; + const driftThreshold = isPlaying ? 0.35 : 0.01; + + return timelineJumped || Math.abs(webcamCurrentTime - desiredTime) > driftThreshold; +} diff --git a/src/lib/exporter/frameRenderer.test.ts b/src/lib/exporter/frameRenderer.test.ts index 651bb08f0..32af67a9d 100644 --- a/src/lib/exporter/frameRenderer.test.ts +++ b/src/lib/exporter/frameRenderer.test.ts @@ -439,6 +439,38 @@ describe("FrameRenderer webcam export path", () => { expect((outputContext as MockContext).drawImage.mock.calls[0][0]).toBe(bubbleCanvas); }); + it("uses the live webcam frame when sync is correct on an offset webcam timeline", () => { + const renderer = createRenderer() as unknown as FrameRendererTestAccess & { + config: { webcam: { timeOffsetMs?: number } }; + }; + const outputContext = createMockContext(); + const webcamVideo = new FakeVideoElement({ + currentTime: 1.75, + readyState: 2, + videoWidth: 800, + videoHeight: 600, + duration: 10, + }); + + renderer.config.webcam = { + ...renderer.config.webcam, + timeOffsetMs: 250, + }; + renderer.webcamVideoElement = webcamVideo; + renderer.lastSyncedWebcamTime = 1.75; + renderer.currentVideoTime = 2; + renderer.animationState.appliedScale = 1; + + renderer.drawWebcamOverlay(outputContext as unknown as CanvasRenderingContext2D, 1280, 720); + + const bubbleCanvas = createdCanvases[0]; + const cacheCanvas = createdCanvases[1]; + expect(cacheCanvas).toBeDefined(); + expect((cacheCanvas.context as MockContext).drawImage.mock.calls[0][0]).toBe(webcamVideo); + expect((bubbleCanvas.context as MockContext).drawImage.mock.calls[0][0]).toBe(cacheCanvas); + expect((outputContext as MockContext).drawImage.mock.calls[0][0]).toBe(bubbleCanvas); + }); + it("reuses the webcam bubble canvas across frames", () => { const renderer = createRenderer() as unknown as FrameRendererTestAccess; const outputContext = createMockContext(); diff --git a/src/lib/exporter/frameRenderer.ts b/src/lib/exporter/frameRenderer.ts index 8b2e335ca..049c25bbc 100644 --- a/src/lib/exporter/frameRenderer.ts +++ b/src/lib/exporter/frameRenderer.ts @@ -2443,16 +2443,22 @@ export class FrameRenderer { bubbleCtx.imageSmoothingEnabled = true; bubbleCtx.imageSmoothingQuality = "high"; + const expectedWebcamTargetTime = getWebcamMediaTargetTimeSeconds({ + currentTime: this.currentVideoTime, + webcamDuration: Number.isFinite(webcamVideo?.duration) ? webcamVideo?.duration : null, + timeOffsetMs: webcam.timeOffsetMs, + }); + const canRefreshCache = hasLiveWebcamFrame && this.lastSyncedWebcamTime !== null && - Math.abs(this.lastSyncedWebcamTime - this.currentVideoTime) <= 0.02 && + Math.abs(this.lastSyncedWebcamTime - expectedWebcamTargetTime) <= 0.02 && (webcamDecodedFrame ? true : Boolean( webcamVideo && !webcamVideo.seeking && - Math.abs(webcamVideo.currentTime - this.currentVideoTime) <= 0.02 && + Math.abs(webcamVideo.currentTime - expectedWebcamTargetTime) <= 0.02 && webcamVideo.videoWidth > 0 && webcamVideo.videoHeight > 0, )); diff --git a/src/lib/exporter/modernFrameRenderer.test.ts b/src/lib/exporter/modernFrameRenderer.test.ts index 718b02eb9..1f43a9c92 100644 --- a/src/lib/exporter/modernFrameRenderer.test.ts +++ b/src/lib/exporter/modernFrameRenderer.test.ts @@ -547,14 +547,14 @@ describe("ModernFrameRenderer webcam export fallback", () => { }; renderer.config.webcamUrl = "file:///tmp/webcam.webm"; - await renderer.setupWebcamSource(); - const syncPromise = renderer.syncWebcamFrame(1); + await renderer.setupWebcamSource(); + const syncPromise = renderer.syncWebcamFrame(1); await vi.advanceTimersByTimeAsync(5_001); - await expect(syncPromise).resolves.toBeUndefined(); + await expect(syncPromise).resolves.toBeUndefined(); - expect(cancelForwardFrameSourceMock).toHaveBeenCalled(); - expect(destroyForwardFrameSourceMock).toHaveBeenCalled(); + expect(cancelForwardFrameSourceMock).toHaveBeenCalled(); + expect(destroyForwardFrameSourceMock).toHaveBeenCalled(); expect(revoke).toHaveBeenCalled(); expect(renderer.webcamForwardFrameSource).toBeNull(); expect(renderer.webcamVideoElement).toBeNull(); From 5fee014186a30e73bb311bc56fdcb55ebc50c1c4 Mon Sep 17 00:00:00 2001 From: webadderall <131426131+webadderall@users.noreply.github.com> Date: Tue, 2 Jun 2026 19:35:12 +1000 Subject: [PATCH 2/2] docs(webcam): document sync helpers --- src/components/video-editor/videoPlayback/webcamSync.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/components/video-editor/videoPlayback/webcamSync.ts b/src/components/video-editor/videoPlayback/webcamSync.ts index 27d310a5a..f7cfbfb40 100644 --- a/src/components/video-editor/videoPlayback/webcamSync.ts +++ b/src/components/video-editor/videoPlayback/webcamSync.ts @@ -1,5 +1,9 @@ import { clampMediaTimeToDuration } from "@/lib/mediaTiming"; +/** + * Maps the editor timeline time to the corresponding webcam media timestamp, + * accounting for any recorded webcam start offset and media duration clamps. + */ export function getWebcamMediaTargetTimeSeconds({ currentTime, webcamDuration, @@ -16,6 +20,10 @@ export function getWebcamMediaTargetTimeSeconds({ export const getWebcamPreviewTargetTimeSeconds = getWebcamMediaTargetTimeSeconds; +/** + * Decides whether the webcam media element needs a corrective seek for the + * current preview frame, while avoiding repeated seeks during active media seeks. + */ export function shouldSeekWebcamMedia({ desiredTime, isPlaying,