From 8f1ecccb4d15b2ebca7fd1ebf477b619670a7a34 Mon Sep 17 00:00:00 2001 From: Akanksha Trehun Date: Tue, 3 Nov 2026 13:00:22 +0530 Subject: [PATCH 1/6] fix(ffmpeg): stop multi-threaded exports from hanging at 100% Production (Netlify) is cross-origin isolated, so every browser loads the multi-threaded core. With FFmpeg choosing its own thread count, core-mt's 32-worker pool deadlocks and every export sits at "Exporting 100%" forever. Reproduced locally with COOP/COEP for MP4, WebM, portrait and no-audio inputs; the single-threaded core was unaffected. Ports the export hardening from the earlier branch (33b2c5a, bf26cf1): - src/lib/ffmpegArgs.ts: one command builder used by the worker (the old copy in ffmpeg.ts was dead code); decoder, filter and encoder threads capped at 4 on the multi-threaded core only - stall watchdog with worker heartbeats: 60 s without activity fails the export with ExportStalledError, and later attempts use the single-threaded core - WebM uses VP8 + Vorbis: libvpx-vp9 and libopus trap in @ffmpeg/core 0.12.10 - SRI hashes for the multi-threaded core files; falls back to the single-threaded core if core-mt fails to load - id-less worker errors, cancel during engine load, and string rejections from wasm traps no longer leave the editor stuck Co-Authored-By: Claude Opus 5.5 --- src/lib/exportEstimate.ts | 2 +- src/lib/exportTelemetry.ts | 4 +- src/lib/ffmpeg.ts | 433 ++++++++--------------- src/lib/ffmpeg.worker.ts | 543 ++++++++--------------------- src/lib/ffmpegArgs.ts | 364 +++++++++++++++++++ src/lib/tests/ffmpegArgs.test.ts | 169 +++++++++ src/lib/tests/ffmpegClient.test.ts | 237 +++++++++++++ src/lib/tests/ffmpegWorker.test.ts | 220 ++++++++++++ 8 files changed, 1272 insertions(+), 700 deletions(-) create mode 100644 src/lib/ffmpegArgs.ts create mode 100644 src/lib/tests/ffmpegArgs.test.ts create mode 100644 src/lib/tests/ffmpegClient.test.ts create mode 100644 src/lib/tests/ffmpegWorker.test.ts diff --git a/src/lib/exportEstimate.ts b/src/lib/exportEstimate.ts index bc4d335bd..aee5aa342 100644 --- a/src/lib/exportEstimate.ts +++ b/src/lib/exportEstimate.ts @@ -55,7 +55,7 @@ function resolutionMultiplier(width: number, height: number): number { // --------------------------------------------------------------------------- // Format overhead factor -// MP4 and MKV are close; WebM (VP9) tends to produce slightly smaller files +// MP4 and MKV are close; WebM (VP8) tends to produce slightly smaller files // at the same CRF, so we apply a small discount. // --------------------------------------------------------------------------- function formatFactor(format: string | undefined): number { diff --git a/src/lib/exportTelemetry.ts b/src/lib/exportTelemetry.ts index 6d0397d0e..497712492 100644 --- a/src/lib/exportTelemetry.ts +++ b/src/lib/exportTelemetry.ts @@ -7,9 +7,9 @@ * Event names and properties are documented in docs/DATADOG.md. */ import { failureCategory, trackAction, trackVital } from "./telemetry"; +import type { EngineMode } from "./ffmpegArgs"; -/** Which FFmpeg core the worker loaded. */ -export type EngineMode = "multi" | "single"; +export type { EngineMode }; export type ExportStage = "engine" | "encode"; diff --git a/src/lib/ffmpeg.ts b/src/lib/ffmpeg.ts index 497646131..7eb543615 100644 --- a/src/lib/ffmpeg.ts +++ b/src/lib/ffmpeg.ts @@ -1,8 +1,10 @@ import { EditRecipe, ExportResult, BackgroundMusicOptions, ImageOverlayOptions, MAX_FILE_SIZE } from "./types"; -import { getPresetById } from "./presets"; -import { buildTextFilter } from "./text-overlay"; import { trackTiming } from "./telemetry"; -import { startEngineLoad, type EngineMode } from "./exportTelemetry"; +import { startEngineLoad } from "./exportTelemetry"; +import type { EngineMode } from "./ffmpegArgs"; + +// The command line itself is built in ffmpegArgs.ts and run by the worker. +export { buildVideoFilter, buildAudioFilter } from "./ffmpegArgs"; export class FFmpegLoadError extends Error { name = "FFmpegLoadError"; @@ -12,6 +14,22 @@ export class FFmpegLoadError extends Error { } } +/** The engine stopped reporting activity mid-export (see STALL_TIMEOUT_MS). */ +export class ExportStalledError extends Error { + name = "ExportStalledError"; + constructor(readonly mode: EngineMode) { + super("Export stopped responding"); + } +} + +/** + * FFmpeg prints a stats line about every 0.5 s while encoding and the worker + * forwards a heartbeat at most every 2 s, so a minute of silence means the + * run is wedged (the multi-threaded core's thread-pool deadlock looked + * exactly like this: stuck at 0% forever). Fail instead of spinning. + */ +export const STALL_TIMEOUT_MS = 60_000; + type SerializedFile = { name: string; @@ -31,7 +49,8 @@ type WorkerExportRequest = { overlayOptions?: ImageOverlayOptions; }; -type WorkerLoadResponse = { type: "ready"; mode?: EngineMode }; +type WorkerLoadResponse = { type: "ready"; mode: EngineMode }; +type WorkerHeartbeatResponse = { type: "heartbeat" }; type WorkerProgressResponse = { type: "progress"; percent: number }; type WorkerResultResponse = { type: "result"; @@ -48,6 +67,7 @@ type WorkerCancelledResponse = { type: "cancelled"; id?: string }; type WorkerResponse = | WorkerLoadResponse + | WorkerHeartbeatResponse | WorkerProgressResponse | WorkerResultResponse | WorkerErrorResponse @@ -63,10 +83,16 @@ let pendingExport: { reject: (reason: unknown) => void; } | null = null; let pendingProgress: ((percent: number) => void) | null = null; - +/** Called on any sign of life from the worker during an export. */ +let pendingActivity: (() => void) | null = null; /** The core the current worker loaded, once it is ready. */ let engineMode: EngineMode | null = null; -/** Telemetry for the engine load in progress, if any (one outcome per worker). */ +/** + * Set after a multi-threaded export stalls: later workers in this page + * session load the single-threaded core even though the page is isolated. + */ +let preferSingleThread = false; +/** Telemetry for the load in progress, if any. */ let engineLoad: ReturnType | null = null; function createWorker(): Worker { @@ -96,12 +122,30 @@ function createWorker(): Worker { return ffmpegWorker; } +/** + * Wraps a progress callback so it only fires when the rounded percentage + * changes. FFmpeg reports progress per encoded frame; forwarding every event + * re-rendered the editor hundreds of times per export for no visible change. + */ +export function dedupeProgress(onProgress?: (percent: number) => void) { + if (!onProgress) return undefined; + let last = -1; + return (percent: number) => { + const rounded = Math.max(0, Math.min(100, Math.round(percent))); + if (rounded === last) return; + last = rounded; + onProgress(rounded); + }; +} + function resetWorker() { // A load that never finished was abandoned (cancelled), not failed. engineLoad?.abandoned(); engineLoad = null; - engineMode = null; + ffmpegWorker?.terminate(); ffmpegWorker = null; + engineMode = null; + pendingActivity = null; workerReady = null; workerReadyResolve = null; workerReadyReject = null; @@ -123,7 +167,13 @@ function handleWorkerMessage(event: MessageEvent) { return; } + if (data.type === "heartbeat") { + pendingActivity?.(); + return; + } + if (data.type === "progress") { + pendingActivity?.(); pendingProgress?.(data.percent); return; } @@ -158,13 +208,14 @@ function handleWorkerMessage(event: MessageEvent) { return; } - const loadError = new FFmpegLoadError(data.message); - if (workerReadyReject) engineLoad?.failed(loadError); + // An error without a matching id means the worker itself failed. Settle + // whatever is waiting on it — an export left pending here used to keep + // the UI on "Exporting" forever. + const error = new FFmpegLoadError(data.message); + if (workerReadyReject) engineLoad?.failed(error); engineLoad = null; - workerReadyReject?.(loadError); - workerReady = null; - workerReadyResolve = null; - workerReadyReject = null; + workerReadyReject?.(error); + pendingExport?.reject(error); resetWorker(); return; } @@ -202,14 +253,20 @@ export async function loadFFmpeg( // 2. Use the captured flag to securely trigger the worker's internal load phase if (isFirstLoad) { const isolated = typeof crossOriginIsolated !== "undefined" && crossOriginIsolated; - engineLoad = startEngineLoad(isolated ? "multi" : "single", isolated ? null : "not_isolated"); - ffmpegWorker!.postMessage({ type: "load" }); + engineLoad = startEngineLoad( + isolated && !preferSingleThread ? "multi" : "single", + !isolated ? "not_isolated" : preferSingleThread ? "after_stall" : null + ); + ffmpegWorker!.postMessage({ type: "load", singleThread: preferSingleThread }); } - pendingProgress = onProgress ?? null; + pendingProgress = dedupeProgress(onProgress) ?? null; + // A load can't be interrupted inside the worker, so cancelling one + // discards the worker; the next attempt starts a fresh one (the core files + // come back from the HTTP cache). if (signal?.aborted) { - ffmpegWorker?.postMessage({ type: "cancel" }); + terminateFFmpeg(); throw new DOMException("Aborted", "AbortError"); } @@ -218,8 +275,7 @@ export async function loadFFmpeg( }; const onAbort = () => { - ffmpegWorker?.postMessage({ type: "cancel" }); - workerReadyReject?.(new DOMException("Aborted", "AbortError")); + terminateFFmpeg(); cleanup(); }; @@ -227,7 +283,7 @@ export async function loadFFmpeg( try { // ffmpeg_load_success / ffmpeg_load_failure are reported once per worker - // by the message handlers, however many callers wait on the same load. + // by handleWorkerMessage, however many callers wait on the same load. await workerReady; if (isFirstLoad) trackTiming("ffmpeg_ready"); } finally { @@ -241,6 +297,7 @@ function cancelPendingExport(reason?: unknown) { pendingExport = null; } pendingProgress = null; + pendingActivity = null; } export async function exportVideo( @@ -249,9 +306,11 @@ export async function exportVideo( onProgress: (percent: number) => void, signal?: AbortSignal, musicOptions?: BackgroundMusicOptions, - overlayOptions?: ImageOverlayOptions + overlayOptions?: ImageOverlayOptions, + /** Duration already read by the editor; skips a second metadata load. */ + knownDuration?: number ): Promise { - await loadFFmpeg(signal, onProgress); + await loadFFmpeg(signal); if (!ffmpegWorker) { throw new Error("FFmpeg worker is not available."); @@ -300,7 +359,7 @@ export async function exportVideo( ? { ...overlayOptions, file: null } : undefined; - pendingProgress = onProgress; + pendingProgress = dedupeProgress(onProgress) ?? null; const exportPromise = new Promise((resolve, reject) => { pendingExport = { id: sessionId, resolve, reject }; @@ -318,6 +377,21 @@ export async function exportVideo( }; signal?.addEventListener("abort", onAbort, { once: true }); + const mode = engineMode ?? "single"; + let lastActivity = Date.now(); + pendingActivity = () => { + lastActivity = Date.now(); + }; + const watchdog = setInterval(() => { + if (Date.now() - lastActivity < STALL_TIMEOUT_MS) return; + clearInterval(watchdog); + if (mode === "multi") preferSingleThread = true; + const stalled = new ExportStalledError(mode); + cancelPendingExport(stalled); + // The worker is wedged and can't be told to stop; discard it. + resetWorker(); + }, 5_000); + const transfers: Transferable[] = [arrayBuffer]; if (musicFilePayload) transfers.push(musicFilePayload.data); if (overlayFilePayload) transfers.push(overlayFilePayload.data); @@ -328,7 +402,10 @@ export async function exportVideo( id: sessionId, file: filePayload, recipe, - videoDuration: await getVideoDuration(file), + videoDuration: + knownDuration && Number.isFinite(knownDuration) && knownDuration > 0 + ? knownDuration + : await getVideoDuration(file), musicFile: musicFilePayload, musicOptions: sanitizedMusicOptions, overlayFile: overlayFilePayload, @@ -340,6 +417,8 @@ export async function exportVideo( try { return await exportPromise; } finally { + clearInterval(watchdog); + pendingActivity = null; signal?.removeEventListener("abort", onAbort); } } @@ -347,30 +426,52 @@ export async function exportVideo( async function getVideoDuration(file: File): Promise { return new Promise((resolve) => { const video = document.createElement("video"); - video.preload = "metadata"; - video.onloadedmetadata = () => { - URL.revokeObjectURL(video.src); - resolve(video.duration); + const url = URL.createObjectURL(file); + const finish = (value: number) => { + clearTimeout(timer); + URL.revokeObjectURL(url); + video.removeAttribute("src"); + resolve(Number.isFinite(value) ? value : 0); }; - video.onerror = () => resolve(0); - video.src = URL.createObjectURL(file); + // 0 makes the worker omit -t, so a stalled metadata read degrades safely. + const timer = setTimeout(() => finish(0), 5000); + video.preload = "metadata"; + video.onloadedmetadata = () => finish(video.duration); + video.onerror = () => finish(0); + video.src = url; }); } -/** The core the worker loaded ("multi" or "single"), or null before it is ready. */ -export function getEngineMode(): EngineMode | null { - return engineMode; -} - export function terminateFFmpeg() { if (ffmpegWorker) { ffmpegWorker.postMessage({ type: "terminate" }); ffmpegWorker.terminate(); } + // Settle a load that is still in flight so its caller never hangs. + workerReadyReject?.(new DOMException("Export cancelled", "AbortError")); cancelPendingExport(new DOMException("Export cancelled", "AbortError")); resetWorker(); } +/** Test/diagnostic helper: whether a worker or export is currently held. */ +export function getFFmpegClientState() { + return { + hasWorker: ffmpegWorker !== null, + hasPendingExport: pendingExport !== null, + hasPendingProgress: pendingProgress !== null, + }; +} + +/** The loaded core ("multi" | "single"), or null before the engine is ready. */ +export function getEngineMode(): EngineMode | null { + return engineMode; +} + +/** Test helper: forget the single-thread preference set by a stall. */ +export function resetEnginePreferenceForTests() { + preferSingleThread = false; +} + function buildSessionId(): string { if (typeof crypto !== "undefined" && "randomUUID" in crypto) { return crypto.randomUUID(); @@ -398,268 +499,6 @@ function buildSessionId(): string { return `${timestamp}-${randomPart}${counterPart}`; } -export function buildVideoFilter(recipe: EditRecipe, targetW: number, targetH: number): string { - const filters: string[] = []; - - if (recipe.trimStart > 0 || recipe.trimEnd !== null) { - // Only use trim filter with precise bounds to avoid scanning entire file - // If trimEnd is null, use duration parameter instead of large placeholder value - if (recipe.trimEnd !== null) { - filters.push(`trim=start=${recipe.trimStart}:end=${recipe.trimEnd}`); - } else if (recipe.trimStart > 0) { - // When only trimStart is set, let FFmpeg infer end (don't use 999999 placeholder) - filters.push(`trim=start=${recipe.trimStart}`); - } - } - - if (recipe.stabilization) { - filters.push("deshake"); - } - - if (recipe.rotate === 90) { - filters.push("transpose=1"); - } else if (recipe.rotate === 180) { - filters.push("transpose=1,transpose=1"); - } else if (recipe.rotate === 270) { - filters.push("transpose=2"); - } - - if (recipe.framing === "fit") { - filters.push( - `scale=${targetW}:${targetH}:force_original_aspect_ratio=decrease`, - `pad=${targetW}:${targetH}:(ow-iw)/2:(oh-ih)/2:color=black` - ); - } else { - filters.push( - `scale=${targetW}:${targetH}:force_original_aspect_ratio=increase`, - `crop=${targetW}:${targetH}` - ); - } - if (recipe.sharpness !== 0) { - filters.push(`unsharp=5:5:${recipe.sharpness}:5:5:0.0`); - } - - // Normalize timestamps only when needed — trim or speed change both - // require a clean 0-based timeline to produce correct output duration. - if (recipe.trimStart > 0 || recipe.trimEnd !== null || recipe.speed !== 1) { - filters.push("setpts=PTS-STARTPTS"); - } - - if (recipe.speed !== 1) { - const pts = (1 / recipe.speed).toFixed(4); - filters.push(`setpts=${pts}*PTS`); - } - - if (recipe.denoise) { - filters.push("hqdn3d=1.5:1.5:6:6"); - } - - const needsEq = - recipe.brightness !== 0 || - recipe.contrast !== 1 || - recipe.saturation !== 1; - - if (needsEq) { - filters.push( - `eq=brightness=${recipe.brightness}:contrast=${recipe.contrast}:saturation=${recipe.saturation}` - ); - } - - // Add text overlays - const textOverlays = recipe.textOverlays || []; - textOverlays.forEach((overlay) => { - filters.push(buildTextFilter(overlay, targetW, targetH)); - }); - - return filters.join(","); -} - -export function buildAudioFilter(recipe: EditRecipe): string { - if (recipe.speed <= 0) return ""; - const filters: string[] = []; - - let remaining = recipe.speed; - while (remaining < 0.5) { - filters.push("atempo=0.5"); - remaining /= 0.5; - } - - while (remaining > 2.0) { - filters.push("atempo=2.0"); - remaining /= 2.0; - } - - if (Math.abs(remaining - 1.0) > 0.001) { - filters.push(`atempo=${Number(remaining.toFixed(4))}`); - } - - if (recipe.volume !== undefined && recipe.volume !== 100) { - filters.push(`volume=${(recipe.volume / 100).toFixed(2)}`); - } - - if (recipe.normalizeAudio) filters.push("loudnorm=I=-14:TP=-1.5:LRA=11"); - - return filters.join(","); -} - -function buildAudioTrimFilter(recipe: EditRecipe): string { - if (recipe.trimStart === 0 && recipe.trimEnd === null) return ""; - - // Avoid scanning entire audio with large placeholder values - // Use precise trim bounds when available - let trimFilter = `atrim=start=${recipe.trimStart}`; - if (recipe.trimEnd !== null) { - trimFilter += `:end=${recipe.trimEnd}`; - } - // asetpts normalizes timestamps after trim for correct stream positioning - return `${trimFilter},asetpts=PTS-STARTPTS`; -} - -function buildArguments( - recipe: EditRecipe, - format: "mp4" | "webm" | "mkv" | "gif", - outputName: string, - inputName: string, - targetW: number, - targetH: number, - hasMusicTrack: boolean, - musicInputName: string, - musicOptions: BackgroundMusicOptions | undefined, - hasOverlay: boolean, - overlayInputName: string, - overlayOptions: ImageOverlayOptions | undefined, - hasOriginalAudio: boolean, - videoDuration: number -): string[] { - const vf = buildVideoFilter(recipe, targetW, targetH); - const audioTrim = hasOriginalAudio ? buildAudioTrimFilter(recipe) : ""; - const audioSpeed = hasOriginalAudio ? buildAudioFilter(recipe) : ""; - const afParts = [audioTrim, audioSpeed].filter(Boolean); - const af = afParts.join(","); - - const musicIdx = 1; - const overlayIdx = hasMusicTrack ? 2 : 1; - - const args: string[] = []; - args.push("-i", inputName); - if (hasMusicTrack) { - if (musicOptions!.loopMusic) args.push("-stream_loop", "-1"); - args.push("-i", musicInputName); - } - if (hasOverlay) { - args.push("-i", overlayInputName); - } - - const needsFilterComplex = hasOverlay || hasMusicTrack; - const shouldKeepAudio = recipe.keepAudio && (hasOriginalAudio || hasMusicTrack); - - if (needsFilterComplex) { - const filterParts: string[] = []; - let videoOut = "[0:v]"; - - if (vf) { - filterParts.push(`[0:v]${vf}[vbase]`); - videoOut = "[vbase]"; - } - -if (hasOverlay) { - const scaledW = overlayOptions!.size; - const alpha = (overlayOptions!.opacity / 100).toFixed(2); - const posMap: Record = { - "top-left": "20:20", - "top-right": "main_w-w-20:20", - "bottom-left": "20:main_h-h-20", - "bottom-right": "main_w-w-20:main_h-h-20", - }; - -interface PositionCoords { - x: number; - y: number; - } - - const pos = typeof overlayOptions?.position === "string" - ? (posMap[overlayOptions.position] ?? "main_w-w-20:main_h-h-20") - : overlayOptions?.position - ? `(main_w)*${(overlayOptions.position as PositionCoords).x}/100:(main_h)*${(overlayOptions.position as PositionCoords).y}/100` - : "main_w-w-20:main_h-h-20"; - - filterParts.push(`[${overlayIdx}:v]scale=${scaledW}:-2,format=rgba,colorchannelmixer=aa=${alpha}[logo]`); - filterParts.push(`${videoOut}[logo]overlay=${pos}[vout]`); - videoOut = "[vout]"; -} - - let audioOut = ""; - if (shouldKeepAudio) { - if (hasMusicTrack) { - const musicVol = (musicOptions!.musicVolume / 100).toFixed(2); - if (hasOriginalAudio) { - const origVol = (musicOptions!.originalAudioVolume / 100).toFixed(2); - const origChain = afParts.length > 0 - ? `[0:a]${afParts.join(",")},volume=${origVol}[orig]` - : `[0:a]volume=${origVol}[orig]`; - filterParts.push(origChain); - filterParts.push(`[${musicIdx}:a]volume=${musicVol}[music]`); - filterParts.push(`[orig][music]amix=inputs=2:duration=first:dropout_transition=0[aout]`); - audioOut = "[aout]"; - } else { - filterParts.push(`[${musicIdx}:a]volume=${musicVol}[aout]`); - audioOut = "[aout]"; - } - } else if (hasOriginalAudio && af) { - filterParts.push(`[0:a]${af}[aout]`); - audioOut = "[aout]"; - } - } - - if (filterParts.length > 0) { - args.push("-filter_complex", filterParts.join(";")); - } - args.push("-map", videoOut === "[0:v]" ? "0:v" : videoOut); - - if (!shouldKeepAudio) { - args.push("-an"); - } else if (audioOut) { - args.push("-map", audioOut); - } else if (hasOriginalAudio) { - args.push("-map", "0:a"); - } - } else { - if (vf) args.push("-vf", vf); - if (!shouldKeepAudio) { - args.push("-an"); - } else if (af && hasOriginalAudio) { - args.push("-af", af); - } - } - - if (format === "webm") { - args.push( - "-c:v", "libvpx-vp9", - "-b:v", "0", - "-crf", String(recipe.quality), - "-cpu-used", "4", - "-deadline", "realtime" - ); - if (shouldKeepAudio) args.push("-c:a", "libopus"); - } else if (format === "mkv") { - args.push("-c:v", "libx264", "-crf", String(recipe.quality), "-preset", "ultrafast"); - if (shouldKeepAudio) args.push("-c:a", "aac", "-b:a", "128k"); - } else { - args.push("-c:v", "libx264", "-crf", String(recipe.quality), "-preset", "ultrafast", "-movflags", "+faststart"); - if (shouldKeepAudio) args.push("-c:a", "aac", "-b:a", "128k"); - } - - // Add explicit output duration to prevent slight duration overshoot - // caused by encoder/filter pipeline frame flush at stream end, - // and to prevent infinite loops when adding looped audio to silent videos. - const sourceDuration = (recipe.trimEnd ?? videoDuration) - recipe.trimStart; - const outputDuration = sourceDuration / recipe.speed; - args.push("-t", outputDuration.toFixed(6)); - - args.push(outputName); - return args; -} - export function formatBytes(bytes: number): string { if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; diff --git a/src/lib/ffmpeg.worker.ts b/src/lib/ffmpeg.worker.ts index 52fbed3e0..72ed5ee5e 100644 --- a/src/lib/ffmpeg.worker.ts +++ b/src/lib/ffmpeg.worker.ts @@ -2,7 +2,7 @@ import { FFmpeg } from "@ffmpeg/ffmpeg"; import { toBlobURL } from "@ffmpeg/util"; import { EditRecipe, BackgroundMusicOptions, ImageOverlayOptions } from "./types"; import { getPresetById } from "./presets"; -import { buildTextFilter } from "./text-overlay"; +import { buildArguments, buildGifArguments, resolveThreadCount, type EngineMode } from "./ffmpegArgs"; const CORE_BASE_URL = "https://cdn.jsdelivr.net/npm/@ffmpeg/core@0.12.10/dist/umd"; // MT core aligned to the same version and distribution format as CORE_BASE_URL. @@ -15,13 +15,17 @@ const CORE_BASE_URL = "https://cdn.jsdelivr.net/npm/@ffmpeg/core@0.12.10/dist/um const MT_CORE_BASE_URL = "https://cdn.jsdelivr.net/npm/@ffmpeg/core-mt@0.12.10/dist/umd"; // Keys are full URLs so hashes for the non-MT and MT packages never collide. -// MT hashes are omitted intentionally: fetchWithIntegrity falls back to a plain -// fetch when no hash entry exists, which is safe for a trusted CDN origin and -// avoids the previous bug where a wrong hash caused a hard IntegrityError. -// Add MT-specific sha384 entries here once they are verified against the CDN. -const SRI_HASHES: Record = { - [`${CORE_BASE_URL}/ffmpeg-core.js`]: "sha384-sKfkiFtvUk+vexk+0EUhEh366190/4WpgUAsUvaxEfyg7+E1Zt5Y5hrsU808g8Q9", - [`${CORE_BASE_URL}/ffmpeg-core.wasm`]: "sha384-U1VDhkPYrM3wTCT4/vjSpSsKqG/UjljYrYCI4hBSJ02svbCkxuCi6U6u/peg5vpW", +// Every engine file is integrity-checked, so a tampered CDN response fails to +// load instead of running. Each hash was computed from the npm tarball (itself +// verified against the registry's sha512 `dist.integrity`) and matched against +// what jsDelivr serves (2026-10-22, docs/SECURITY-PIPELINE.md). When bumping +// @ffmpeg/core*, recompute all of them the same way. +export const SRI_HASHES: Record = { + [`${CORE_BASE_URL}/ffmpeg-core.js`]: "sha384-sKfkiFtvUk+vexk+0EUhEh366190/4WpgUAsUvaxEfyg7+E1Zt5Y5hrsU808g8Q9", + [`${CORE_BASE_URL}/ffmpeg-core.wasm`]: "sha384-U1VDhkPYrM3wTCT4/vjSpSsKqG/UjljYrYCI4hBSJ02svbCkxuCi6U6u/peg5vpW", + [`${MT_CORE_BASE_URL}/ffmpeg-core.js`]: "sha384-91GpZ3Ow5AIa5wmICIKrQaOAmJPc0sPFNaeuZKtD1Dx7Xr/F8kfGZHM++VOj308v", + [`${MT_CORE_BASE_URL}/ffmpeg-core.wasm`]: "sha384-IXnr5PE2UFcQ5DvI5LyubPqmMF46EkyIMlbdn4CNQR1iQ8/2irEkyhDFnVDxv4f/", + [`${MT_CORE_BASE_URL}/ffmpeg-core.worker.js`]: "sha384-h19AXK35916sCbdJrzAeYw2kl/zEt4B6PTy94TTPup7kgO7r/7mwb43Orcu+xTW1", }; type SerializedFile = { @@ -42,7 +46,11 @@ type ExportRequest = { overlayOptions?: ImageOverlayOptions; }; -type LoadRequest = { type: "load" }; +type LoadRequest = { + type: "load"; + /** Load the single-threaded core even when cross-origin isolated (after a stall). */ + singleThread?: boolean; +}; type CancelRequest = { type: "cancel" }; @@ -52,8 +60,9 @@ type WorkerCommand = LoadRequest | ExportRequest | CancelRequest | TerminateRequ type ProgressPayload = { type: "progress"; percent: number }; -/** `mode`: which core loaded, multi-threaded (cross-origin isolated) or single. */ -type ReadyPayload = { type: "ready"; mode: "multi" | "single" }; +type ReadyPayload = { type: "ready"; mode: EngineMode }; + +type HeartbeatPayload = { type: "heartbeat" }; type ResultPayload = { type: "result"; @@ -70,305 +79,80 @@ type ErrorPayload = { type: "error"; id?: string; message: string }; type CancelledPayload = { type: "cancelled"; id?: string }; -type WorkerResponse = ProgressPayload | ReadyPayload | ResultPayload | ErrorPayload | CancelledPayload; +type WorkerResponse = ProgressPayload | ReadyPayload | HeartbeatPayload | ResultPayload | ErrorPayload | CancelledPayload; let ffmpeg: FFmpeg | null = null; let ffmpegLoaded = false; -let engineMode: "multi" | "single" = "single"; +/** Which core actually loaded — decides the thread flags, not crossOriginIsolated alone. */ +let engineMode: EngineMode = "single"; let activeExportAbortController: AbortController | null = null; let activeExportId: string | null = null; +/** Liveness signal for the client's stall watchdog (see ffmpeg.ts). */ +const HEARTBEAT_INTERVAL_MS = 2000; +let lastHeartbeat = 0; +function heartbeat() { + const now = Date.now(); + if (now - lastHeartbeat < HEARTBEAT_INTERVAL_MS) return; + lastHeartbeat = now; + postMessage({ type: "heartbeat" }); +} + async function fetchWithIntegrity(url: string, mimeType: string): Promise { // Look up by full URL so non-MT and MT hashes never collide when filenames match. const integrity = SRI_HASHES[url]; - // Fallback to standard fetch if SRI is missing (Prevents ffmpeg-core.worker.js from crashing the thread) - if (!integrity) { - const response = await fetch(url, { credentials: "omit" }); - const blob = new Blob([await response.arrayBuffer()], { type: mimeType }); - return URL.createObjectURL(blob); - } + // Never fetch an engine file unverified. + if (!integrity) throw new Error(`No integrity hash for engine file: ${url}`); const response = await fetch(url, { integrity, credentials: "omit" }); + if (!response.ok) throw new Error(`Failed to fetch engine file (HTTP ${response.status})`); const blob = new Blob([await response.arrayBuffer()], { type: mimeType }); return URL.createObjectURL(blob); } -function buildVideoFilter(recipe: EditRecipe, targetW: number, targetH: number): string { - const filters: string[] = []; - - if (recipe.trimStart > 0 || recipe.trimEnd !== null) { - const end = recipe.trimEnd !== null ? recipe.trimEnd : 999999; - filters.push(`trim=start=${recipe.trimStart}:end=${end}`); - } - - if (recipe.stabilization) { - filters.push("deshake"); - } - - if (recipe.rotate === 90) { - filters.push("transpose=1"); - } else if (recipe.rotate === 180) { - filters.push("transpose=1,transpose=1"); - } else if (recipe.rotate === 270) { - filters.push("transpose=2"); - } - - if (recipe.framing === "fit") { - filters.push( - `scale=${targetW}:${targetH}:force_original_aspect_ratio=decrease`, - `pad=${targetW}:${targetH}:(ow-iw)/2:(oh-ih)/2:color=black` - ); - } else { - filters.push( - `scale=${targetW}:${targetH}:force_original_aspect_ratio=increase`, - `crop=${targetW}:${targetH}` - ); - } - - if (recipe.trimStart > 0 || recipe.trimEnd !== null || recipe.speed !== 1) { - filters.push("setpts=PTS-STARTPTS"); - } - - if (recipe.speed !== 1) { - const pts = (1 / recipe.speed).toFixed(4); - filters.push(`setpts=${pts}*PTS`); - } - - if (recipe.denoise) { - filters.push("hqdn3d=1.5:1.5:6:6"); - } - - const needsEq = - recipe.brightness !== 0 || - recipe.contrast !== 1 || - recipe.saturation !== 1; - - if (needsEq) { - filters.push( - `eq=brightness=${recipe.brightness}:contrast=${recipe.contrast}:saturation=${recipe.saturation}` - ); - } - - const textOverlays = recipe.textOverlays || []; - textOverlays.forEach((overlay) => { - filters.push(buildTextFilter(overlay, targetW, targetH)); - }); - - return filters.join(","); -} - -function buildAudioFilter(speed: number, normalizeAudio: boolean): string { - if (speed <= 0) return ""; - const filters: string[] = []; - - let remaining = speed; - while (remaining < 0.5) { - filters.push("atempo=0.5"); - remaining /= 0.5; - } - - while (remaining > 2.0) { - filters.push("atempo=2.0"); - remaining /= 2.0; - } - - if (Math.abs(remaining - 1.0) > 0.001) { - filters.push(`atempo=${Number(remaining.toFixed(4))}`); - } - - if (normalizeAudio) filters.push("loudnorm=I=-14:TP=-1.5:LRA=11"); - - return filters.join(","); -} - -function buildAudioTrimFilter(recipe: EditRecipe): string { - if (recipe.trimStart === 0 && recipe.trimEnd === null) return ""; - const end = recipe.trimEnd !== null ? recipe.trimEnd : 999999; - return `atrim=start=${recipe.trimStart}:end=${end},asetpts=PTS-STARTPTS`; -} - -function buildArguments( - recipe: EditRecipe, - format: "mp4" | "webm" | "mkv" | "gif", - outputName: string, - inputName: string, - targetW: number, - targetH: number, - hasMusicTrack: boolean, - musicInputName: string, - musicOptions: BackgroundMusicOptions | undefined, - hasOverlay: boolean, - overlayInputName: string, - overlayOptions: ImageOverlayOptions | undefined, - hasOriginalAudio: boolean, - videoDuration: number -): string[] { - const vf = buildVideoFilter(recipe, targetW, targetH); - const audioTrim = hasOriginalAudio ? buildAudioTrimFilter(recipe) : ""; - const audioSpeed = hasOriginalAudio ? buildAudioFilter(recipe.speed, recipe.normalizeAudio ?? false) : ""; - const afParts = [audioTrim, audioSpeed].filter(Boolean); - const af = afParts.join(","); - - const musicIdx = 1; - const overlayIdx = hasMusicTrack ? 2 : 1; - - const args: string[] = []; - args.push("-i", inputName); - if (hasMusicTrack) { - if (musicOptions!.loopMusic) args.push("-stream_loop", "-1"); - args.push("-i", musicInputName); - } - if (hasOverlay) { - args.push("-i", overlayInputName); - } - - const needsFilterComplex = hasOverlay || hasMusicTrack; - const shouldKeepAudio = recipe.keepAudio && (hasOriginalAudio || hasMusicTrack); - - if (needsFilterComplex) { - const filterParts: string[] = []; - let videoOut = "[0:v]"; - - if (vf) { - filterParts.push(`[0:v]${vf}[vbase]`); - videoOut = "[vbase]"; - } - -if (hasOverlay) { - const scaledW = overlayOptions!.size; - const alpha = (overlayOptions!.opacity / 100).toFixed(2); - const posMap: Record = { - "top-left": "20:20", - "top-right": "W-w-20:20", - "bottom-left": "20:H-h-20", - "bottom-right": "W-w-20:H-h-20", - }; - -interface PositionCoords { - x: number; - y: number; - } - - const pos = typeof overlayOptions?.position === "string" - ? (posMap[overlayOptions.position] ?? "W-w-20:H-h-20") - : overlayOptions?.position - ? `(W-w)*${(overlayOptions.position as PositionCoords).x}/100:(H-h)*${(overlayOptions.position as PositionCoords).y}/100` - : "W-w-20:H-h-20"; - - filterParts.push(`[${overlayIdx}:v]scale=${scaledW}:-2,format=rgba,colorchannelmixer=aa=${alpha}[logo]`); - filterParts.push(`${videoOut}[logo]overlay=${pos}[vout]`); - videoOut = "[vout]"; -} - - let audioOut = ""; - if (shouldKeepAudio) { - if (hasMusicTrack) { - const musicVol = (musicOptions!.musicVolume / 100).toFixed(2); - if (hasOriginalAudio) { - const origVol = (musicOptions!.originalAudioVolume / 100).toFixed(2); - const origChain = afParts.length > 0 - ? `[0:a]${afParts.join(",")},volume=${origVol}[orig]` - : `[0:a]volume=${origVol}[orig]`; - filterParts.push(origChain); - filterParts.push(`[${musicIdx}:a]volume=${musicVol}[music]`); - filterParts.push(`[orig][music]amix=inputs=2:duration=first:dropout_transition=0[aout]`); - audioOut = "[aout]"; - } else { - filterParts.push(`[${musicIdx}:a]volume=${musicVol}[aout]`); - audioOut = "[aout]"; - } - } else if (hasOriginalAudio && af) { - filterParts.push(`[0:a]${af}[aout]`); - audioOut = "[aout]"; - } - } - - if (filterParts.length > 0) { - args.push("-filter_complex", filterParts.join(";")); - } - args.push("-map", videoOut === "[0:v]" ? "0:v" : videoOut); - - if (!shouldKeepAudio) { - args.push("-an"); - } else if (audioOut) { - args.push("-map", audioOut); - } else if (hasOriginalAudio) { - args.push("-map", "0:a"); - } - } else { - if (vf) args.push("-vf", vf); - if (!shouldKeepAudio) { - args.push("-an"); - } else if (af && hasOriginalAudio) { - args.push("-af", af); - } - } - - if (format === "webm") { - args.push( - "-c:v", "libvpx-vp9", - "-b:v", "0", - "-crf", String(recipe.quality), - "-cpu-used", "4", - "-deadline", "realtime" - ); - if (shouldKeepAudio) args.push("-c:a", "libopus"); - } else if (format === "mkv") { - args.push("-c:v", "libx264", "-crf", String(recipe.quality), "-preset", "ultrafast"); - if (shouldKeepAudio) args.push("-c:a", "aac", "-b:a", "128k"); - } else { - args.push("-c:v", "libx264", "-crf", String(recipe.quality), "-preset", "ultrafast", "-movflags", "+faststart"); - if (shouldKeepAudio) args.push("-c:a", "aac", "-b:a", "128k"); - } - - if (recipe.speed !== 1) { - const sourceDuration = (recipe.trimEnd ?? videoDuration) - recipe.trimStart; - // Skip -t when sourceDuration <= 0 (happens when video metadata fails to - // load and videoDuration falls back to 0). Passing -t 0 to FFmpeg produces - // a zero-length output file; omitting it lets FFmpeg encode the full stream. - if (sourceDuration > 0) { - const outputDuration = sourceDuration / recipe.speed; - args.push("-t", outputDuration.toFixed(6)); - } - } - - args.push(outputName); - return args; -} - -async function loadCore(onProgress?: (percent: number) => void): Promise { - if (ffmpegLoaded) { - onProgress?.(100); - return; - } - +async function loadCoreFrom(mode: EngineMode): Promise { ffmpeg = new FFmpeg(); - - const isIsolated = typeof self !== "undefined" && self.crossOriginIsolated; - const baseURL = isIsolated ? MT_CORE_BASE_URL : CORE_BASE_URL; - engineMode = isIsolated ? "multi" : "single"; - - const handleProgress = ({ progress }: { progress: number }) => { - onProgress?.(Math.round(progress * 100)); - }; - - ffmpeg.on("progress", handleProgress); - + const baseURL = mode === "multi" ? MT_CORE_BASE_URL : CORE_BASE_URL; try { await ffmpeg.load({ coreURL: await fetchWithIntegrity(`${baseURL}/ffmpeg-core.js`, "text/javascript"), wasmURL: await fetchWithIntegrity(`${baseURL}/ffmpeg-core.wasm`, "application/wasm"), - ...(isIsolated && { + ...(mode === "multi" && { workerURL: await fetchWithIntegrity(`${baseURL}/ffmpeg-core.worker.js`, "text/javascript"), }), }); + } catch (error) { + ffmpeg.terminate(); + ffmpeg = null; + throw error; + } + engineMode = mode; + ffmpegLoaded = true; +} - ffmpegLoaded = true; - onProgress?.(100); - } finally { - ffmpeg.off("progress", handleProgress); +async function loadCore(singleThread = false): Promise { + if (ffmpegLoaded) return; + + // The multi-threaded core needs SharedArrayBuffer, which only exists when + // the page is cross-origin isolated (COOP + COEP headers). + const canUseMulti = !singleThread && typeof self !== "undefined" && self.crossOriginIsolated; + if (!canUseMulti) { + await loadCoreFrom("single"); + return; } + try { + await loadCoreFrom("multi"); + } catch { + // e.g. the MT files failed to download or instantiate. The single-threaded + // core is slower but works everywhere, so fall back rather than fail. + await loadCoreFrom("single"); + } +} + +/** ffmpeg.exec rejects with a plain string when the wasm core traps. */ +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); } function serializeFileBuffer(file: SerializedFile): Uint8Array { @@ -451,52 +235,64 @@ async function runExport(request: ExportRequest): Promise { } const videoDuration = request.videoDuration; + const threads = resolveThreadCount(engineMode, self.navigator?.hardwareConcurrency); + const signal = activeExportAbortController?.signal; + // FFmpeg emits progress per frame; post only when the whole percentage + // changes so the main thread isn't flooded with identical updates. + let lastPercent = -1; const handleProgress = ({ progress }: { progress: number }) => { if (activeExportId !== sessionId) return; - postMessage({ type: "progress", percent: Math.min(99, Math.round(progress * 100)) }); + heartbeat(); + const percent = Math.max(0, Math.min(99, Math.round(progress * 100))); + if (percent === lastPercent) return; + lastPercent = percent; + postMessage({ type: "progress", percent }); + }; + + // FFmpeg prints a stats line about twice a second while it works, so log + // output doubles as a heartbeat even when progress can't be computed. + let missingAudioDetected = false; + const handleLog = ({ message }: { message: string }) => { + heartbeat(); + const msg = message.toLowerCase(); + if ( + msg.includes("matches no streams") || + msg.includes("specifier '0:a'") || + msg.includes("input pad 0 on filter src") + ) { + missingAudioDetected = true; + } }; - let logListener: ((event: { message: string }) => void) | null = null; ffmpeg.on("progress", handleProgress); + ffmpeg.on("log", handleLog); + lastHeartbeat = 0; + heartbeat(); + + const read = async (name: string) => { + const data = await ffmpeg!.readFile(name, undefined, { signal }); + return (data as Uint8Array).buffer as ArrayBuffer; + }; try { if (recipe.format === "gif") { - const vf = buildVideoFilter(recipe, targetW, targetH); - const vfWithPalette = vf ? `${vf},palettegen` : "palettegen"; - const vfWithPaletteUse = vf - ? `[0:v]${vf}[x];[x][1:v]paletteuse` - : "[0:v][1:v]paletteuse"; - - const gifDurationArgs = recipe.speed !== 1 - ? (() => { - const sourceDuration = (recipe.trimEnd ?? videoDuration) - recipe.trimStart; - // Skip -t when sourceDuration <= 0 (metadata load failure fallback) - // to avoid producing a zero-length GIF. - if (sourceDuration <= 0) return []; - const outputDuration = sourceDuration / recipe.speed; - return ["-t", outputDuration.toFixed(6)]; - })() - : []; - - const pass1Code = await ffmpeg.exec( - ["-i", inputName, "-vf", vfWithPalette, ...gifDurationArgs, "-y", paletteName], - undefined, - { signal: activeExportAbortController?.signal } - ); + const gif = buildGifArguments({ + recipe, + inputName, + paletteName, + outputName, + targetW, + targetH, + videoDuration, + threads, + }); + const pass1Code = await ffmpeg.exec(gif.palette, undefined, { signal }); if (pass1Code !== 0) throw new Error("GIF palette generation failed"); - - const pass2Code = await ffmpeg.exec( - ["-i", inputName, "-i", paletteName, "-lavfi", vfWithPaletteUse, ...gifDurationArgs, "-y", outputName], - undefined, - { signal: activeExportAbortController?.signal } - ); + const pass2Code = await ffmpeg.exec(gif.render, undefined, { signal }); if (pass2Code !== 0) throw new Error("GIF export failed"); - const data = await ffmpeg.readFile(outputName, undefined, { - signal: activeExportAbortController?.signal, - }); - const payload = (data as Uint8Array).buffer as ArrayBuffer; + const payload = await read(outputName); return { type: "result", id: sessionId, @@ -509,90 +305,38 @@ async function runExport(request: ExportRequest): Promise { }; } - let missingAudioDetected = false; - const logListener = ({ message }: { message: string }) => { - const msg = message.toLowerCase(); - if ( - msg.includes("matches no streams") || - msg.includes("specifier '0:a'") || - msg.includes("input pad 0 on filter src") - ) { - missingAudioDetected = true; - } - }; - ffmpeg.on("log", logListener); - - let args = buildArguments( - recipe, - recipe.format, - outputName, - inputName, - targetW, - targetH, - hasMusicTrack, - musicInputName, - request.musicOptions, - hasOverlay, - overlayInputName, - request.overlayOptions, - true, - videoDuration - ); - - let exitCode = await ffmpeg.exec(args, undefined, { - signal: activeExportAbortController?.signal, - }); - - if (exitCode !== 0 && missingAudioDetected) { - missingAudioDetected = false; - args = buildArguments( + const argsFor = (format: "mp4" | "webm" | "mkv", output: string, hasOriginalAudio: boolean) => + buildArguments({ recipe, - recipe.format, - outputName, + format, inputName, + outputName: output, targetW, targetH, - hasMusicTrack, - musicInputName, - request.musicOptions, - hasOverlay, - overlayInputName, - request.overlayOptions, - false, - videoDuration - ); - exitCode = await ffmpeg.exec(args, undefined, { - signal: activeExportAbortController?.signal, + videoDuration, + hasOriginalAudio, + music: hasMusicTrack ? { inputName: musicInputName, options: request.musicOptions! } : undefined, + overlay: hasOverlay ? { inputName: overlayInputName, options: request.overlayOptions! } : undefined, + threads, }); + + let exitCode = await ffmpeg.exec(argsFor(recipe.format, outputName, true), undefined, { signal }); + + if (exitCode !== 0 && missingAudioDetected) { + // Retry without the original audio track (the source has none). + exitCode = await ffmpeg.exec(argsFor(recipe.format, outputName, false), undefined, { signal }); } if (exitCode !== 0) { - args = buildArguments( - recipe, - "webm", - fallbackOutputName, - inputName, - targetW, - targetH, - hasMusicTrack, - musicInputName, - request.musicOptions, - hasOverlay, - overlayInputName, - request.overlayOptions, - !missingAudioDetected, - videoDuration + // Last resort: WebM (VP8), for inputs the H.264 path rejects. + const fallbackCode = await ffmpeg.exec( + argsFor("webm", fallbackOutputName, !missingAudioDetected), + undefined, + { signal } ); - - const fallbackCode = await ffmpeg.exec(args, undefined, { - signal: activeExportAbortController?.signal, - }); if (fallbackCode !== 0) throw new Error("Export failed"); - const data = await ffmpeg.readFile(fallbackOutputName, undefined, { - signal: activeExportAbortController?.signal, - }); - const payload = (data as Uint8Array).buffer as ArrayBuffer; + const payload = await read(fallbackOutputName); return { type: "result", id: sessionId, @@ -605,10 +349,7 @@ async function runExport(request: ExportRequest): Promise { }; } - const data = await ffmpeg.readFile(outputName, undefined, { - signal: activeExportAbortController?.signal, - }); - const payload = (data as Uint8Array).buffer as ArrayBuffer; + const payload = await read(outputName); return { type: "result", id: sessionId, @@ -620,8 +361,10 @@ async function runExport(request: ExportRequest): Promise { format: recipe.format, }; } finally { + // Both listeners must come off: the worker outlives each export, and a + // leaked log listener used to accumulate on every run. ffmpeg.off("progress", handleProgress); - if (logListener) ffmpeg.off("log", logListener); + ffmpeg.off("log", handleLog); for (const path of cleanupFiles) { await removeFile(path); } @@ -656,10 +399,10 @@ async function handleCommand(message: WorkerCommand) { switch (message.type) { case "load": { try { - await loadCore(); + await loadCore(message.singleThread); postMessage({ type: "ready", mode: engineMode }); } catch (error) { - postMessage({ type: "error", message: (error as Error).message }); + postMessage({ type: "error", message: errorMessage(error) }); } return; } @@ -687,7 +430,7 @@ async function handleCommand(message: WorkerCommand) { if (activeExportAbortController?.signal.aborted) { postMessage({ type: "cancelled", id: message.id }); } else { - postMessage({ type: "error", id: message.id, message: (error as Error).message }); + postMessage({ type: "error", id: message.id, message: errorMessage(error) }); } } finally { activeExportAbortController = null; @@ -713,6 +456,6 @@ async function handleCommand(message: WorkerCommand) { self.addEventListener("message", (event) => { handleCommand(event.data as WorkerCommand).catch((error) => { - postMessage({ type: "error", message: (error as Error).message }); + postMessage({ type: "error", message: errorMessage(error) }); }); }); diff --git a/src/lib/ffmpegArgs.ts b/src/lib/ffmpegArgs.ts new file mode 100644 index 000000000..09f623a51 --- /dev/null +++ b/src/lib/ffmpegArgs.ts @@ -0,0 +1,364 @@ +import { EditRecipe, BackgroundMusicOptions, ImageOverlayOptions, OverlayPosition } from "./types"; +import { buildTextFilter } from "./text-overlay"; + +// The FFmpeg command line for every export is built here and only here. +// ffmpeg.worker.ts runs these commands; ffmpeg.ts re-exports the filter +// builders for tests. There used to be a second copy in ffmpeg.ts that tests +// exercised while the worker ran its own — sharpness, volume and the `-t` fix +// all landed in the copy that never ran. Keep this module free of DOM/worker +// APIs so it can be imported from both sides. + +export type OutputFormat = "mp4" | "webm" | "mkv" | "gif"; +export type EngineMode = "multi" | "single"; + +/** + * Upper bound on threads per FFmpeg stage (decode, filter, encode) for the + * multi-threaded core. @ffmpeg/core-mt pre-spawns a pool of 32 pthread + * workers; a thread beyond the pool needs the FFmpeg worker's event loop to + * start, but that loop is blocked inside `exec`, so the run deadlocks at 0%. + * With "auto" threading x264/libvpx and the decoder each size themselves from + * the CPU count and together exceed the pool on many-core machines. Measured + * on an 11-core M3 Pro (Chrome 155): auto hung, 4 completed ~4× faster than + * the single-threaded core. + */ +export const MT_THREAD_CAP = 4; + +/** Threads to request per stage; 0 means "don't pass any thread flags". */ +export function resolveThreadCount(mode: EngineMode, hardwareConcurrency?: number): number { + if (mode !== "multi") return 0; + const cores = + hardwareConcurrency && Number.isFinite(hardwareConcurrency) && hardwareConcurrency > 0 + ? Math.floor(hardwareConcurrency) + : MT_THREAD_CAP; + return Math.max(1, Math.min(MT_THREAD_CAP, cores)); +} + +/** Decoder thread flag, placed before an `-i`. */ +function inputThreadArgs(threads: number): string[] { + return threads > 0 ? ["-threads", String(threads)] : []; +} + +/** Encoder and filter-graph thread flags, placed before the output name. */ +function outputThreadArgs(threads: number, hasFilterComplex: boolean): string[] { + if (threads <= 0) return []; + const args = ["-threads", String(threads), "-filter_threads", String(threads)]; + if (hasFilterComplex) args.push("-filter_complex_threads", String(threads)); + return args; +} + +/** + * Explicit output duration. Stops encoder flush overshoot and, with looped + * background music (`-stream_loop -1`) on a silent video, an export that + * never ends. Omitted when the duration is unknown (metadata read failed): + * `-t 0` would produce an empty file. + */ +export function outputDurationArgs(recipe: EditRecipe, videoDuration: number): string[] { + const sourceDuration = (recipe.trimEnd ?? videoDuration) - recipe.trimStart; + if (!(sourceDuration > 0) || !(recipe.speed > 0)) return []; + return ["-t", (sourceDuration / recipe.speed).toFixed(6)]; +} + +export function buildVideoFilter(recipe: EditRecipe, targetW: number, targetH: number): string { + const filters: string[] = []; + + if (recipe.trimStart > 0 || recipe.trimEnd !== null) { + // Precise bounds only; an open end lets FFmpeg infer it instead of a + // large placeholder that makes it scan the whole file. + if (recipe.trimEnd !== null) { + filters.push(`trim=start=${recipe.trimStart}:end=${recipe.trimEnd}`); + } else if (recipe.trimStart > 0) { + filters.push(`trim=start=${recipe.trimStart}`); + } + } + + if (recipe.stabilization) { + filters.push("deshake"); + } + + if (recipe.rotate === 90) { + filters.push("transpose=1"); + } else if (recipe.rotate === 180) { + filters.push("transpose=1,transpose=1"); + } else if (recipe.rotate === 270) { + filters.push("transpose=2"); + } + + if (recipe.framing === "fit") { + filters.push( + `scale=${targetW}:${targetH}:force_original_aspect_ratio=decrease`, + `pad=${targetW}:${targetH}:(ow-iw)/2:(oh-ih)/2:color=black` + ); + } else { + filters.push( + `scale=${targetW}:${targetH}:force_original_aspect_ratio=increase`, + `crop=${targetW}:${targetH}` + ); + } + if (recipe.sharpness !== 0) { + filters.push(`unsharp=5:5:${recipe.sharpness}:5:5:0.0`); + } + + // Normalize timestamps only when needed — trim or speed change both + // require a clean 0-based timeline to produce correct output duration. + if (recipe.trimStart > 0 || recipe.trimEnd !== null || recipe.speed !== 1) { + filters.push("setpts=PTS-STARTPTS"); + } + + if (recipe.speed !== 1) { + const pts = (1 / recipe.speed).toFixed(4); + filters.push(`setpts=${pts}*PTS`); + } + + if (recipe.denoise) { + filters.push("hqdn3d=1.5:1.5:6:6"); + } + + const needsEq = + recipe.brightness !== 0 || + recipe.contrast !== 1 || + recipe.saturation !== 1; + + if (needsEq) { + filters.push( + `eq=brightness=${recipe.brightness}:contrast=${recipe.contrast}:saturation=${recipe.saturation}` + ); + } + + const textOverlays = recipe.textOverlays || []; + textOverlays.forEach((overlay) => { + filters.push(buildTextFilter(overlay, targetW, targetH)); + }); + + return filters.join(","); +} + +export function buildAudioFilter(recipe: EditRecipe): string { + if (recipe.speed <= 0) return ""; + const filters: string[] = []; + + let remaining = recipe.speed; + while (remaining < 0.5) { + filters.push("atempo=0.5"); + remaining /= 0.5; + } + + while (remaining > 2.0) { + filters.push("atempo=2.0"); + remaining /= 2.0; + } + + if (Math.abs(remaining - 1.0) > 0.001) { + filters.push(`atempo=${Number(remaining.toFixed(4))}`); + } + + if (recipe.volume !== undefined && recipe.volume !== 100) { + filters.push(`volume=${(recipe.volume / 100).toFixed(2)}`); + } + + if (recipe.normalizeAudio) filters.push("loudnorm=I=-14:TP=-1.5:LRA=11"); + + return filters.join(","); +} + +export function buildAudioTrimFilter(recipe: EditRecipe): string { + if (recipe.trimStart === 0 && recipe.trimEnd === null) return ""; + let trimFilter = `atrim=start=${recipe.trimStart}`; + if (recipe.trimEnd !== null) { + trimFilter += `:end=${recipe.trimEnd}`; + } + return `${trimFilter},asetpts=PTS-STARTPTS`; +} + +const OVERLAY_POSITIONS: Record = { + "top-left": "20:20", + "top-right": "W-w-20:20", + "bottom-left": "20:H-h-20", + "bottom-right": "W-w-20:H-h-20", +}; + +export interface ExportArgsOptions { + recipe: EditRecipe; + format: Exclude; + inputName: string; + outputName: string; + targetW: number; + targetH: number; + videoDuration: number; + hasOriginalAudio: boolean; + music?: { inputName: string; options: BackgroundMusicOptions }; + overlay?: { inputName: string; options: ImageOverlayOptions }; + /** From resolveThreadCount; 0 for the single-threaded core. */ + threads: number; +} + +export function buildArguments({ + recipe, + format, + inputName, + outputName, + targetW, + targetH, + videoDuration, + hasOriginalAudio, + music, + overlay, + threads, +}: ExportArgsOptions): string[] { + const vf = buildVideoFilter(recipe, targetW, targetH); + const audioTrim = hasOriginalAudio ? buildAudioTrimFilter(recipe) : ""; + const audioSpeed = hasOriginalAudio ? buildAudioFilter(recipe) : ""; + const afParts = [audioTrim, audioSpeed].filter(Boolean); + const af = afParts.join(","); + + const musicIdx = 1; + const overlayIdx = music ? 2 : 1; + + const args: string[] = []; + args.push(...inputThreadArgs(threads), "-i", inputName); + if (music) { + if (music.options.loopMusic) args.push("-stream_loop", "-1"); + args.push("-i", music.inputName); + } + if (overlay) { + args.push("-i", overlay.inputName); + } + + const needsFilterComplex = Boolean(overlay || music); + const shouldKeepAudio = recipe.keepAudio && (hasOriginalAudio || Boolean(music)); + let usesFilterComplex = false; + + if (needsFilterComplex) { + const filterParts: string[] = []; + let videoOut = "[0:v]"; + + if (vf) { + filterParts.push(`[0:v]${vf}[vbase]`); + videoOut = "[vbase]"; + } + + if (overlay) { + const alpha = (overlay.options.opacity / 100).toFixed(2); + filterParts.push(`[${overlayIdx}:v]scale=${overlay.options.size}:-2,format=rgba,colorchannelmixer=aa=${alpha}[logo]`); + const pos = OVERLAY_POSITIONS[overlay.options.position] ?? OVERLAY_POSITIONS["bottom-right"]; + filterParts.push(`${videoOut}[logo]overlay=${pos}[vout]`); + videoOut = "[vout]"; + } + + let audioOut = ""; + if (shouldKeepAudio) { + if (music) { + const musicVol = (music.options.musicVolume / 100).toFixed(2); + if (hasOriginalAudio) { + const origVol = (music.options.originalAudioVolume / 100).toFixed(2); + const origChain = afParts.length > 0 + ? `[0:a]${af},volume=${origVol}[orig]` + : `[0:a]volume=${origVol}[orig]`; + filterParts.push(origChain); + filterParts.push(`[${musicIdx}:a]volume=${musicVol}[music]`); + filterParts.push(`[orig][music]amix=inputs=2:duration=first:dropout_transition=0[aout]`); + audioOut = "[aout]"; + } else { + filterParts.push(`[${musicIdx}:a]volume=${musicVol}[aout]`); + audioOut = "[aout]"; + } + } else if (hasOriginalAudio && af) { + filterParts.push(`[0:a]${af}[aout]`); + audioOut = "[aout]"; + } + } + + if (filterParts.length > 0) { + args.push("-filter_complex", filterParts.join(";")); + usesFilterComplex = true; + } + args.push("-map", videoOut === "[0:v]" ? "0:v" : videoOut); + + if (!shouldKeepAudio) { + args.push("-an"); + } else if (audioOut) { + args.push("-map", audioOut); + } else if (hasOriginalAudio) { + args.push("-map", "0:a"); + } + } else { + if (vf) args.push("-vf", vf); + if (!shouldKeepAudio) { + args.push("-an"); + } else if (af && hasOriginalAudio) { + args.push("-af", af); + } + } + + if (format === "webm") { + // VP8 + Vorbis, not VP9 + Opus: in @ffmpeg/core 0.12.10 the libvpx-vp9 + // and libopus encoders trap ("memory access out of bounds") on every + // setting tried, and hang the multi-threaded core, so WebM exports could + // never succeed. VP8's CRF mode needs a bitrate ceiling. Realtime + // keeps WebM about as fast as MP4 (measured: 9 s vs 21–32 s with + // "good" for a 6 s 1080p clip), as the old VP9 settings intended. + args.push( + "-c:v", "libvpx", + "-crf", String(Math.min(63, Math.max(4, recipe.quality))), + "-b:v", "8M", + "-deadline", "realtime", + "-cpu-used", "8" + ); + if (shouldKeepAudio) args.push("-c:a", "libvorbis", "-q:a", "4"); + } else if (format === "mkv") { + args.push("-c:v", "libx264", "-crf", String(recipe.quality), "-preset", "ultrafast"); + if (shouldKeepAudio) args.push("-c:a", "aac", "-b:a", "128k"); + } else { + args.push("-c:v", "libx264", "-crf", String(recipe.quality), "-preset", "ultrafast", "-movflags", "+faststart"); + if (shouldKeepAudio) args.push("-c:a", "aac", "-b:a", "128k"); + } + + args.push(...outputThreadArgs(threads, usesFilterComplex)); + args.push(...outputDurationArgs(recipe, videoDuration)); + args.push(outputName); + return args; +} + +export interface GifArgsOptions { + recipe: EditRecipe; + inputName: string; + paletteName: string; + outputName: string; + targetW: number; + targetH: number; + videoDuration: number; + threads: number; +} + +/** Two-pass GIF: palettegen, then paletteuse. */ +export function buildGifArguments({ + recipe, + inputName, + paletteName, + outputName, + targetW, + targetH, + videoDuration, + threads, +}: GifArgsOptions): { palette: string[]; render: string[] } { + const vf = buildVideoFilter(recipe, targetW, targetH); + const vfWithPalette = vf ? `${vf},palettegen` : "palettegen"; + const vfWithPaletteUse = vf ? `[0:v]${vf}[x];[x][1:v]paletteuse` : "[0:v][1:v]paletteuse"; + // The trim filter already bounds the clip; -t is only needed when speed + // changes the output length. + const durationArgs = recipe.speed !== 1 ? outputDurationArgs(recipe, videoDuration) : []; + + return { + palette: [ + ...inputThreadArgs(threads), "-i", inputName, + "-vf", vfWithPalette, + ...durationArgs, ...outputThreadArgs(threads, false), + "-y", paletteName, + ], + render: [ + ...inputThreadArgs(threads), "-i", inputName, "-i", paletteName, + "-lavfi", vfWithPaletteUse, + ...durationArgs, ...outputThreadArgs(threads, true), + "-y", outputName, + ], + }; +} diff --git a/src/lib/tests/ffmpegArgs.test.ts b/src/lib/tests/ffmpegArgs.test.ts new file mode 100644 index 000000000..d354ddc59 --- /dev/null +++ b/src/lib/tests/ffmpegArgs.test.ts @@ -0,0 +1,169 @@ +import { describe, it, expect } from "vitest"; +import { + buildArguments, + buildGifArguments, + outputDurationArgs, + resolveThreadCount, + MT_THREAD_CAP, + type ExportArgsOptions, +} from "../ffmpegArgs"; +import { DEFAULT_RECIPE } from "../constants"; +import type { EditRecipe } from "../types"; + +const recipe = (overrides: Partial = {}): EditRecipe => ({ ...DEFAULT_RECIPE, ...overrides }); + +const opts = (overrides: Partial = {}): ExportArgsOptions => ({ + recipe: recipe(), + format: "mp4", + inputName: "input.mp4", + outputName: "output.mp4", + targetW: 1280, + targetH: 720, + videoDuration: 6, + hasOriginalAudio: true, + threads: 0, + ...overrides, +}); + +/** Value following `flag`, at every occurrence. */ +const valuesOf = (args: string[], flag: string) => + args.flatMap((a, i) => (a === flag ? [args[i + 1]] : [])); + +describe("resolveThreadCount", () => { + it("never passes thread flags to the single-threaded core", () => { + expect(resolveThreadCount("single", 16)).toBe(0); + expect(resolveThreadCount("single")).toBe(0); + }); + + it("caps the multi-threaded core at MT_THREAD_CAP however many cores there are", () => { + expect(MT_THREAD_CAP).toBe(4); + expect(resolveThreadCount("multi", 11)).toBe(4); + expect(resolveThreadCount("multi", 64)).toBe(4); + }); + + it("uses fewer threads on small devices and falls back to the cap when unknown", () => { + expect(resolveThreadCount("multi", 2)).toBe(2); + expect(resolveThreadCount("multi", 1)).toBe(1); + expect(resolveThreadCount("multi", 0)).toBe(4); + expect(resolveThreadCount("multi", undefined)).toBe(4); + expect(resolveThreadCount("multi", NaN)).toBe(4); + }); +}); + +describe("buildArguments", () => { + it("single-threaded: no -threads / -filter_threads anywhere", () => { + const args = buildArguments(opts({ threads: 0 })); + expect(args).not.toContain("-threads"); + expect(args).not.toContain("-filter_threads"); + expect(args).not.toContain("-filter_complex_threads"); + }); + + it("multi-threaded: caps decoder, filter and encoder threads", () => { + const args = buildArguments(opts({ threads: 4 })); + // Decoder flag must precede -i to apply to the input. + expect(args.indexOf("-threads")).toBeLessThan(args.indexOf("-i")); + expect(valuesOf(args, "-threads")).toEqual(["4", "4"]); + expect(valuesOf(args, "-filter_threads")).toEqual(["4"]); + expect(args.at(-1)).toBe("output.mp4"); + }); + + it("adds -filter_complex_threads only when a filter graph is used", () => { + const withOverlay = buildArguments( + opts({ + threads: 4, + overlay: { inputName: "logo.png", options: { file: null, position: "top-left", size: 120, opacity: 80 } }, + }) + ); + expect(withOverlay).toContain("-filter_complex"); + expect(valuesOf(withOverlay, "-filter_complex_threads")).toEqual(["4"]); + expect(buildArguments(opts({ threads: 4 }))).not.toContain("-filter_complex_threads"); + }); + + it("applies sharpness and volume (these used to be dropped by the worker's copy)", () => { + const args = buildArguments(opts({ recipe: recipe({ sharpness: 2, volume: 150 }) })); + expect(valuesOf(args, "-vf")[0]).toContain("unsharp=5:5:2:5:5:0.0"); + expect(valuesOf(args, "-af")[0]).toContain("volume=1.50"); + }); + + it("always bounds output duration when it is known, and omits it when not", () => { + expect(valuesOf(buildArguments(opts()), "-t")).toEqual(["6.000000"]); + expect(valuesOf(buildArguments(opts({ recipe: recipe({ trimStart: 1, trimEnd: 5, speed: 2 }) })), "-t")).toEqual([ + "2.000000", + ]); + expect(buildArguments(opts({ videoDuration: 0 }))).not.toContain("-t"); + }); + + it("positions an image overlay inside the frame", () => { + const args = buildArguments( + opts({ overlay: { inputName: "logo.png", options: { file: null, position: "bottom-right", size: 100, opacity: 50 } } }) + ); + const graph = valuesOf(args, "-filter_complex")[0]!; + expect(graph).toContain("overlay=W-w-20:H-h-20"); + expect(graph).toContain("colorchannelmixer=aa=0.50"); + }); + + it("mixes background music with the original audio and loops it when asked", () => { + const args = buildArguments( + opts({ + music: { inputName: "music.mp3", options: { file: null, musicVolume: 50, originalAudioVolume: 100, loopMusic: true } }, + }) + ); + expect(args.slice(0, 6)).toEqual(["-i", "input.mp4", "-stream_loop", "-1", "-i", "music.mp3"]); + expect(valuesOf(args, "-filter_complex")[0]).toContain("amix=inputs=2"); + expect(valuesOf(args, "-map")).toEqual(["[vbase]", "[aout]"]); + }); + + it("drops audio when the source has none", () => { + expect(buildArguments(opts({ hasOriginalAudio: false }))).toContain("-an"); + }); + + it("uses VP8/Vorbis for WebM (VP9 and Opus crash in @ffmpeg/core 0.12.10)", () => { + const args = buildArguments(opts({ format: "webm", outputName: "out.webm" })); + expect(valuesOf(args, "-c:v")).toEqual(["libvpx"]); + expect(valuesOf(args, "-c:a")).toEqual(["libvorbis"]); + expect(args).not.toContain("libvpx-vp9"); + expect(args).not.toContain("libopus"); + expect(valuesOf(args, "-b:v")).toEqual(["8M"]); + }); +}); + +describe("buildGifArguments", () => { + const gif = (threads: number, overrides: Partial = {}) => + buildGifArguments({ + recipe: recipe({ format: "gif", ...overrides }), + inputName: "in.mp4", + paletteName: "palette.png", + outputName: "out.gif", + targetW: 480, + targetH: 270, + videoDuration: 6, + threads, + }); + + it("runs palettegen then paletteuse", () => { + const { palette, render } = gif(0); + expect(valuesOf(palette, "-vf")[0]).toMatch(/palettegen$/); + expect(valuesOf(render, "-lavfi")[0]).toMatch(/paletteuse$/); + expect(palette).not.toContain("-threads"); + }); + + it("caps threads in both passes on the multi-threaded core", () => { + const { palette, render } = gif(4); + expect(valuesOf(palette, "-threads")).toEqual(["4", "4"]); + expect(valuesOf(render, "-threads")).toEqual(["4", "4"]); + expect(valuesOf(render, "-filter_complex_threads")).toEqual(["4"]); + }); + + it("only adds -t when speed changes the length", () => { + expect(gif(0).render).not.toContain("-t"); + expect(valuesOf(gif(0, { speed: 2 }).render, "-t")).toEqual(["3.000000"]); + }); +}); + +describe("outputDurationArgs", () => { + it("guards against zero, negative and invalid durations", () => { + expect(outputDurationArgs(recipe(), 0)).toEqual([]); + expect(outputDurationArgs(recipe({ trimStart: 8 }), 6)).toEqual([]); + expect(outputDurationArgs(recipe({ speed: 0 }), 6)).toEqual([]); + }); +}); diff --git a/src/lib/tests/ffmpegClient.test.ts b/src/lib/tests/ffmpegClient.test.ts new file mode 100644 index 000000000..5cc3c2cda --- /dev/null +++ b/src/lib/tests/ffmpegClient.test.ts @@ -0,0 +1,237 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { + loadFFmpeg, + exportVideo, + terminateFFmpeg, + dedupeProgress, + getFFmpegClientState, + getEngineMode, + resetEnginePreferenceForTests, + FFmpegLoadError, + ExportStalledError, + STALL_TIMEOUT_MS, +} from "../ffmpeg"; +import { DEFAULT_RECIPE } from "../constants"; + +/** + * Scripted stand-in for the FFmpeg worker. Each test decides how the worker + * answers "load" and "export"; every instance is recorded so tests can check + * that terminated workers are never reused. + */ +type Script = { + onLoad?: (w: FakeWorker) => void; + onExport?: (w: FakeWorker, id: string) => void; +}; +let script: Script = {}; +const workers: FakeWorker[] = []; + +class FakeWorker { + onmessage: ((e: MessageEvent) => void) | null = null; + onerror: ((e: ErrorEvent) => void) | null = null; + posted: { type: string; id?: string; singleThread?: boolean }[] = []; + terminated = false; + constructor() { + workers.push(this); + } + postMessage(msg: { type: string; id?: string; singleThread?: boolean }) { + this.posted.push(msg); + if (msg.type === "load") queueMicrotask(() => (script.onLoad ?? ((w) => w.emit({ type: "ready", mode: "single" })))(this)); + if (msg.type === "export") queueMicrotask(() => script.onExport?.(this, msg.id!)); + } + terminate() { + this.terminated = true; + } + emit(data: unknown) { + if (!this.terminated) this.onmessage?.({ data } as MessageEvent); + } +} + +const result = (id: string) => ({ + type: "result", + id, + data: new ArrayBuffer(8), + mimeType: "video/mp4", + size: 8, + width: 1920, + height: 1080, + format: "mp4", +}); + +const file = () => new File([new Uint8Array(16)], "clip.mp4", { type: "video/mp4" }); +const run = (signal?: AbortSignal, onProgress = vi.fn()) => + exportVideo(file(), { ...DEFAULT_RECIPE }, onProgress, signal, undefined, undefined, 6); + +beforeEach(() => { + script = {}; + workers.length = 0; + vi.stubGlobal("Worker", FakeWorker); + vi.stubGlobal("URL", Object.assign(URL, { createObjectURL: vi.fn(() => "blob:out"), revokeObjectURL: vi.fn() })); +}); + +afterEach(() => { + terminateFFmpeg(); + resetEnginePreferenceForTests(); + vi.useRealTimers(); + vi.unstubAllGlobals(); +}); + +describe("dedupeProgress", () => { + it("only forwards changes of the rounded percentage", () => { + const cb = vi.fn(); + const p = dedupeProgress(cb)!; + [0, 0.2, 0.4, 1, 1.3, 1, 2, 2, 150].forEach(p); + expect(cb.mock.calls.map((c) => c[0])).toEqual([0, 1, 2, 100]); + }); +}); + +describe("FFmpeg client lifecycle", () => { + it("loads once and reuses the worker across repeated exports", async () => { + script.onExport = (w, id) => w.emit(result(id)); + await loadFFmpeg(); + const a = await run(); + const b = await run(); + expect(a.size).toBe(8); + expect(b.size).toBe(8); + expect(workers).toHaveLength(1); + expect(workers[0]!.posted.filter((m) => m.type === "load")).toHaveLength(1); + expect(getFFmpegClientState()).toMatchObject({ hasPendingExport: false, hasPendingProgress: false }); + }); + + it("does not accumulate abort listeners over repeated exports", async () => { + script.onExport = (w, id) => w.emit(result(id)); + const controller = new AbortController(); + const add = vi.spyOn(controller.signal, "addEventListener"); + const remove = vi.spyOn(controller.signal, "removeEventListener"); + for (let i = 0; i < 5; i++) await run(controller.signal); + expect(add.mock.calls.length).toBeGreaterThan(0); + expect(remove.mock.calls.length).toBe(add.mock.calls.length); + }); + + it("forwards de-duplicated progress for the active export", async () => { + script.onExport = (w, id) => { + [0, 0, 10, 10, 10, 55].forEach((percent) => w.emit({ type: "progress", percent })); + w.emit(result(id)); + }; + const onProgress = vi.fn(); + await loadFFmpeg(); + await run(undefined, onProgress); + expect(onProgress.mock.calls.map((c) => c[0])).toEqual([0, 10, 55]); + }); + + it("cancelling rejects with AbortError and tells the worker to stop", async () => { + script.onExport = () => {}; // never finishes on its own + const controller = new AbortController(); + await loadFFmpeg(); + const pending = run(controller.signal); + await new Promise((r) => setTimeout(r, 0)); + controller.abort(); + await expect(pending).rejects.toMatchObject({ name: "AbortError" }); + expect(workers[0]!.posted.some((m) => m.type === "cancel")).toBe(true); + expect(getFFmpegClientState().hasPendingExport).toBe(false); + }); + + it("a worker crash fails the export, and the next export gets a fresh worker", async () => { + script.onExport = (w) => w.onerror?.({ message: "boom" } as ErrorEvent); + await loadFFmpeg(); + await expect(run()).rejects.toBeInstanceOf(FFmpegLoadError); + expect(getFFmpegClientState().hasWorker).toBe(false); + + script.onExport = (w, id) => w.emit(result(id)); + const ok = await run(); + expect(ok.size).toBe(8); + expect(workers).toHaveLength(2); + }); + + it("a failed engine load can be retried", async () => { + script.onLoad = (w) => w.emit({ type: "error", message: "Failed to fetch" }); + await expect(loadFFmpeg()).rejects.toBeInstanceOf(FFmpegLoadError); + + script.onLoad = undefined; + script.onExport = (w, id) => w.emit(result(id)); + await loadFFmpeg(); + await expect(run()).resolves.toMatchObject({ format: "mp4" }); + }); + + it("terminating during engine load settles the load instead of hanging", async () => { + script.onLoad = () => {}; // load never completes + const loading = loadFFmpeg(); + await new Promise((r) => setTimeout(r, 0)); + terminateFFmpeg(); + await expect(loading).rejects.toMatchObject({ name: "AbortError" }); + expect(workers[0]!.terminated).toBe(true); + expect(getFFmpegClientState()).toEqual({ hasWorker: false, hasPendingExport: false, hasPendingProgress: false }); + }); + + it("export results own a blob URL that dispose() revokes", async () => { + script.onExport = (w, id) => w.emit(result(id)); + const res = await run(); + res.dispose(); + expect(URL.revokeObjectURL).toHaveBeenCalledWith("blob:out"); + }); + + it("records which core the worker actually loaded", async () => { + script.onLoad = (w) => w.emit({ type: "ready", mode: "multi" }); + expect(getEngineMode()).toBeNull(); + await loadFFmpeg(); + expect(getEngineMode()).toBe("multi"); + }); + + it("a worker error without an export id fails the pending export instead of hanging it", async () => { + script.onExport = (w) => w.emit({ type: "error", message: "unexpected" }); + await loadFFmpeg(); + await expect(run()).rejects.toBeInstanceOf(FFmpegLoadError); + expect(workers[0]!.terminated).toBe(true); + expect(getFFmpegClientState()).toEqual({ hasWorker: false, hasPendingExport: false, hasPendingProgress: false }); + }); + + it("cancelling during engine load discards the half-loaded worker", async () => { + script.onLoad = () => {}; // never finishes + const controller = new AbortController(); + const loading = loadFFmpeg(controller.signal); + await new Promise((r) => setTimeout(r, 0)); + controller.abort(); + await expect(loading).rejects.toMatchObject({ name: "AbortError" }); + expect(workers[0]!.terminated).toBe(true); + + script.onLoad = undefined; + script.onExport = (w, id) => w.emit(result(id)); + await expect(run()).resolves.toMatchObject({ format: "mp4" }); + expect(workers).toHaveLength(2); + }); + + it("heartbeats keep a slow export alive", async () => { + vi.useFakeTimers(); + let worker!: FakeWorker; + let jobId = ""; + script.onExport = (w, id) => { + worker = w; + jobId = id; + }; + const pending = run(); + for (let t = 0; t < STALL_TIMEOUT_MS * 2; t += 10_000) { + await vi.advanceTimersByTimeAsync(10_000); + worker?.emit({ type: "heartbeat" }); + } + worker.emit(result(jobId)); + await expect(pending).resolves.toMatchObject({ size: 8 }); + }); + + it("a stalled multi-threaded export fails, and the retry loads the single-threaded core", async () => { + vi.useFakeTimers(); + script.onLoad = (w) => w.emit({ type: "ready", mode: "multi" }); + script.onExport = () => {}; // wedged: no progress, no heartbeat + const pending = run(); + const assertion = expect(pending).rejects.toBeInstanceOf(ExportStalledError); + await vi.advanceTimersByTimeAsync(STALL_TIMEOUT_MS + 5_000); + await assertion; + await expect(pending).rejects.toMatchObject({ mode: "multi" }); + expect(workers[0]!.terminated).toBe(true); + expect(getFFmpegClientState().hasWorker).toBe(false); + + script.onLoad = undefined; + script.onExport = (w, id) => w.emit(result(id)); + await expect(run()).resolves.toMatchObject({ format: "mp4" }); + expect(workers[1]!.posted.find((m) => m.type === "load")).toMatchObject({ singleThread: true }); + }); +}); + diff --git a/src/lib/tests/ffmpegWorker.test.ts b/src/lib/tests/ffmpegWorker.test.ts new file mode 100644 index 000000000..05ba9a0ef --- /dev/null +++ b/src/lib/tests/ffmpegWorker.test.ts @@ -0,0 +1,220 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { DEFAULT_RECIPE } from "../constants"; +import type { EditRecipe } from "../types"; + +// Drives the real worker module (the code that runs in production) against a +// fake FFmpeg, so command-line fixes are tested where they execute rather +// than in a copy. + +const h = vi.hoisted(() => { + type Listener = (event: { message?: string; progress?: number }) => void; + class FakeFFmpeg { + static instances: FakeFFmpeg[] = []; + static loadImpl: (config: Record) => Promise = async () => {}; + static execImpl: ((args: string[]) => Promise) | null = null; + listeners = { progress: new Set(), log: new Set() }; + config: Record = {}; + execCalls: string[][] = []; + deleted: string[] = []; + files = new Map(); + terminated = false; + constructor() { + FakeFFmpeg.instances.push(this); + } + on(event: "progress" | "log", fn: Listener) { + this.listeners[event].add(fn); + } + off(event: "progress" | "log", fn: Listener) { + this.listeners[event].delete(fn); + } + async load(config: Record) { + this.config = config; + await FakeFFmpeg.loadImpl(config); + } + async writeFile(name: string, data: Uint8Array) { + this.files.set(name, data); + } + async exec(args: string[]) { + this.execCalls.push(args); + if (FakeFFmpeg.execImpl) return FakeFFmpeg.execImpl(args); + this.listeners.log.forEach((fn) => fn({ message: "frame= 1 fps=0.0 time=00:00:00.04" })); + this.listeners.progress.forEach((fn) => fn({ progress: 0.5 })); + this.files.set(args.at(-1)!, new Uint8Array([1, 2, 3])); + return 0; + } + async readFile(name: string) { + return this.files.get(name)!; + } + async deleteFile(name: string) { + this.deleted.push(name); + } + terminate() { + this.terminated = true; + } + } + return { FakeFFmpeg }; +}); + +vi.mock("@ffmpeg/ffmpeg", () => ({ FFmpeg: h.FakeFFmpeg })); +vi.mock("@ffmpeg/util", () => ({ toBlobURL: vi.fn() })); + +const posted: { type: string; [k: string]: unknown }[] = []; +let handler: ((event: MessageEvent) => void) | null = null; + +async function startWorker({ isolated, cores = 11 }: { isolated: boolean; cores?: number }) { + Object.defineProperty(window, "crossOriginIsolated", { configurable: true, value: isolated }); + Object.defineProperty(navigator, "hardwareConcurrency", { configurable: true, value: cores }); + const add = vi.spyOn(self, "addEventListener"); + vi.resetModules(); + await import("../ffmpeg.worker"); + const call = add.mock.calls.find(([type]) => type === "message")!; + handler = call[1] as (event: MessageEvent) => void; + add.mockRestore(); +} + +const send = (data: unknown) => handler!({ data } as MessageEvent); +const waitFor = (type: string) => vi.waitFor(() => expect(posted.some((m) => m.type === type)).toBe(true)); +const ffmpeg = () => h.FakeFFmpeg.instances.at(-1)!; + +function exportRequest(recipe: Partial = {}) { + return { + type: "export", + id: "job1", + file: { name: "My holiday.mp4", type: "video/mp4", data: new ArrayBuffer(16) }, + recipe: { ...DEFAULT_RECIPE, ...recipe }, + videoDuration: 6, + }; +} + +beforeEach(() => { + posted.length = 0; + h.FakeFFmpeg.instances.length = 0; + h.FakeFFmpeg.loadImpl = async () => {}; + h.FakeFFmpeg.execImpl = null; + vi.stubGlobal("postMessage", vi.fn((msg) => posted.push(msg))); + vi.stubGlobal( + "fetch", + vi.fn(async () => ({ ok: true, status: 200, arrayBuffer: async () => new ArrayBuffer(1) })) + ); + URL.createObjectURL = vi.fn(() => "blob:core"); +}); + +afterEach(() => { + vi.unstubAllGlobals(); + if (handler) self.removeEventListener("message", handler as EventListener); + handler = null; +}); + +describe("ffmpeg.worker", () => { + it("isolated: loads the MT core and caps threads in the command it runs", async () => { + await startWorker({ isolated: true, cores: 16 }); + send({ type: "load" }); + await waitFor("ready"); + expect(posted.find((m) => m.type === "ready")).toMatchObject({ mode: "multi" }); + expect(ffmpeg().config.coreURL).toBeDefined(); + expect(ffmpeg().config.workerURL).toBeDefined(); + expect(vi.mocked(fetch).mock.calls[0]![0]).toContain("@ffmpeg/core-mt@"); + // Every engine file, including the pthread worker script, is fetched + // with a Subresource Integrity hash. + expect(vi.mocked(fetch).mock.calls.map(([url, init]) => [String(url).split("/").pop(), Boolean((init as RequestInit)?.integrity)])).toEqual([ + ["ffmpeg-core.js", true], + ["ffmpeg-core.wasm", true], + ["ffmpeg-core.worker.js", true], + ]); + + send(exportRequest({ sharpness: 2, volume: 50 })); + await waitFor("result"); + const args = ffmpeg().execCalls[0]!; + const threads = args.flatMap((a, i) => (a === "-threads" ? [args[i + 1]] : [])); + expect(threads).toEqual(["4", "4"]); + // Settings the old in-worker copy silently ignored: + expect(args.join(" ")).toContain("unsharp=5:5:2"); + expect(args.join(" ")).toContain("volume=0.50"); + }); + + it("not isolated: loads the single-threaded core and passes no thread flags", async () => { + await startWorker({ isolated: false }); + send({ type: "load" }); + await waitFor("ready"); + expect(posted.find((m) => m.type === "ready")).toMatchObject({ mode: "single" }); + expect(ffmpeg().config.workerURL).toBeUndefined(); + expect(vi.mocked(fetch).mock.calls[0]![0]).toContain("@ffmpeg/core@"); + expect(vi.mocked(fetch).mock.calls.every(([, init]) => Boolean((init as RequestInit)?.integrity))).toBe(true); + + send(exportRequest()); + await waitFor("result"); + expect(ffmpeg().execCalls[0]).not.toContain("-threads"); + }); + + it("falls back to the single-threaded core if the MT core fails to load", async () => { + await startWorker({ isolated: true }); + h.FakeFFmpeg.loadImpl = async (config) => { + if (config.workerURL) throw new Error("MT instantiate failed"); + }; + send({ type: "load" }); + await waitFor("ready"); + expect(posted.find((m) => m.type === "ready")).toMatchObject({ mode: "single" }); + expect(h.FakeFFmpeg.instances[0]!.terminated).toBe(true); + + send(exportRequest()); + await waitFor("result"); + expect(ffmpeg().execCalls[0]).not.toContain("-threads"); + }); + + it("honours singleThread on load even when isolated", async () => { + await startWorker({ isolated: true }); + send({ type: "load", singleThread: true }); + await waitFor("ready"); + expect(posted.find((m) => m.type === "ready")).toMatchObject({ mode: "single" }); + }); + + it("reports a failed engine download instead of loading garbage", async () => { + await startWorker({ isolated: false }); + vi.mocked(fetch).mockResolvedValue({ ok: false, status: 404, arrayBuffer: async () => new ArrayBuffer(0) } as Response); + send({ type: "load" }); + await waitFor("error"); + expect(posted.find((m) => m.type === "error")?.message).toMatch(/HTTP 404/); + }); + + it("removes its listeners and temp files after every export, and sends heartbeats", async () => { + await startWorker({ isolated: false }); + send({ type: "load" }); + await waitFor("ready"); + for (let i = 0; i < 3; i++) { + posted.length = 0; + send({ ...exportRequest(), id: `job${i}` }); + await waitFor("result"); + } + expect(ffmpeg().listeners.log.size).toBe(0); + expect(ffmpeg().listeners.progress.size).toBe(0); + expect(ffmpeg().deleted).toEqual(expect.arrayContaining(["input_job2.mp4", "output_job2.mp4"])); + expect(posted.some((m) => m.type === "heartbeat")).toBe(true); + // Temp names come from the job id, never the user's file name. + expect([...ffmpeg().files.keys()].some((name) => name.includes("holiday"))).toBe(false); + }); + + it("reports a wasm trap (a rejected string) with its text, not 'undefined'", async () => { + await startWorker({ isolated: false }); + send({ type: "load" }); + await waitFor("ready"); + h.FakeFFmpeg.execImpl = () => Promise.reject("RuntimeError: memory access out of bounds"); + send(exportRequest()); + await waitFor("error"); + expect(posted.find((m) => m.type === "error")).toMatchObject({ + id: "job1", + message: "RuntimeError: memory access out of bounds", + }); + }); + + it("GIF exports use the shared two-pass builder with thread caps", async () => { + await startWorker({ isolated: true }); + send({ type: "load" }); + await waitFor("ready"); + send(exportRequest({ format: "gif" })); + await waitFor("result"); + const [palette, render] = ffmpeg().execCalls; + expect(palette!.join(" ")).toContain("palettegen"); + expect(render!.join(" ")).toContain("paletteuse"); + expect(render).toContain("-filter_complex_threads"); + }); +}); From e85c63c3dd445fafb242fbb177ec12f67848e326 Mon Sep 17 00:00:00 2001 From: Akanksha Trehun Date: Tue, 3 Nov 2026 13:40:11 +0530 Subject: [PATCH 2/6] fix(export): unknown durations, stuck-progress watchdog, e2e export tests - Live-recorded files (MediaRecorder screen recordings) report an Infinity duration; seek to the end to find the real one (src/lib/mediaDuration.ts), in the editor and the export. Trim works on them again. - ffmpegArgs: time values are repaired before use (withSafeTimes), -t only for a finite positive length, -shortest for looped music when the length is unknown, and a final check that no argument carries Infinity or NaN. An infinite speed no longer hangs the audio filter builder. - Watchdog: besides 60 s of silence, stop exports whose encoded time stops changing for 120 s while FFmpeg keeps logging (stuck at 100%), or that run far past the expected length. The worker reports encoded time and computes progress from it. The message says what happened and to press Retry. - Telemetry: unknown_duration failure category and stall_reason. - scripts/make-fixtures.sh (test/fixtures/, gitignored), a static server with COOP/COEP (scripts/serve-isolated.mjs), Playwright export tests on the multi- and single-threaded engines, and a Tests workflow running Vitest and Playwright in CI. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/test.yml | 62 ++++++++ .gitignore | 7 + bun.lock | 7 + docs/DATADOG.md | 13 +- .../datadog/export-reliability.dashboard.json | 52 ++++++- package.json | 1 + playwright.config.ts | 44 ++++++ scripts/make-fixtures.sh | 60 ++++++++ scripts/serve-isolated.mjs | 64 +++++++++ src/hooks/tests/useVideoEditor.test.ts | 1 + src/hooks/tests/useVideoEditorProject.test.ts | 1 + src/hooks/useVideoEditor.ts | 19 ++- src/lib/exportTelemetry.ts | 27 +++- src/lib/ffmpeg.ts | 99 ++++++++++--- src/lib/ffmpeg.worker.ts | 40 ++++-- src/lib/ffmpegArgs.ts | 87 ++++++++--- src/lib/mediaDuration.ts | 45 ++++++ src/lib/tests/exportTelemetry.test.ts | 36 +++-- src/lib/tests/ffmpegArgs.test.ts | 88 ++++++++++++ src/lib/tests/ffmpegClient.test.ts | 100 +++++++++++++ src/lib/tests/mediaDuration.test.ts | 53 +++++++ test/e2e/export.spec.ts | 135 ++++++++++++++++++ 22 files changed, 977 insertions(+), 64 deletions(-) create mode 100644 .github/workflows/test.yml create mode 100644 playwright.config.ts create mode 100755 scripts/make-fixtures.sh create mode 100644 scripts/serve-isolated.mjs create mode 100644 src/lib/mediaDuration.ts create mode 100644 src/lib/tests/mediaDuration.test.ts create mode 100644 test/e2e/export.spec.ts diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 000000000..319617403 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,62 @@ +name: Tests + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +concurrency: + group: tests-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + unit: + name: Unit tests (Vitest) + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v6 + - uses: oven-sh/setup-bun@v2 + - run: bun install --frozen-lockfile + - run: bun run test --run + + e2e: + name: Export end-to-end (Playwright) + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v6 + - uses: oven-sh/setup-bun@v2 + - run: bun install --frozen-lockfile + + # ffmpeg makes the input fixtures and ffprobe checks the exported files. + - name: Install ffmpeg + run: sudo apt-get update && sudo apt-get install -y --no-install-recommends ffmpeg + + - name: Make video fixtures + run: scripts/make-fixtures.sh --small + + # Telemetry off: CI must never report to Datadog. + - name: Build the static site + run: bun run build + env: + NEXT_PUBLIC_DATADOG_ENV: "off" + + - name: Install Chromium for Playwright + run: bunx playwright install --with-deps chromium + + # Served with COOP/COEP (multi-threaded core, as in production) and + # without (single-threaded core); see playwright.config.ts. + - name: Run Playwright + run: bunx playwright test + + - name: Upload Playwright report + if: failure() + uses: actions/upload-artifact@v4 + with: + name: playwright-results + path: test-results/ + retention-days: 7 diff --git a/.gitignore b/.gitignore index 1d982f0b4..cc717af46 100644 --- a/.gitignore +++ b/.gitignore @@ -50,3 +50,10 @@ chromatic.log # Local Netlify folder .netlify + +# generated video fixtures (scripts/make-fixtures.sh) +/test/fixtures/ + +# playwright +/test-results/ +/playwright-report/ diff --git a/bun.lock b/bun.lock index c91ebf99a..ffc20a3a5 100644 --- a/bun.lock +++ b/bun.lock @@ -22,6 +22,7 @@ "wasm-feature-detect": "^1.8.0", }, "devDependencies": { + "@playwright/test": "1.64.1", "@storybook/addon-a11y": "^10.5.10", "@storybook/addon-docs": "^10.5.10", "@storybook/addon-themes": "^10.5.10", @@ -566,6 +567,8 @@ "@oxc-resolver/binding-win32-x64-msvc": ["@oxc-resolver/binding-win32-x64-msvc@11.21.2", "", { "os": "win32", "cpu": "x64" }, "sha512-VPoCAhKvCQTlG7vxqaBXcmuvbh77BfnGXj8g0pbvVXpm1F/R8rDVwqIWcEbMzrI1JvJlm8v7T9uMVQb6UctMRg=="], + "@playwright/test": ["@playwright/test@1.64.1", "", { "dependencies": { "playwright": "1.64.1" }, "bin": { "playwright": "cli.js" } }, "sha512-IaYGml/nSOiZWROfPtuIL7neuu9qDOZ3TNvNljUIrhDMxiGTgPjzj/g9p9RxpS2tSepPDRY+UwAaEeiKtUvIAg=="], + "@pmmmwh/react-refresh-webpack-plugin": ["@pmmmwh/react-refresh-webpack-plugin@0.5.17", "", { "dependencies": { "ansi-html": "^0.0.9", "core-js-pure": "^3.23.3", "error-stack-parser": "^2.0.6", "html-entities": "^2.1.0", "loader-utils": "^2.0.4", "schema-utils": "^4.2.0", "source-map": "^0.7.3" }, "peerDependencies": { "@types/webpack": "4.x || 5.x", "react-refresh": ">=0.10.0 <1.0.0", "sockjs-client": "^1.4.0", "type-fest": ">=0.17.0 <5.0.0", "webpack": ">=4.43.0 <6.0.0", "webpack-dev-server": "3.x || 4.x || 5.x", "webpack-hot-middleware": "2.x", "webpack-plugin-serve": "0.x || 1.x" }, "optionalPeers": ["@types/webpack", "sockjs-client", "type-fest", "webpack-dev-server", "webpack-hot-middleware", "webpack-plugin-serve"] }, "sha512-tXDyE1/jzFsHXjhRZQ3hMl0IVhYe5qula43LDWIhVfjp9G/nT5OQY5AORVOrkEGAUltBJOfOWeETbmhm6kHhuQ=="], "@rolldown/binding-android-arm-eabi": ["@rolldown/binding-android-arm-eabi@1.2.6", "", { "os": "android", "cpu": "arm" }, "sha512-b+jTcARdTiFLI6jB4a5XjTm0RWd6KcRfQj/I2356fxUZemiho9zQLxo0RtCuMDAyKcLo6cEltkgbQp6d1+sjjQ=="], @@ -1626,6 +1629,10 @@ "pkg-dir": ["pkg-dir@7.0.0", "", { "dependencies": { "find-up": "^6.3.0" } }, "sha512-Ie9z/WINcxxLp27BKOCHGde4ITq9UklYKDzVo1nhk5sqGEXU3FpkwP5GM2voTGJkGd9B3Otl+Q4uwSOeSUtOBA=="], + "playwright": ["playwright@1.64.1", "", { "dependencies": { "playwright-core": "1.64.1" }, "bin": { "playwright": "cli.js" } }, "sha512-HCrVhHr5QQuwYoswEUGTb7gGp522yflbv/ST8nF4sFWGuLjQCt17Q/EubpGGSNKYn1tsTCPZxqHhHCfVURu7UA=="], + + "playwright-core": ["playwright-core@1.64.1", "", { "bin": { "playwright-core": "cli.js" } }, "sha512-BC4s8y6SEVA6Q1rSxZTto8hI1KUI3xXlHGZgvGqP4IqPdI/a6FX9H29IDNz7vKo9l21b6LfYosScPL6M3xxyDQ=="], + "possible-typed-array-names": ["possible-typed-array-names@1.1.0", "", {}, "sha512-/+5VFTchJDoVj3bhoqi6UeymcD00DAwb1nJwamzPvHEszJ4FpF6SNNbUbOS8yI56qHzdV8eK0qEfOSiodkTdxg=="], "postcss": ["postcss@8.5.26", "", { "dependencies": { "nanoid": "^3.3.17", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-u82N74LFzG8ca+dD8puPnplTXoGH4fTPpVGuIbt36G3qvNlkvfD0lEAZSxaly3KX8TS/L1A1gsCEmvKmBcVbkQ=="], diff --git a/docs/DATADOG.md b/docs/DATADOG.md index 38f011b21..5bbea22c3 100644 --- a/docs/DATADOG.md +++ b/docs/DATADOG.md @@ -44,7 +44,7 @@ its own, so we can see which browser and device combinations fail. |---|---|---| | `export_started` | Export clicked and settings validated | `format`, `resolution`, `input_size`, `clip_length`, `has_text`, `has_music`, `has_image_overlay`, `is_retry`, `retry_after` (retries only: the previous failure's category) | | `export_succeeded` | Terminal | the context above, plus `engine`, `durationMs` (click to done), `encode_ms`, `output_size` | -| `export_failed` | Terminal | the context, plus `engine`, `durationMs`, `stage` (`engine` / `encode`), `category` | +| `export_failed` | Terminal | the context, plus `engine`, `durationMs`, `stage` (`engine` / `encode`), `category`, and `stall_reason` for stalls | | `export_cancelled` | Terminal: the user cancelled | the context, plus `engine`, `durationMs`, `stage` | | `ffmpeg_load_success` | A worker loaded the engine (once per worker) | `engine` (`multi` / `single`, what actually loaded), `engine_requested`, `single_reason` (`not_isolated` / `after_stall` / `mt_load_failed`, single-threaded only), `load_number` (`first` / `repeat`), `durationMs` | | `ffmpeg_load_failure` | A worker failed to load (not sent when the user cancelled) | `engine_requested`, `load_number`, `category` (`network` / `integrity` / `wasm` / `worker_script` / `unknown`), `durationMs` | @@ -68,12 +68,21 @@ as part of the schema. | Category | Meaning | |---|---| | `engine_load` | The engine never became ready (any failure in the `engine` stage) | -| `timeout` | No progress for 60 s; the next attempt uses the single-threaded engine | +| `timeout` | The stall watchdog stopped the export (see `stall_reason`); the next attempt uses the single-threaded engine | | `unsupported_format` | FFmpeg couldn't decode the input | | `out_of_memory` | An allocation failed, or a `RangeError` | | `encoder_crash` | A WASM trap in the encoder (`memory access out of bounds`, `unreachable`) | | `network` | A network error after the engine started | | `encode_error` | Any other FFmpeg failure | +| `unknown_duration` | The browser couldn't tell how long the clip is (`clip_length: unknown`), and the export then stalled or failed. Usually a live recording with no duration header that the seek-to-end fallback couldn't resolve | + +`export_failed.stall_reason` (stalls only): + +| Reason | Meaning | +|---|---| +| `no_activity` | No heartbeat or log line for 60 s: the engine is wedged (the multi-threaded thread-pool deadlock looked like this) | +| `no_progress` | FFmpeg kept logging, but the encoded time didn't change for 120 s ("stuck at 100%") | +| `overrun` | Encoding ran past 1.25 × the expected length + 15 s, for example looped music that never stops | ### Guarantees (covered by tests) diff --git a/docs/datadog/export-reliability.dashboard.json b/docs/datadog/export-reliability.dashboard.json index 3ebee75c5..53b2a31b9 100644 --- a/docs/datadog/export-reliability.dashboard.json +++ b/docs/datadog/export-reliability.dashboard.json @@ -1,5 +1,5 @@ { - "title": "Reframe — export reliability", + "title": "Reframe \u2014 export reliability", "description": "Export outcomes from Reframe's browser RUM events (src/lib/exportTelemetry.ts). Success rate = export_succeeded / (export_succeeded + export_failed); cancellations are excluded. Treat groups with few finished exports as noise. Setup and facets: docs/DATADOG.md.", "layout_type": "ordered", "template_variables": [ @@ -1009,6 +1009,56 @@ } ] } + }, + { + "definition": { + "title": "Stalled exports by reason and engine", + "type": "query_table", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "rum", + "name": "stalled", + "indexes": [ + "*" + ], + "compute": { + "aggregation": "count" + }, + "search": { + "query": "@type:action service:reframe $env @action.target.name:export_failed @context.category:(timeout OR unknown_duration)" + }, + "group_by": [ + { + "facet": "@context.stall_reason", + "limit": 3, + "sort": { + "aggregation": "count", + "order": "desc" + } + }, + { + "facet": "@context.engine", + "limit": 3, + "sort": { + "aggregation": "count", + "order": "desc" + } + } + ] + } + ], + "formulas": [ + { + "formula": "stalled", + "alias": "Stalled exports" + } + ] + } + ] + } } ] } diff --git a/package.json b/package.json index 8cf6b3057..5037164c9 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "wasm-feature-detect": "^1.8.0" }, "devDependencies": { + "@playwright/test": "1.64.1", "@storybook/addon-a11y": "^10.5.10", "@storybook/addon-docs": "^10.5.10", "@storybook/addon-themes": "^10.5.10", diff --git a/playwright.config.ts b/playwright.config.ts new file mode 100644 index 000000000..533b7f532 --- /dev/null +++ b/playwright.config.ts @@ -0,0 +1,44 @@ +import { defineConfig, devices } from "@playwright/test"; + +// End-to-end export tests against the production build (out/), served with +// the same cross-origin isolation headers as Netlify. +// +// scripts/make-fixtures.sh --small +// NEXT_PUBLIC_DATADOG_ENV=off bun run build +// bunx playwright test +// +// Two projects: "multi-threaded" (COOP/COEP set, so @ffmpeg/core-mt loads, +// as on the live site) and "single-threaded" (no isolation headers). +export default defineConfig({ + testDir: "test/e2e", + // Each test runs real FFmpeg.wasm encodes; one worker keeps CPU contention + // from making timings meaningless. + workers: 1, + fullyParallel: false, + timeout: 240_000, + expect: { timeout: 15_000 }, + retries: process.env.CI ? 1 : 0, + reporter: process.env.CI ? [["list"], ["junit", { outputFile: "test-results/e2e-junit.xml" }]] : "list", + use: { + ...devices["Desktop Chrome"], + acceptDownloads: true, + trace: "retain-on-failure", + }, + projects: [ + { name: "multi-threaded", use: { baseURL: "http://127.0.0.1:4173" } }, + { name: "single-threaded", use: { baseURL: "http://127.0.0.1:4174" } }, + ], + webServer: [ + { + command: "node scripts/serve-isolated.mjs 4173", + url: "http://127.0.0.1:4173", + reuseExistingServer: !process.env.CI, + }, + { + command: "node scripts/serve-isolated.mjs 4174", + url: "http://127.0.0.1:4174", + env: { ISOLATED: "0" }, + reuseExistingServer: !process.env.CI, + }, + ], +}); diff --git a/scripts/make-fixtures.sh b/scripts/make-fixtures.sh new file mode 100755 index 000000000..495a63284 --- /dev/null +++ b/scripts/make-fixtures.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# Recreates the video fixtures used by the end-to-end export tests. +# Output goes to test/fixtures/ (gitignored). Needs the ffmpeg CLI +# (macOS: `brew install ffmpeg`, Ubuntu: `sudo apt-get install ffmpeg`). +# +# scripts/make-fixtures.sh # all fixtures +# scripts/make-fixtures.sh --small # everything except the ~300 MB clip +set -euo pipefail + +cd "$(dirname "$0")/.." +OUT=test/fixtures +mkdir -p "$OUT" + +command -v ffmpeg >/dev/null || { echo "ffmpeg not found; install it first." >&2; exit 1; } + +ff() { ffmpeg -hide_banner -loglevel error -y "$@"; } + +# A moving test pattern with a tone, so trims, speed changes and audio edits +# are visible in the output. +video() { echo "testsrc2=size=$1:rate=30:duration=$2"; } +tone() { echo "sine=frequency=440:sample_rate=48000:duration=$1"; } + +echo "normal.mp4: 1280x720 H.264/AAC, 10 s" +ff -f lavfi -i "$(video 1280x720 10)" -f lavfi -i "$(tone 10)" \ + -c:v libx264 -pix_fmt yuv420p -preset veryfast -c:a aac -b:a 128k \ + -movflags +faststart -shortest "$OUT/normal.mp4" + +# Browser screen recordings (MediaRecorder) are written as a live stream: the +# WebM header has no duration and there are no cues, so