From c30f25816a5f44b235ea684842de9bd91933a68f Mon Sep 17 00:00:00 2001 From: Dmitriy Kovalenko Date: Sat, 31 Oct 2026 13:10:26 -0700 Subject: [PATCH 1/2] feat(video): add a playback speed setting for synced video frames --- AGENTS.md | 1 + examples/fframes-intro/src/scenes/preview.rs | 1 + examples/fframes-intro/src/scenes/video.rs | 1 + .../src/scenes/pixel_single_video.rs | 1 + examples/teej-podcast/src/teej_podcast.rs | 2 + fframes-media/src/video_decoder.rs | 36 ++++++++++++++++- fframes-media/tests/video_decoder_eof.rs | 36 +++++++++++++++++ fframes/src/frame.rs | 1 + fframes/src/video_data.rs | 39 ++++++++++++++++++- skills/fframes-video/references/api.md | 5 ++- 10 files changed, 118 insertions(+), 5 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index adceb224..475c32d8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -472,6 +472,7 @@ lets scenes cross-fade; `ctx.get_scene_info(&scene)` gives the resolved frame ra let Some(video_frame) = frame.get_synced_video_frame(ctx, "clip.mp4", &fframes::SyncVideoFrameInput { start_from: 0., looping: false, + speed: 1.5, // 1.5x from start_from on; 0.5 for slow motion, default 1.0 editor_fallback_image: ctx.get_image("clip_poster.jpg"), }) else { return Svgr::empty() }; let image = video_frame.into_image(); // diff --git a/examples/fframes-intro/src/scenes/preview.rs b/examples/fframes-intro/src/scenes/preview.rs index 1a46033b..fe5e0f83 100644 --- a/examples/fframes-intro/src/scenes/preview.rs +++ b/examples/fframes-intro/src/scenes/preview.rs @@ -59,6 +59,7 @@ impl Scene for PreviewScene { start_from: 0.0, looping: true, editor_fallback_image: None, + ..Default::default() }, ) .map(|f| f.into_image()); diff --git a/examples/fframes-intro/src/scenes/video.rs b/examples/fframes-intro/src/scenes/video.rs index 386cbc5d..6b82be4c 100644 --- a/examples/fframes-intro/src/scenes/video.rs +++ b/examples/fframes-intro/src/scenes/video.rs @@ -21,6 +21,7 @@ pub fn clip_input() -> SyncVideoFrameInput<'static> { start_from: 0.0, looping: true, editor_fallback_image: None, + ..Default::default() } } diff --git a/examples/pixel-memory/src/scenes/pixel_single_video.rs b/examples/pixel-memory/src/scenes/pixel_single_video.rs index ae3a97fd..fc2a063f 100644 --- a/examples/pixel-memory/src/scenes/pixel_single_video.rs +++ b/examples/pixel-memory/src/scenes/pixel_single_video.rs @@ -30,6 +30,7 @@ impl Scene for PixelSingleVideoScene<'_> { start_from: 0., looping: false, editor_fallback_image: ctx.get_image("10.jpg"), + ..Default::default() }, ) else { return Svgr::empty(); diff --git a/examples/teej-podcast/src/teej_podcast.rs b/examples/teej-podcast/src/teej_podcast.rs index 445d540b..446ed485 100644 --- a/examples/teej-podcast/src/teej_podcast.rs +++ b/examples/teej-podcast/src/teej_podcast.rs @@ -88,6 +88,7 @@ impl Video for TeejPodcast<'_> { start_from: 0., looping: false, editor_fallback_image: ctx.get_image("left.jpg"), + ..Default::default() }, ) .map(|fr| fr.into_image()); @@ -100,6 +101,7 @@ impl Video for TeejPodcast<'_> { start_from: 0., looping: false, editor_fallback_image: ctx.get_image("right.jpg"), + ..Default::default() }, ) .map(|fr| fr.into_image()); diff --git a/fframes-media/src/video_decoder.rs b/fframes-media/src/video_decoder.rs index 391bc7cd..cf7eb287 100644 --- a/fframes-media/src/video_decoder.rs +++ b/fframes-media/src/video_decoder.rs @@ -66,6 +66,9 @@ unsafe fn duration_tag(stream: *mut AVStream) -> Option { (total > 0).then_some(total) } +/// `FFmpegDecoder::with_playback_speed` keeps the speed to 1/1000. +const PLAYBACK_SPEED_SCALE: i32 = 1000; + const FFRAMES_VIDEO_PATH_TAG: &str = "___fframes_internal_video_frame_pts___"; // just a little bit faster than the format! macro pub fn encode_video_resource(resource: &str, pts: i64) -> String { @@ -365,7 +368,9 @@ pub struct FFmpegDecoder { fmt_ctx: *mut AVFormatContext, video_stream_info: VideoStreamInfo, pkt: *mut AVPacket, + /// Offsets are counted in this time base: `speed / target_fps` seconds per offset. custom_time_base: AVRational, + target_fps: i32, duration_in_frames: i64, /// The offset of the previous `decode_up_to` call, in `custom_time_base` units. last_offset: Option, @@ -633,6 +638,32 @@ impl FFmpegDecoder { )) } + /// Plays the stream `speed` times faster: offset `n` shows the frame at + /// `n * speed / target_fps` seconds of the stream. The speed is kept to 1/1000. + /// A speed that is not positive and finite plays at normal speed. + pub fn with_playback_speed(mut self, speed: f32) -> Self { + let speed = if speed.is_finite() && speed > 0.0 { + ((speed * PLAYBACK_SPEED_SCALE as f32).round() as i32).max(1) + } else { + PLAYBACK_SPEED_SCALE + }; + + self.custom_time_base = AVRational { + num: speed, + den: self.target_fps * PLAYBACK_SPEED_SCALE, + }; + self.duration_in_frames = unsafe { + av_rescale_q( + self.video_stream_info.duration, + self.video_stream_info.time_base, + self.custom_time_base, + ) + }; + self.last_offset = None; + self.current_loop = 0; + self + } + /// Creates a new decoder for the video file at the specified path /// # Safety /// Generally safe but uses libav functions @@ -724,6 +755,7 @@ impl FFmpegDecoder { )?), video_stream_info, custom_time_base, + target_fps: target_fps as i32, duration_in_frames, current_loop: 0, last_offset: None, @@ -942,7 +974,9 @@ impl FFmpegDecoder { // would be returned. Far jumps forward seek to the closest keyframe instead of // decoding every frame in between (frames can be requested out of order by // parallel renderers). - let seek_ahead_frames = 2 * i64::from(self.custom_time_base.den.max(1)); + // 2 seconds of the stream + let seek_ahead_frames = 2 * i64::from(self.custom_time_base.den.max(1)) + / i64::from(self.custom_time_base.num.max(1)); let needs_seek = self.last_offset.is_some_and(|last_offset| { offset < last_offset || offset - last_offset > seek_ahead_frames }); diff --git a/fframes-media/tests/video_decoder_eof.rs b/fframes-media/tests/video_decoder_eof.rs index b2c3817a..baa405c7 100644 --- a/fframes-media/tests/video_decoder_eof.rs +++ b/fframes-media/tests/video_decoder_eof.rs @@ -326,3 +326,39 @@ fn frames_can_be_converted_at_widths_that_are_not_a_multiple_of_16() { } } } + +/// Offset `n` at `num / den` speed shows the first source frame at or after `n * num / den`. +fn assert_speed(speed: f32, num: i64, den: i64) { + let video = VideoFixture::new(3); + let mut decoder = unsafe { FFmpegDecoder::new(&video.path(), FPS, 1) } + .unwrap() + .with_playback_speed(speed); + let offsets = FRAME_COUNT * den / num; + for offset in 0..offsets { + assert!( + unsafe { decoder.decode_up_to(offset) }.unwrap(), + "{speed}x: offset {offset}" + ); + let source_frame = ((offset * num + den - 1) / den).min(FRAME_COUNT - 1); + let seconds = unsafe { decoder.get_raw_frame().timestamp_seconds() }; + assert!( + (seconds - source_frame as f32 / FPS as f32).abs() < 0.0001, + "{speed}x: offset {offset} shows {seconds}s, expected frame {source_frame}" + ); + } + assert!(!unsafe { decoder.decode_up_to(offsets) }.unwrap()); + + // A loop of the faster video is shorter. + assert_eq!( + unsafe { decoder.adjust_offset_for_looping(offsets) }.unwrap(), + 0 + ); + assert_frame(&mut decoder, 0); +} + +#[test] +fn playback_speed_scales_the_source_time() { + assert_speed(2.0, 2, 1); + assert_speed(1.5, 3, 2); + assert_speed(0.5, 1, 2); +} diff --git a/fframes/src/frame.rs b/fframes/src/frame.rs index 72c1b713..6c7ff475 100644 --- a/fframes/src/frame.rs +++ b/fframes/src/frame.rs @@ -443,6 +443,7 @@ impl Frame { /// Returns a synced video frame of the video file media. /// If the video file is not found or there is no frame or error during decoding returns None. /// Video frame is decoded and timebase (fps) is synced with the target fframe's video frame. + /// Each frame shows the source frame at `(seconds - start_from) * speed`. /// /// **Make sure this function is not working in the editor for now. Editor will automatically fallback to /// `{your_video_file}.{your_extension}_fallback.{jpg|png}` image file for preview if it exists diff --git a/fframes/src/video_data.rs b/fframes/src/video_data.rs index 2483397e..5b1623e0 100644 --- a/fframes/src/video_data.rs +++ b/fframes/src/video_data.rs @@ -73,7 +73,7 @@ pub trait FFramesSyncedVideoFrame<'media> { fn stream_duration_in_seconds(&self) -> f32; } -#[derive(Debug, Clone, Copy, Default)] +#[derive(Debug, Clone, Copy)] pub struct SyncVideoFrameInput<'media> { /// The seconds timestamp in seconds to start playing the video from /// If not provided starts playing from the beginning of the video or scene. @@ -82,11 +82,28 @@ pub struct SyncVideoFrameInput<'media> { /// Also if `true` the `get_synced_video_frame` API should never return None, only in case of /// decoding errors, so in some cases it might be okay to panic if the frame is not available. pub looping: bool, + /// Playback speed of the video: `2.0` plays it twice as fast, `0.5` at half speed. + /// Every frame shows the source frame at `seconds since start_from * speed`, so the + /// motion stays even at any speed and frame rate. With `looping` a loop takes + /// `duration / speed` seconds. The speed is kept to 1/1000; a value that is not + /// positive and finite plays at normal speed. Default `1.0`. + pub speed: f32, /// The wasm preview editor is not supporting decoding video frames in runtime /// so this allows to provide a fallback image for the debugging pub editor_fallback_image: Option<&'media ImageData<'media>>, } +impl Default for SyncVideoFrameInput<'_> { + fn default() -> Self { + Self { + start_from: 0.0, + looping: false, + speed: 1.0, + editor_fallback_image: None, + } + } +} + /// This is used by the editor as a fallback for the static image #[derive(Debug, Clone)] pub struct WasmEditorVideoFrameFallback<'media> { @@ -237,6 +254,15 @@ impl VideoDecodersWorker { return Ok(None); }; + // The editor decodes on the grid of the video's frames. The speed is kept to + // 1/1000 like in `FFmpegDecoder::with_playback_speed`. + let speed = if options.speed.is_finite() && options.speed > 0.0 { + ((options.speed * 1000.0).round() as i64).max(1) + } else { + 1000 + }; + let offset = (offset * speed + 999) / 1000; + let last_frame_to_display_image = metadata.duration * ctx.time_base.fps as f32; if offset > last_frame_to_display_image as i64 && !options.looping { return Ok(None); @@ -316,6 +342,14 @@ impl VideoDecodersWorker { .ok_or_else(|| FFramesError::MediaError(FFramesMediaError::MediaDirectoryProvided))? .to_string_lossy(); + // Every speed gets its own decoder, offsets of different speeds point to + // different parts of the file. + let resource_name = if options.speed == 1.0 { + resource_name + } else { + format!("{resource_name}@{}x", options.speed).into() + }; + // It is important to lock it for the whole duration of he function to // avoid any parallel encoder creation. let mut map_guard = self.map.write()?; @@ -325,7 +359,8 @@ impl VideoDecodersWorker { if needs_new_decoder { let decoder = unsafe { let mut decoder = - fframes_media::FFmpegDecoder::new(path, ctx.time_base.fps, self.buffer_size)?; + fframes_media::FFmpegDecoder::new(path, ctx.time_base.fps, self.buffer_size)? + .with_playback_speed(options.speed); if offset > 0 { decoder.seek_to_offset(offset)?; diff --git a/skills/fframes-video/references/api.md b/skills/fframes-video/references/api.md index a39a99f4..47f75f84 100644 --- a/skills/fframes-video/references/api.md +++ b/skills/fframes-video/references/api.md @@ -172,7 +172,8 @@ family, size and weight in the `` attributes. `text-anchor="middle"` cente `include_media_dir!(pub struct MyMedia, "media", stereo)` keeps stereo files stereo. - Large or stereo files: `let dir = MediaDirectory::read_folder("assets")?; let media = dir.process_media_source()?;` and combine providers with `CombinedMediaProvider::from([&static_media as &dyn MediaProvider, &media])`. -- Video frames: `frame.get_synced_video_frame(ctx, "clip.mp4", &SyncVideoFrameInput { start_from, looping, editor_fallback_image })` +- Video frames: `frame.get_synced_video_frame(ctx, "clip.mp4", &SyncVideoFrameInput { start_from, looping, speed, editor_fallback_image })` + (`speed: 2.0` plays twice as fast, `..Default::default()` keeps 1.0) then `` (needs the `compile-time-svgtree` feature). - Subtitles: `ctx.get_subtitles("subs.vtt").and_then(|s| frame.get_subtitle_phrase(s))`. - Audio spectrum: `frame.visualize_audio_frame(VisualizeFrameInput { audio, sample_size: SampleSize::S256, smooth_level: 4, window: Some(WindowFunction::Hann) })`. @@ -233,7 +234,7 @@ fframes::svgr!( Date: Sat, 31 Oct 2026 13:25:45 -0700 Subject: [PATCH 2/2] fix(video): use the exact stream duration and share decoders of equal speeds --- fframes-media/src/video_decoder.rs | 147 +++++++++++----------- fframes-media/src/video_types.rs | 46 +++++++ fframes-media/tests/video_decoder_eof.rs | 74 +++++++---- fframes/src/video_data.rs | 150 ++++++++++++++--------- 4 files changed, 262 insertions(+), 155 deletions(-) diff --git a/fframes-media/src/video_decoder.rs b/fframes-media/src/video_decoder.rs index cf7eb287..fa7525b6 100644 --- a/fframes-media/src/video_decoder.rs +++ b/fframes-media/src/video_decoder.rs @@ -1,6 +1,6 @@ use crate::FFramesMediaError; use crate::error::Result; -use crate::video_types::FrameConvertOptions; +use crate::video_types::{FrameConvertOptions, PlaybackSpeed}; use ffmpeg_sys_fframes::SwsFlags::{SWS_BICUBIC, SWS_BILINEAR}; use ffmpeg_sys_fframes::*; use std::cell::UnsafeCell; @@ -66,9 +66,6 @@ unsafe fn duration_tag(stream: *mut AVStream) -> Option { (total > 0).then_some(total) } -/// `FFmpegDecoder::with_playback_speed` keeps the speed to 1/1000. -const PLAYBACK_SPEED_SCALE: i32 = 1000; - const FFRAMES_VIDEO_PATH_TAG: &str = "___fframes_internal_video_frame_pts___"; // just a little bit faster than the format! macro pub fn encode_video_resource(resource: &str, pts: i64) -> String { @@ -371,9 +368,8 @@ pub struct FFmpegDecoder { /// Offsets are counted in this time base: `speed / target_fps` seconds per offset. custom_time_base: AVRational, target_fps: i32, - duration_in_frames: i64, - /// The offset of the previous `decode_up_to` call, in `custom_time_base` units. - last_offset: Option, + /// The stream position of the previous decode or seek, in the stream's time base. + last_pts: Option, /// A null packet has been sent; receive delayed frames until decoder EOF. draining: bool, /// The target frame holds a frame decoded since the last seek. @@ -639,29 +635,32 @@ impl FFmpegDecoder { } /// Plays the stream `speed` times faster: offset `n` shows the frame at - /// `n * speed / target_fps` seconds of the stream. The speed is kept to 1/1000. - /// A speed that is not positive and finite plays at normal speed. - pub fn with_playback_speed(mut self, speed: f32) -> Self { - let speed = if speed.is_finite() && speed > 0.0 { - ((speed * PLAYBACK_SPEED_SCALE as f32).round() as i32).max(1) - } else { - PLAYBACK_SPEED_SCALE - }; - + /// `n * speed / target_fps` seconds of the stream. + pub fn with_playback_speed(mut self, speed: PlaybackSpeed) -> Self { self.custom_time_base = AVRational { - num: speed, - den: self.target_fps * PLAYBACK_SPEED_SCALE, + num: speed.thousandths(), + den: self.target_fps * PlaybackSpeed::SCALE, }; - self.duration_in_frames = unsafe { + self.last_pts = None; + self.current_loop = 0; + self + } + + /// The stream position of the offset, in the stream's time base. + fn offset_pts(&self, offset: i64) -> i64 { + unsafe { av_rescale_q( - self.video_stream_info.duration, - self.video_stream_info.time_base, + offset, self.custom_time_base, + self.video_stream_info.time_base, ) - }; - self.last_offset = None; - self.current_loop = 0; - self + } + } + + /// The exact stream duration, in the stream's time base. `None` if it is unknown. + fn stream_duration(&self) -> Option { + let duration = self.video_stream_info.duration; + (duration > 0).then_some(duration) } /// Creates a new decoder for the video file at the specified path @@ -737,12 +736,6 @@ impl FFmpegDecoder { den: target_fps as i32, }; - let duration_in_frames = av_rescale_q( - video_stream_info.duration, - video_stream_info.time_base, - custom_time_base, - ); - Ok(FFmpegDecoder { pkt, fmt_ctx, @@ -756,9 +749,8 @@ impl FFmpegDecoder { video_stream_info, custom_time_base, target_fps: target_fps as i32, - duration_in_frames, current_loop: 0, - last_offset: None, + last_pts: None, draining: false, has_decoded_frame: false, }) @@ -865,13 +857,11 @@ impl FFmpegDecoder { /// # Safety /// Generally safe but uses libav functions pub unsafe fn seek_to_offset(&mut self, offset: i64) -> Result<()> { - unsafe { - let timestamp = av_rescale_q( - offset, - self.custom_time_base, - self.video_stream_info.time_base, - ); + unsafe { self.seek_to_pts(self.offset_pts(offset)) } + } + unsafe fn seek_to_pts(&mut self, timestamp: i64) -> Result<()> { + unsafe { let ret = av_seek_frame( self.fmt_ctx, self.video_stream_info.stream_index, @@ -890,33 +880,42 @@ impl FFmpegDecoder { avcodec_flush_buffers(self.video_stream_info.codec_ctx); self.draining = false; self.has_decoded_frame = false; + self.last_pts = Some(timestamp); av_packet_unref(self.pkt); Ok(()) } } - /// Moves offset to the start of the video if need to loop + /// Decodes the frame at the offset of the video played in a loop. The position in + /// the loop comes from the exact stream duration, so loops at a fractional speed + /// do not drift and do not end early. /// # Safety /// Generally safe but uses libav functions - pub unsafe fn adjust_offset_for_looping(&mut self, offset: i64) -> Result { - // An unknown or empty duration can't be looped over (and would divide by zero below). - if self.duration_in_frames <= 0 || offset < self.duration_in_frames { - return Ok(offset); - } + pub unsafe fn decode_looping(&mut self, offset: i64) -> Result { + unsafe { + let pts = self.offset_pts(offset); + // An unknown duration can't be looped over (and would divide by zero below). + let Some(duration) = self.stream_duration() else { + return self.decode_up_to_pts(pts); + }; - let new_loop_index = offset / self.duration_in_frames; - let new_offset = offset % self.duration_in_frames; - if new_loop_index == self.current_loop { - return Ok(new_offset); - } + let loop_index = pts.div_euclid(duration); + let pts = pts.rem_euclid(duration); + if loop_index != self.current_loop { + self.current_loop = loop_index; + self.seek_to_pts(pts)?; + } - self.current_loop = new_loop_index; - unsafe { - self.seek_to_offset(new_offset)?; - } + if self.decode_up_to_pts(pts)? { + return Ok(true); + } - Ok(new_offset) + // The frames end before the stream's duration: start the next loop. + self.current_loop += 1; + self.seek_to_pts(0)?; + self.decode_up_to_pts(0) + } } /// If we are actually using hardware accelerated frames it is required to correctly @@ -969,27 +968,25 @@ impl FFmpegDecoder { /// # Safety /// Generally safe but uses libav functions pub unsafe fn decode_up_to(&mut self, offset: i64) -> Result { + unsafe { self.decode_up_to_pts(self.offset_pts(offset)) } + } + + unsafe fn decode_up_to_pts(&mut self, target_pts: i64) -> Result { unsafe { // Going back needs a seek, otherwise the newer frame that was already decoded - // would be returned. Far jumps forward seek to the closest keyframe instead of - // decoding every frame in between (frames can be requested out of order by - // parallel renderers). - // 2 seconds of the stream - let seek_ahead_frames = 2 * i64::from(self.custom_time_base.den.max(1)) - / i64::from(self.custom_time_base.num.max(1)); - let needs_seek = self.last_offset.is_some_and(|last_offset| { - offset < last_offset || offset - last_offset > seek_ahead_frames - }); - self.last_offset = Some(offset); - if needs_seek { - self.seek_to_offset(offset)?; - } - - let target_pts = av_rescale_q( - offset, - self.custom_time_base, + // would be returned. Far jumps forward (more than 2 seconds of the stream) seek + // to the closest keyframe instead of decoding every frame in between (frames can + // be requested out of order by parallel renderers). A new decoder is at the start. + let seek_ahead = av_rescale_q( + 2, + AVRational { num: 1, den: 1 }, self.video_stream_info.time_base, ); + let last_pts = self.last_pts.unwrap_or(0); + if target_pts < last_pts || target_pts - last_pts > seek_ahead { + self.seek_to_pts(target_pts)?; + } + self.last_pts = Some(target_pts); if (*self.frame_buf.latest_av_frame).pts >= target_pts { return Ok(true); @@ -1020,8 +1017,10 @@ impl FFmpegDecoder { // The target lies after the last frame's timestamp but before the end of // the stream (e.g. a 24 fps clip sampled at 30 fps): the last frame is // still the one on screen, so return it instead of reporting the end. - let shows_last_frame = - self.has_decoded_frame && offset < self.duration_in_frames; + let shows_last_frame = self.has_decoded_frame + && self + .stream_duration() + .is_some_and(|duration| target_pts < duration); if shows_last_frame { self.transfer_hardware_surface_data(target_frame)?; } diff --git a/fframes-media/src/video_types.rs b/fframes-media/src/video_types.rs index c2f79684..59e58ead 100644 --- a/fframes-media/src/video_types.rs +++ b/fframes-media/src/video_types.rs @@ -27,3 +27,49 @@ pub struct VideoMedia { pub metadata: Option, pub path: PathBuf, } + +/// Playback speed of a video source, kept to 1/1000. Speeds that round to the same +/// value are equal, so they share a decoder. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct PlaybackSpeed(i32); + +impl PlaybackSpeed { + /// The denominator of `thousandths`. + pub const SCALE: i32 = 1000; + pub const NORMAL: Self = Self(Self::SCALE); + + /// A speed that is not positive and finite plays at normal speed. + pub fn new(speed: f32) -> Self { + if speed.is_finite() && speed > 0.0 { + Self(((speed * Self::SCALE as f32).round() as i32).max(1)) + } else { + Self::NORMAL + } + } + + /// The speed in 1/1000. + pub fn thousandths(self) -> i32 { + self.0 + } +} + +impl Default for PlaybackSpeed { + fn default() -> Self { + Self::NORMAL + } +} + +#[cfg(test)] +mod playback_speed_tests { + use super::PlaybackSpeed; + + #[test] + fn speeds_are_kept_to_a_thousandth() { + assert_eq!(PlaybackSpeed::new(1.000_099), PlaybackSpeed::NORMAL); + assert_eq!(PlaybackSpeed::new(1.4).thousandths(), 1400); + assert_eq!(PlaybackSpeed::new(0.0001).thousandths(), 1); + for invalid in [0.0, -2.0, f32::NAN, f32::INFINITY] { + assert_eq!(PlaybackSpeed::new(invalid), PlaybackSpeed::NORMAL); + } + } +} diff --git a/fframes-media/tests/video_decoder_eof.rs b/fframes-media/tests/video_decoder_eof.rs index baa405c7..ecca2a30 100644 --- a/fframes-media/tests/video_decoder_eof.rs +++ b/fframes-media/tests/video_decoder_eof.rs @@ -5,7 +5,7 @@ use std::path::PathBuf; use std::process::Command; use std::sync::atomic::{AtomicUsize, Ordering}; -use fframes_media::{FFmpegDecoder, FrameConvertOptions, ResizeVideoFrame}; +use fframes_media::{FFmpegDecoder, FrameConvertOptions, PlaybackSpeed, ResizeVideoFrame}; const FPS: usize = 30; const FRAME_COUNT: i64 = 300; @@ -118,9 +118,8 @@ fn b_frames_are_drained_and_decoding_can_restart_after_eof() { assert_frame(&mut decoder, FRAME_COUNT - 1); // Looping also seeks after decoding the delayed tail. - let offset = unsafe { decoder.adjust_offset_for_looping(FRAME_COUNT) }.unwrap(); - assert_eq!(offset, 0); - assert_frame(&mut decoder, offset); + assert!(unsafe { decoder.decode_looping(FRAME_COUNT) }.unwrap()); + assert_frame(&mut decoder, 0); assert_frame(&mut decoder, FRAME_COUNT - 1); } @@ -254,8 +253,7 @@ fn last_frame_is_shown_until_the_stream_ends_at_a_higher_fps() { for _ in 0..2 { assert!(!unsafe { decoder.decode_up_to(60) }.unwrap()); } - let offset = unsafe { decoder.adjust_offset_for_looping(60) }.unwrap(); - assert_eq!(offset, 0); + assert!(unsafe { decoder.decode_looping(60) }.unwrap()); assert_frame(&mut decoder, 0); // A seek straight to the tail must not return a frame left over from before the seek. @@ -327,38 +325,70 @@ fn frames_can_be_converted_at_widths_that_are_not_a_multiple_of_16() { } } -/// Offset `n` at `num / den` speed shows the first source frame at or after `n * num / den`. +/// The source frame that offset `n` shows at `num / den` speed: the first frame at or +/// after `n * num / den` (the last frame until the stream ends), in a loop if `looping`. +fn expected_frame(offset: i64, num: i64, den: i64, looping: bool) -> i64 { + let mut position = offset * num; + if looping { + position %= FRAME_COUNT * den; + } + ((position + den - 1) / den).min(FRAME_COUNT - 1) +} + +fn assert_shows(decoder: &FFmpegDecoder, speed: f32, offset: i64, frame: i64) { + let seconds = unsafe { decoder.get_raw_frame().timestamp_seconds() }; + assert!( + (seconds - frame as f32 / FPS as f32).abs() < 0.0001, + "{speed}x: offset {offset} shows {seconds}s, expected frame {frame}" + ); +} + fn assert_speed(speed: f32, num: i64, den: i64) { let video = VideoFixture::new(3); - let mut decoder = unsafe { FFmpegDecoder::new(&video.path(), FPS, 1) } - .unwrap() - .with_playback_speed(speed); - let offsets = FRAME_COUNT * den / num; + let speed_of = |speed| { + unsafe { FFmpegDecoder::new(&video.path(), FPS, 1) } + .unwrap() + .with_playback_speed(PlaybackSpeed::new(speed)) + }; + + // Every offset before the end of the stream has a frame, also when the stream's + // duration is not a whole number of offsets. + let mut decoder = speed_of(speed); + let offsets = (FRAME_COUNT * den + num - 1) / num; for offset in 0..offsets { assert!( unsafe { decoder.decode_up_to(offset) }.unwrap(), "{speed}x: offset {offset}" ); - let source_frame = ((offset * num + den - 1) / den).min(FRAME_COUNT - 1); - let seconds = unsafe { decoder.get_raw_frame().timestamp_seconds() }; - assert!( - (seconds - source_frame as f32 / FPS as f32).abs() < 0.0001, - "{speed}x: offset {offset} shows {seconds}s, expected frame {source_frame}" + assert_shows( + &decoder, + speed, + offset, + expected_frame(offset, num, den, false), ); } assert!(!unsafe { decoder.decode_up_to(offsets) }.unwrap()); - // A loop of the faster video is shorter. - assert_eq!( - unsafe { decoder.adjust_offset_for_looping(offsets) }.unwrap(), - 0 - ); - assert_frame(&mut decoder, 0); + // Loops follow the exact duration and do not drift. + let mut decoder = speed_of(speed); + for offset in 0..3 * offsets { + assert!( + unsafe { decoder.decode_looping(offset) }.unwrap(), + "{speed}x looping: offset {offset}" + ); + assert_shows( + &decoder, + speed, + offset, + expected_frame(offset, num, den, true), + ); + } } #[test] fn playback_speed_scales_the_source_time() { assert_speed(2.0, 2, 1); assert_speed(1.5, 3, 2); + assert_speed(1.4, 7, 5); assert_speed(0.5, 1, 2); } diff --git a/fframes/src/video_data.rs b/fframes/src/video_data.rs index 5b1623e0..1fab172f 100644 --- a/fframes/src/video_data.rs +++ b/fframes/src/video_data.rs @@ -6,7 +6,7 @@ use std::sync::Arc; use crate::error::Result; #[cfg(not(target_arch = "wasm32"))] use fframes_media::ImageMetadata; -pub use fframes_media::{FrameConvertOptions, ResizeVideoFrame}; +pub use fframes_media::{FrameConvertOptions, PlaybackSpeed, ResizeVideoFrame}; #[cfg(not(target_arch = "wasm32"))] use std::collections::HashMap; @@ -85,8 +85,10 @@ pub struct SyncVideoFrameInput<'media> { /// Playback speed of the video: `2.0` plays it twice as fast, `0.5` at half speed. /// Every frame shows the source frame at `seconds since start_from * speed`, so the /// motion stays even at any speed and frame rate. With `looping` a loop takes - /// `duration / speed` seconds. The speed is kept to 1/1000; a value that is not - /// positive and finite plays at normal speed. Default `1.0`. + /// `duration / speed` seconds. The speed is kept to 1/1000 (`PlaybackSpeed`); a value + /// that is not positive and finite plays at normal speed. Default `1.0`. + /// The speed is constant: each speed of a file has its own decoder, and the + /// renderer keeps the 4 used last. pub speed: f32, /// The wasm preview editor is not supporting decoding video frames in runtime /// so this allows to provide a fallback image for the debugging @@ -216,9 +218,18 @@ impl<'media> FFramesSyncedVideoFrame<'media> for crate::media::FFmpegFrameBuf { pub struct VideoDecodersWorker { buffer_size: usize, #[cfg(not(target_arch = "wasm32"))] - map: Arc>>, + map: Arc>>>, } +/// A decoder of a file at one playback speed. +#[cfg(not(target_arch = "wasm32"))] +type SpeedDecoder = (PlaybackSpeed, fframes_media::FFmpegDecoder); + +/// Each decoder keeps the file open and its frames in memory. A file played at more +/// speeds closes the decoder that was used least recently. +#[cfg(not(target_arch = "wasm32"))] +const MAX_DECODERS_PER_VIDEO: usize = 4; + impl std::fmt::Debug for VideoDecodersWorker { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("WorkerLocalDecoders").finish() @@ -254,14 +265,10 @@ impl VideoDecodersWorker { return Ok(None); }; - // The editor decodes on the grid of the video's frames. The speed is kept to - // 1/1000 like in `FFmpegDecoder::with_playback_speed`. - let speed = if options.speed.is_finite() && options.speed > 0.0 { - ((options.speed * 1000.0).round() as i64).max(1) - } else { - 1000 - }; - let offset = (offset * speed + 999) / 1000; + // The editor decodes on the grid of the video's frames. + let speed = i64::from(PlaybackSpeed::new(options.speed).thousandths()); + let scale = i64::from(PlaybackSpeed::SCALE); + let offset = (offset * speed + scale - 1) / scale; let last_frame_to_display_image = metadata.duration * ctx.time_base.fps as f32; if offset > last_frame_to_display_image as i64 && !options.looping { @@ -316,12 +323,11 @@ impl VideoDecodersWorker { pts: i64, filename: &str, ) -> Result>> { - let frame = self - .map - .read() - .unwrap() - .get(filename) - .and_then(|decoder| decoder.get_decoded_image_in_buf(pts)); + let frame = self.map.read().unwrap().get(filename).and_then(|decoders| { + decoders + .iter() + .find_map(|(_, decoder)| decoder.get_decoded_image_in_buf(pts)) + }); Ok(frame) } @@ -330,7 +336,7 @@ impl VideoDecodersWorker { pub(crate) fn get_synced_frame<'media>( &self, VideoMedia { path, .. }: &VideoMedia, - mut offset: i64, + offset: i64, ctx: &FFramesContext<'_, 'media>, options: &SyncVideoFrameInput, ) -> Result + 'media>>> { @@ -342,57 +348,83 @@ impl VideoDecodersWorker { .ok_or_else(|| FFramesError::MediaError(FFramesMediaError::MediaDirectoryProvided))? .to_string_lossy(); - // Every speed gets its own decoder, offsets of different speeds point to - // different parts of the file. - let resource_name = if options.speed == 1.0 { - resource_name - } else { - format!("{resource_name}@{}x", options.speed).into() - }; + let speed = PlaybackSpeed::new(options.speed); // It is important to lock it for the whole duration of he function to // avoid any parallel encoder creation. let mut map_guard = self.map.write()?; - let needs_new_decoder = !map_guard.contains_key(resource_name.as_ref()); - - // Create new decoder if needed - if needs_new_decoder { - let decoder = unsafe { - let mut decoder = - fframes_media::FFmpegDecoder::new(path, ctx.time_base.fps, self.buffer_size)? - .with_playback_speed(options.speed); - - if offset > 0 { - decoder.seek_to_offset(offset)?; - } - - decoder + // Every speed gets its own decoder, offsets of different speeds point to + // different parts of the file. + let decoders = map_guard.entry(resource_name.into_owned()).or_default(); + let decoder = decoder_for_speed(decoders, speed, || unsafe { + fframes_media::FFmpegDecoder::new(path, ctx.time_base.fps, self.buffer_size) + .map(|decoder| decoder.with_playback_speed(speed)) + })?; + unsafe { + let has_frame = if options.looping { + decoder.decode_looping(offset)? + } else { + decoder.decode_up_to(offset)? }; - - map_guard.insert(resource_name.to_string(), decoder); + Ok(has_frame.then(|| decoder.get_raw_frame())) } + } +} - unsafe { - // Use write lock for modifying decoder state - let decoder = map_guard.get_mut(resource_name.as_ref()).unwrap(); +/// The decoder of `speed`, made with `create` if there is none. The most recently used +/// decoder is first, and at most `MAX_DECODERS_PER_VIDEO` are kept. +#[cfg(not(target_arch = "wasm32"))] +fn decoder_for_speed( + decoders: &mut Vec<(PlaybackSpeed, D)>, + speed: PlaybackSpeed, + create: impl FnOnce() -> std::result::Result, +) -> std::result::Result<&mut D, E> { + if let Some(index) = decoders.iter().position(|(s, _)| *s == speed) { + decoders[..=index].rotate_right(1); + } else { + decoders.insert(0, (speed, create()?)); + decoders.truncate(MAX_DECODERS_PER_VIDEO); + } + Ok(&mut decoders[0].1) +} - if options.looping { - offset = decoder.adjust_offset_for_looping(offset)?; - } +#[cfg(all(test, not(target_arch = "wasm32")))] +mod decoder_cache_tests { + use super::*; - let has_frame = decoder.decode_up_to(offset)?; - if has_frame { - Ok(Some(decoder.get_raw_frame())) - } else if options.looping { - decoder.seek_to_offset(0)?; - let has_first_frame = decoder.decode_up_to(0)?; + fn use_speed(decoders: &mut Vec<(PlaybackSpeed, f32)>, speed: f32, created: &mut usize) { + decoder_for_speed(decoders, PlaybackSpeed::new(speed), || { + *created += 1; + Ok::<_, ()>(speed) + }) + .unwrap(); + } - decoder.current_loop += 1; - Ok(has_first_frame.then_some(decoder.get_raw_frame())) - } else { - Ok(None) - } + #[test] + fn speeds_that_round_to_the_same_value_share_a_decoder() { + let (mut decoders, mut created) = (Vec::new(), 0); + for i in 0..100 { + use_speed(&mut decoders, 1.0 + i as f32 * 0.000_001, &mut created); + } + assert_eq!((decoders.len(), created), (1, 1)); + } + + #[test] + fn least_recently_used_decoders_are_closed() { + let (mut decoders, mut created) = (Vec::new(), 0); + for i in 0..100 { + use_speed(&mut decoders, 1.0 + i as f32 * 0.1, &mut created); } + assert_eq!((decoders.len(), created), (MAX_DECODERS_PER_VIDEO, 100)); + + // A used decoder moves to the front and is kept. + use_speed(&mut decoders, 1.0 + 97.0 * 0.1, &mut created); + use_speed(&mut decoders, 0.25, &mut created); + use_speed(&mut decoders, 0.5, &mut created); + use_speed(&mut decoders, 1.0 + 97.0 * 0.1, &mut created); + assert_eq!(created, 102); + let speeds: Vec<_> = decoders.iter().map(|(s, _)| s.thousandths()).collect(); + assert_eq!(speeds, [10_700, 500, 250, 10_900]); } }