diff --git a/AGENTS.md b/AGENTS.md
index adceb224..475c32d8 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -472,6 +472,7 @@ lets scenes cross-fade; `ctx.get_scene_info(&scene)` gives the resolved frame ra
let Some(video_frame) = frame.get_synced_video_frame(ctx, "clip.mp4", &fframes::SyncVideoFrameInput {
start_from: 0.,
looping: false,
+ speed: 1.5, // 1.5x from start_from on; 0.5 for slow motion, default 1.0
editor_fallback_image: ctx.get_image("clip_poster.jpg"),
}) else { return Svgr::empty() };
let image = video_frame.into_image(); //
diff --git a/examples/fframes-intro/src/scenes/preview.rs b/examples/fframes-intro/src/scenes/preview.rs
index 1a46033b..fe5e0f83 100644
--- a/examples/fframes-intro/src/scenes/preview.rs
+++ b/examples/fframes-intro/src/scenes/preview.rs
@@ -59,6 +59,7 @@ impl Scene for PreviewScene {
start_from: 0.0,
looping: true,
editor_fallback_image: None,
+ ..Default::default()
},
)
.map(|f| f.into_image());
diff --git a/examples/fframes-intro/src/scenes/video.rs b/examples/fframes-intro/src/scenes/video.rs
index 386cbc5d..6b82be4c 100644
--- a/examples/fframes-intro/src/scenes/video.rs
+++ b/examples/fframes-intro/src/scenes/video.rs
@@ -21,6 +21,7 @@ pub fn clip_input() -> SyncVideoFrameInput<'static> {
start_from: 0.0,
looping: true,
editor_fallback_image: None,
+ ..Default::default()
}
}
diff --git a/examples/pixel-memory/src/scenes/pixel_single_video.rs b/examples/pixel-memory/src/scenes/pixel_single_video.rs
index ae3a97fd..fc2a063f 100644
--- a/examples/pixel-memory/src/scenes/pixel_single_video.rs
+++ b/examples/pixel-memory/src/scenes/pixel_single_video.rs
@@ -30,6 +30,7 @@ impl Scene for PixelSingleVideoScene<'_> {
start_from: 0.,
looping: false,
editor_fallback_image: ctx.get_image("10.jpg"),
+ ..Default::default()
},
) else {
return Svgr::empty();
diff --git a/examples/teej-podcast/src/teej_podcast.rs b/examples/teej-podcast/src/teej_podcast.rs
index 445d540b..446ed485 100644
--- a/examples/teej-podcast/src/teej_podcast.rs
+++ b/examples/teej-podcast/src/teej_podcast.rs
@@ -88,6 +88,7 @@ impl Video for TeejPodcast<'_> {
start_from: 0.,
looping: false,
editor_fallback_image: ctx.get_image("left.jpg"),
+ ..Default::default()
},
)
.map(|fr| fr.into_image());
@@ -100,6 +101,7 @@ impl Video for TeejPodcast<'_> {
start_from: 0.,
looping: false,
editor_fallback_image: ctx.get_image("right.jpg"),
+ ..Default::default()
},
)
.map(|fr| fr.into_image());
diff --git a/fframes-media/src/video_decoder.rs b/fframes-media/src/video_decoder.rs
index 391bc7cd..fa7525b6 100644
--- a/fframes-media/src/video_decoder.rs
+++ b/fframes-media/src/video_decoder.rs
@@ -1,6 +1,6 @@
use crate::FFramesMediaError;
use crate::error::Result;
-use crate::video_types::FrameConvertOptions;
+use crate::video_types::{FrameConvertOptions, PlaybackSpeed};
use ffmpeg_sys_fframes::SwsFlags::{SWS_BICUBIC, SWS_BILINEAR};
use ffmpeg_sys_fframes::*;
use std::cell::UnsafeCell;
@@ -365,10 +365,11 @@ pub struct FFmpegDecoder {
fmt_ctx: *mut AVFormatContext,
video_stream_info: VideoStreamInfo,
pkt: *mut AVPacket,
+ /// Offsets are counted in this time base: `speed / target_fps` seconds per offset.
custom_time_base: AVRational,
- duration_in_frames: i64,
- /// The offset of the previous `decode_up_to` call, in `custom_time_base` units.
- last_offset: Option,
+ target_fps: i32,
+ /// The stream position of the previous decode or seek, in the stream's time base.
+ last_pts: Option,
/// A null packet has been sent; receive delayed frames until decoder EOF.
draining: bool,
/// The target frame holds a frame decoded since the last seek.
@@ -633,6 +634,35 @@ impl FFmpegDecoder {
))
}
+ /// Plays the stream `speed` times faster: offset `n` shows the frame at
+ /// `n * speed / target_fps` seconds of the stream.
+ pub fn with_playback_speed(mut self, speed: PlaybackSpeed) -> Self {
+ self.custom_time_base = AVRational {
+ num: speed.thousandths(),
+ den: self.target_fps * PlaybackSpeed::SCALE,
+ };
+ self.last_pts = None;
+ self.current_loop = 0;
+ self
+ }
+
+ /// The stream position of the offset, in the stream's time base.
+ fn offset_pts(&self, offset: i64) -> i64 {
+ unsafe {
+ av_rescale_q(
+ offset,
+ self.custom_time_base,
+ self.video_stream_info.time_base,
+ )
+ }
+ }
+
+ /// The exact stream duration, in the stream's time base. `None` if it is unknown.
+ fn stream_duration(&self) -> Option {
+ let duration = self.video_stream_info.duration;
+ (duration > 0).then_some(duration)
+ }
+
/// Creates a new decoder for the video file at the specified path
/// # Safety
/// Generally safe but uses libav functions
@@ -706,12 +736,6 @@ impl FFmpegDecoder {
den: target_fps as i32,
};
- let duration_in_frames = av_rescale_q(
- video_stream_info.duration,
- video_stream_info.time_base,
- custom_time_base,
- );
-
Ok(FFmpegDecoder {
pkt,
fmt_ctx,
@@ -724,9 +748,9 @@ impl FFmpegDecoder {
)?),
video_stream_info,
custom_time_base,
- duration_in_frames,
+ target_fps: target_fps as i32,
current_loop: 0,
- last_offset: None,
+ last_pts: None,
draining: false,
has_decoded_frame: false,
})
@@ -833,13 +857,11 @@ impl FFmpegDecoder {
/// # Safety
/// Generally safe but uses libav functions
pub unsafe fn seek_to_offset(&mut self, offset: i64) -> Result<()> {
- unsafe {
- let timestamp = av_rescale_q(
- offset,
- self.custom_time_base,
- self.video_stream_info.time_base,
- );
+ unsafe { self.seek_to_pts(self.offset_pts(offset)) }
+ }
+ unsafe fn seek_to_pts(&mut self, timestamp: i64) -> Result<()> {
+ unsafe {
let ret = av_seek_frame(
self.fmt_ctx,
self.video_stream_info.stream_index,
@@ -858,33 +880,42 @@ impl FFmpegDecoder {
avcodec_flush_buffers(self.video_stream_info.codec_ctx);
self.draining = false;
self.has_decoded_frame = false;
+ self.last_pts = Some(timestamp);
av_packet_unref(self.pkt);
Ok(())
}
}
- /// Moves offset to the start of the video if need to loop
+ /// Decodes the frame at the offset of the video played in a loop. The position in
+ /// the loop comes from the exact stream duration, so loops at a fractional speed
+ /// do not drift and do not end early.
/// # Safety
/// Generally safe but uses libav functions
- pub unsafe fn adjust_offset_for_looping(&mut self, offset: i64) -> Result {
- // An unknown or empty duration can't be looped over (and would divide by zero below).
- if self.duration_in_frames <= 0 || offset < self.duration_in_frames {
- return Ok(offset);
- }
+ pub unsafe fn decode_looping(&mut self, offset: i64) -> Result {
+ unsafe {
+ let pts = self.offset_pts(offset);
+ // An unknown duration can't be looped over (and would divide by zero below).
+ let Some(duration) = self.stream_duration() else {
+ return self.decode_up_to_pts(pts);
+ };
- let new_loop_index = offset / self.duration_in_frames;
- let new_offset = offset % self.duration_in_frames;
- if new_loop_index == self.current_loop {
- return Ok(new_offset);
- }
+ let loop_index = pts.div_euclid(duration);
+ let pts = pts.rem_euclid(duration);
+ if loop_index != self.current_loop {
+ self.current_loop = loop_index;
+ self.seek_to_pts(pts)?;
+ }
- self.current_loop = new_loop_index;
- unsafe {
- self.seek_to_offset(new_offset)?;
- }
+ if self.decode_up_to_pts(pts)? {
+ return Ok(true);
+ }
- Ok(new_offset)
+ // The frames end before the stream's duration: start the next loop.
+ self.current_loop += 1;
+ self.seek_to_pts(0)?;
+ self.decode_up_to_pts(0)
+ }
}
/// If we are actually using hardware accelerated frames it is required to correctly
@@ -937,25 +968,25 @@ impl FFmpegDecoder {
/// # Safety
/// Generally safe but uses libav functions
pub unsafe fn decode_up_to(&mut self, offset: i64) -> Result {
+ unsafe { self.decode_up_to_pts(self.offset_pts(offset)) }
+ }
+
+ unsafe fn decode_up_to_pts(&mut self, target_pts: i64) -> Result {
unsafe {
// Going back needs a seek, otherwise the newer frame that was already decoded
- // would be returned. Far jumps forward seek to the closest keyframe instead of
- // decoding every frame in between (frames can be requested out of order by
- // parallel renderers).
- let seek_ahead_frames = 2 * i64::from(self.custom_time_base.den.max(1));
- let needs_seek = self.last_offset.is_some_and(|last_offset| {
- offset < last_offset || offset - last_offset > seek_ahead_frames
- });
- self.last_offset = Some(offset);
- if needs_seek {
- self.seek_to_offset(offset)?;
- }
-
- let target_pts = av_rescale_q(
- offset,
- self.custom_time_base,
+ // would be returned. Far jumps forward (more than 2 seconds of the stream) seek
+ // to the closest keyframe instead of decoding every frame in between (frames can
+ // be requested out of order by parallel renderers). A new decoder is at the start.
+ let seek_ahead = av_rescale_q(
+ 2,
+ AVRational { num: 1, den: 1 },
self.video_stream_info.time_base,
);
+ let last_pts = self.last_pts.unwrap_or(0);
+ if target_pts < last_pts || target_pts - last_pts > seek_ahead {
+ self.seek_to_pts(target_pts)?;
+ }
+ self.last_pts = Some(target_pts);
if (*self.frame_buf.latest_av_frame).pts >= target_pts {
return Ok(true);
@@ -986,8 +1017,10 @@ impl FFmpegDecoder {
// The target lies after the last frame's timestamp but before the end of
// the stream (e.g. a 24 fps clip sampled at 30 fps): the last frame is
// still the one on screen, so return it instead of reporting the end.
- let shows_last_frame =
- self.has_decoded_frame && offset < self.duration_in_frames;
+ let shows_last_frame = self.has_decoded_frame
+ && self
+ .stream_duration()
+ .is_some_and(|duration| target_pts < duration);
if shows_last_frame {
self.transfer_hardware_surface_data(target_frame)?;
}
diff --git a/fframes-media/src/video_types.rs b/fframes-media/src/video_types.rs
index c2f79684..59e58ead 100644
--- a/fframes-media/src/video_types.rs
+++ b/fframes-media/src/video_types.rs
@@ -27,3 +27,49 @@ pub struct VideoMedia {
pub metadata: Option,
pub path: PathBuf,
}
+
+/// Playback speed of a video source, kept to 1/1000. Speeds that round to the same
+/// value are equal, so they share a decoder.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct PlaybackSpeed(i32);
+
+impl PlaybackSpeed {
+ /// The denominator of `thousandths`.
+ pub const SCALE: i32 = 1000;
+ pub const NORMAL: Self = Self(Self::SCALE);
+
+ /// A speed that is not positive and finite plays at normal speed.
+ pub fn new(speed: f32) -> Self {
+ if speed.is_finite() && speed > 0.0 {
+ Self(((speed * Self::SCALE as f32).round() as i32).max(1))
+ } else {
+ Self::NORMAL
+ }
+ }
+
+ /// The speed in 1/1000.
+ pub fn thousandths(self) -> i32 {
+ self.0
+ }
+}
+
+impl Default for PlaybackSpeed {
+ fn default() -> Self {
+ Self::NORMAL
+ }
+}
+
+#[cfg(test)]
+mod playback_speed_tests {
+ use super::PlaybackSpeed;
+
+ #[test]
+ fn speeds_are_kept_to_a_thousandth() {
+ assert_eq!(PlaybackSpeed::new(1.000_099), PlaybackSpeed::NORMAL);
+ assert_eq!(PlaybackSpeed::new(1.4).thousandths(), 1400);
+ assert_eq!(PlaybackSpeed::new(0.0001).thousandths(), 1);
+ for invalid in [0.0, -2.0, f32::NAN, f32::INFINITY] {
+ assert_eq!(PlaybackSpeed::new(invalid), PlaybackSpeed::NORMAL);
+ }
+ }
+}
diff --git a/fframes-media/tests/video_decoder_eof.rs b/fframes-media/tests/video_decoder_eof.rs
index b2c3817a..ecca2a30 100644
--- a/fframes-media/tests/video_decoder_eof.rs
+++ b/fframes-media/tests/video_decoder_eof.rs
@@ -5,7 +5,7 @@ use std::path::PathBuf;
use std::process::Command;
use std::sync::atomic::{AtomicUsize, Ordering};
-use fframes_media::{FFmpegDecoder, FrameConvertOptions, ResizeVideoFrame};
+use fframes_media::{FFmpegDecoder, FrameConvertOptions, PlaybackSpeed, ResizeVideoFrame};
const FPS: usize = 30;
const FRAME_COUNT: i64 = 300;
@@ -118,9 +118,8 @@ fn b_frames_are_drained_and_decoding_can_restart_after_eof() {
assert_frame(&mut decoder, FRAME_COUNT - 1);
// Looping also seeks after decoding the delayed tail.
- let offset = unsafe { decoder.adjust_offset_for_looping(FRAME_COUNT) }.unwrap();
- assert_eq!(offset, 0);
- assert_frame(&mut decoder, offset);
+ assert!(unsafe { decoder.decode_looping(FRAME_COUNT) }.unwrap());
+ assert_frame(&mut decoder, 0);
assert_frame(&mut decoder, FRAME_COUNT - 1);
}
@@ -254,8 +253,7 @@ fn last_frame_is_shown_until_the_stream_ends_at_a_higher_fps() {
for _ in 0..2 {
assert!(!unsafe { decoder.decode_up_to(60) }.unwrap());
}
- let offset = unsafe { decoder.adjust_offset_for_looping(60) }.unwrap();
- assert_eq!(offset, 0);
+ assert!(unsafe { decoder.decode_looping(60) }.unwrap());
assert_frame(&mut decoder, 0);
// A seek straight to the tail must not return a frame left over from before the seek.
@@ -326,3 +324,71 @@ fn frames_can_be_converted_at_widths_that_are_not_a_multiple_of_16() {
}
}
}
+
+/// The source frame that offset `n` shows at `num / den` speed: the first frame at or
+/// after `n * num / den` (the last frame until the stream ends), in a loop if `looping`.
+fn expected_frame(offset: i64, num: i64, den: i64, looping: bool) -> i64 {
+ let mut position = offset * num;
+ if looping {
+ position %= FRAME_COUNT * den;
+ }
+ ((position + den - 1) / den).min(FRAME_COUNT - 1)
+}
+
+fn assert_shows(decoder: &FFmpegDecoder, speed: f32, offset: i64, frame: i64) {
+ let seconds = unsafe { decoder.get_raw_frame().timestamp_seconds() };
+ assert!(
+ (seconds - frame as f32 / FPS as f32).abs() < 0.0001,
+ "{speed}x: offset {offset} shows {seconds}s, expected frame {frame}"
+ );
+}
+
+fn assert_speed(speed: f32, num: i64, den: i64) {
+ let video = VideoFixture::new(3);
+ let speed_of = |speed| {
+ unsafe { FFmpegDecoder::new(&video.path(), FPS, 1) }
+ .unwrap()
+ .with_playback_speed(PlaybackSpeed::new(speed))
+ };
+
+ // Every offset before the end of the stream has a frame, also when the stream's
+ // duration is not a whole number of offsets.
+ let mut decoder = speed_of(speed);
+ let offsets = (FRAME_COUNT * den + num - 1) / num;
+ for offset in 0..offsets {
+ assert!(
+ unsafe { decoder.decode_up_to(offset) }.unwrap(),
+ "{speed}x: offset {offset}"
+ );
+ assert_shows(
+ &decoder,
+ speed,
+ offset,
+ expected_frame(offset, num, den, false),
+ );
+ }
+ assert!(!unsafe { decoder.decode_up_to(offsets) }.unwrap());
+
+ // Loops follow the exact duration and do not drift.
+ let mut decoder = speed_of(speed);
+ for offset in 0..3 * offsets {
+ assert!(
+ unsafe { decoder.decode_looping(offset) }.unwrap(),
+ "{speed}x looping: offset {offset}"
+ );
+ assert_shows(
+ &decoder,
+ speed,
+ offset,
+ expected_frame(offset, num, den, true),
+ );
+ }
+}
+
+#[test]
+fn playback_speed_scales_the_source_time() {
+ assert_speed(2.0, 2, 1);
+ assert_speed(1.5, 3, 2);
+ assert_speed(1.4, 7, 5);
+ assert_speed(0.5, 1, 2);
+}
diff --git a/fframes/src/frame.rs b/fframes/src/frame.rs
index 72c1b713..6c7ff475 100644
--- a/fframes/src/frame.rs
+++ b/fframes/src/frame.rs
@@ -443,6 +443,7 @@ impl Frame {
/// Returns a synced video frame of the video file media.
/// If the video file is not found or there is no frame or error during decoding returns None.
/// Video frame is decoded and timebase (fps) is synced with the target fframe's video frame.
+ /// Each frame shows the source frame at `(seconds - start_from) * speed`.
///
/// **Make sure this function is not working in the editor for now. Editor will automatically fallback to
/// `{your_video_file}.{your_extension}_fallback.{jpg|png}` image file for preview if it exists
diff --git a/fframes/src/video_data.rs b/fframes/src/video_data.rs
index 2483397e..1fab172f 100644
--- a/fframes/src/video_data.rs
+++ b/fframes/src/video_data.rs
@@ -6,7 +6,7 @@ use std::sync::Arc;
use crate::error::Result;
#[cfg(not(target_arch = "wasm32"))]
use fframes_media::ImageMetadata;
-pub use fframes_media::{FrameConvertOptions, ResizeVideoFrame};
+pub use fframes_media::{FrameConvertOptions, PlaybackSpeed, ResizeVideoFrame};
#[cfg(not(target_arch = "wasm32"))]
use std::collections::HashMap;
@@ -73,7 +73,7 @@ pub trait FFramesSyncedVideoFrame<'media> {
fn stream_duration_in_seconds(&self) -> f32;
}
-#[derive(Debug, Clone, Copy, Default)]
+#[derive(Debug, Clone, Copy)]
pub struct SyncVideoFrameInput<'media> {
/// The seconds timestamp in seconds to start playing the video from
/// If not provided starts playing from the beginning of the video or scene.
@@ -82,11 +82,30 @@ pub struct SyncVideoFrameInput<'media> {
/// Also if `true` the `get_synced_video_frame` API should never return None, only in case of
/// decoding errors, so in some cases it might be okay to panic if the frame is not available.
pub looping: bool,
+ /// Playback speed of the video: `2.0` plays it twice as fast, `0.5` at half speed.
+ /// Every frame shows the source frame at `seconds since start_from * speed`, so the
+ /// motion stays even at any speed and frame rate. With `looping` a loop takes
+ /// `duration / speed` seconds. The speed is kept to 1/1000 (`PlaybackSpeed`); a value
+ /// that is not positive and finite plays at normal speed. Default `1.0`.
+ /// The speed is constant: each speed of a file has its own decoder, and the
+ /// renderer keeps the 4 used last.
+ pub speed: f32,
/// The wasm preview editor is not supporting decoding video frames in runtime
/// so this allows to provide a fallback image for the debugging
pub editor_fallback_image: Option<&'media ImageData<'media>>,
}
+impl Default for SyncVideoFrameInput<'_> {
+ fn default() -> Self {
+ Self {
+ start_from: 0.0,
+ looping: false,
+ speed: 1.0,
+ editor_fallback_image: None,
+ }
+ }
+}
+
/// This is used by the editor as a fallback for the static image
#[derive(Debug, Clone)]
pub struct WasmEditorVideoFrameFallback<'media> {
@@ -199,9 +218,18 @@ impl<'media> FFramesSyncedVideoFrame<'media> for crate::media::FFmpegFrameBuf {
pub struct VideoDecodersWorker {
buffer_size: usize,
#[cfg(not(target_arch = "wasm32"))]
- map: Arc>>,
+ map: Arc>>>,
}
+/// A decoder of a file at one playback speed.
+#[cfg(not(target_arch = "wasm32"))]
+type SpeedDecoder = (PlaybackSpeed, fframes_media::FFmpegDecoder);
+
+/// Each decoder keeps the file open and its frames in memory. A file played at more
+/// speeds closes the decoder that was used least recently.
+#[cfg(not(target_arch = "wasm32"))]
+const MAX_DECODERS_PER_VIDEO: usize = 4;
+
impl std::fmt::Debug for VideoDecodersWorker {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("WorkerLocalDecoders").finish()
@@ -237,6 +265,11 @@ impl VideoDecodersWorker {
return Ok(None);
};
+ // The editor decodes on the grid of the video's frames.
+ let speed = i64::from(PlaybackSpeed::new(options.speed).thousandths());
+ let scale = i64::from(PlaybackSpeed::SCALE);
+ let offset = (offset * speed + scale - 1) / scale;
+
let last_frame_to_display_image = metadata.duration * ctx.time_base.fps as f32;
if offset > last_frame_to_display_image as i64 && !options.looping {
return Ok(None);
@@ -290,12 +323,11 @@ impl VideoDecodersWorker {
pts: i64,
filename: &str,
) -> Result