Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -472,6 +472,7 @@ lets scenes cross-fade; `ctx.get_scene_info(&scene)` gives the resolved frame ra
let Some(video_frame) = frame.get_synced_video_frame(ctx, "clip.mp4", &fframes::SyncVideoFrameInput {
start_from: 0.,
looping: false,
speed: 1.5, // 1.5x from start_from on; 0.5 for slow motion, default 1.0
editor_fallback_image: ctx.get_image("clip_poster.jpg"),
}) else { return Svgr::empty() };
let image = video_frame.into_image(); // <image href={image.href()} .../>
Expand Down
1 change: 1 addition & 0 deletions examples/fframes-intro/src/scenes/preview.rs
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ impl Scene for PreviewScene {
start_from: 0.0,
looping: true,
editor_fallback_image: None,
..Default::default()
},
)
.map(|f| f.into_image());
Expand Down
1 change: 1 addition & 0 deletions examples/fframes-intro/src/scenes/video.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ pub fn clip_input() -> SyncVideoFrameInput<'static> {
start_from: 0.0,
looping: true,
editor_fallback_image: None,
..Default::default()
}
}

Expand Down
1 change: 1 addition & 0 deletions examples/pixel-memory/src/scenes/pixel_single_video.rs
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ impl Scene for PixelSingleVideoScene<'_> {
start_from: 0.,
looping: false,
editor_fallback_image: ctx.get_image("10.jpg"),
..Default::default()
},
) else {
return Svgr::empty();
Expand Down
2 changes: 2 additions & 0 deletions examples/teej-podcast/src/teej_podcast.rs
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@ impl Video for TeejPodcast<'_> {
start_from: 0.,
looping: false,
editor_fallback_image: ctx.get_image("left.jpg"),
..Default::default()
},
)
.map(|fr| fr.into_image());
Expand All @@ -100,6 +101,7 @@ impl Video for TeejPodcast<'_> {
start_from: 0.,
looping: false,
editor_fallback_image: ctx.get_image("right.jpg"),
..Default::default()
},
)
.map(|fr| fr.into_image());
Expand Down
135 changes: 84 additions & 51 deletions fframes-media/src/video_decoder.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
use crate::FFramesMediaError;
use crate::error::Result;
use crate::video_types::FrameConvertOptions;
use crate::video_types::{FrameConvertOptions, PlaybackSpeed};
use ffmpeg_sys_fframes::SwsFlags::{SWS_BICUBIC, SWS_BILINEAR};
use ffmpeg_sys_fframes::*;
use std::cell::UnsafeCell;
Expand Down Expand Up @@ -365,10 +365,11 @@ pub struct FFmpegDecoder {
fmt_ctx: *mut AVFormatContext,
video_stream_info: VideoStreamInfo,
pkt: *mut AVPacket,
/// Offsets are counted in this time base: `speed / target_fps` seconds per offset.
custom_time_base: AVRational,
duration_in_frames: i64,
/// The offset of the previous `decode_up_to` call, in `custom_time_base` units.
last_offset: Option<i64>,
target_fps: i32,
/// The stream position of the previous decode or seek, in the stream's time base.
last_pts: Option<i64>,
/// A null packet has been sent; receive delayed frames until decoder EOF.
draining: bool,
/// The target frame holds a frame decoded since the last seek.
Expand Down Expand Up @@ -633,6 +634,35 @@ impl FFmpegDecoder {
))
}

/// Plays the stream `speed` times faster: offset `n` shows the frame at
/// `n * speed / target_fps` seconds of the stream.
pub fn with_playback_speed(mut self, speed: PlaybackSpeed) -> Self {
self.custom_time_base = AVRational {
num: speed.thousandths(),
den: self.target_fps * PlaybackSpeed::SCALE,
};
self.last_pts = None;
self.current_loop = 0;
self
}

/// The stream position of the offset, in the stream's time base.
fn offset_pts(&self, offset: i64) -> i64 {
unsafe {
av_rescale_q(
offset,
self.custom_time_base,
self.video_stream_info.time_base,
)
}
}

/// The exact stream duration, in the stream's time base. `None` if it is unknown.
fn stream_duration(&self) -> Option<i64> {
let duration = self.video_stream_info.duration;
(duration > 0).then_some(duration)
}

/// Creates a new decoder for the video file at the specified path
/// # Safety
/// Generally safe but uses libav functions
Expand Down Expand Up @@ -706,12 +736,6 @@ impl FFmpegDecoder {
den: target_fps as i32,
};

let duration_in_frames = av_rescale_q(
video_stream_info.duration,
video_stream_info.time_base,
custom_time_base,
);

Ok(FFmpegDecoder {
pkt,
fmt_ctx,
Expand All @@ -724,9 +748,9 @@ impl FFmpegDecoder {
)?),
video_stream_info,
custom_time_base,
duration_in_frames,
target_fps: target_fps as i32,
current_loop: 0,
last_offset: None,
last_pts: None,
draining: false,
has_decoded_frame: false,
})
Expand Down Expand Up @@ -833,13 +857,11 @@ impl FFmpegDecoder {
/// # Safety
/// Generally safe but uses libav functions
pub unsafe fn seek_to_offset(&mut self, offset: i64) -> Result<()> {
unsafe {
let timestamp = av_rescale_q(
offset,
self.custom_time_base,
self.video_stream_info.time_base,
);
unsafe { self.seek_to_pts(self.offset_pts(offset)) }
}

unsafe fn seek_to_pts(&mut self, timestamp: i64) -> Result<()> {
unsafe {
let ret = av_seek_frame(
self.fmt_ctx,
self.video_stream_info.stream_index,
Expand All @@ -858,33 +880,42 @@ impl FFmpegDecoder {
avcodec_flush_buffers(self.video_stream_info.codec_ctx);
self.draining = false;
self.has_decoded_frame = false;
self.last_pts = Some(timestamp);
av_packet_unref(self.pkt);

Ok(())
}
}

/// Moves offset to the start of the video if need to loop
/// Decodes the frame at the offset of the video played in a loop. The position in
/// the loop comes from the exact stream duration, so loops at a fractional speed
/// do not drift and do not end early.
/// # Safety
/// Generally safe but uses libav functions
pub unsafe fn adjust_offset_for_looping(&mut self, offset: i64) -> Result<i64> {
// An unknown or empty duration can't be looped over (and would divide by zero below).
if self.duration_in_frames <= 0 || offset < self.duration_in_frames {
return Ok(offset);
}
pub unsafe fn decode_looping(&mut self, offset: i64) -> Result<bool> {
unsafe {
let pts = self.offset_pts(offset);
// An unknown duration can't be looped over (and would divide by zero below).
let Some(duration) = self.stream_duration() else {
return self.decode_up_to_pts(pts);
};

let new_loop_index = offset / self.duration_in_frames;
let new_offset = offset % self.duration_in_frames;
if new_loop_index == self.current_loop {
return Ok(new_offset);
}
let loop_index = pts.div_euclid(duration);
let pts = pts.rem_euclid(duration);
if loop_index != self.current_loop {
self.current_loop = loop_index;
self.seek_to_pts(pts)?;
}

self.current_loop = new_loop_index;
unsafe {
self.seek_to_offset(new_offset)?;
}
if self.decode_up_to_pts(pts)? {
return Ok(true);
}

Ok(new_offset)
// The frames end before the stream's duration: start the next loop.
self.current_loop += 1;
self.seek_to_pts(0)?;
self.decode_up_to_pts(0)
}
}

/// If we are actually using hardware accelerated frames it is required to correctly
Expand Down Expand Up @@ -937,25 +968,25 @@ impl FFmpegDecoder {
/// # Safety
/// Generally safe but uses libav functions
pub unsafe fn decode_up_to(&mut self, offset: i64) -> Result<bool> {
unsafe { self.decode_up_to_pts(self.offset_pts(offset)) }
}

unsafe fn decode_up_to_pts(&mut self, target_pts: i64) -> Result<bool> {
unsafe {
// Going back needs a seek, otherwise the newer frame that was already decoded
// would be returned. Far jumps forward seek to the closest keyframe instead of
// decoding every frame in between (frames can be requested out of order by
// parallel renderers).
let seek_ahead_frames = 2 * i64::from(self.custom_time_base.den.max(1));
let needs_seek = self.last_offset.is_some_and(|last_offset| {
offset < last_offset || offset - last_offset > seek_ahead_frames
});
self.last_offset = Some(offset);
if needs_seek {
self.seek_to_offset(offset)?;
}

let target_pts = av_rescale_q(
offset,
self.custom_time_base,
// would be returned. Far jumps forward (more than 2 seconds of the stream) seek
// to the closest keyframe instead of decoding every frame in between (frames can
// be requested out of order by parallel renderers). A new decoder is at the start.
let seek_ahead = av_rescale_q(
2,
AVRational { num: 1, den: 1 },
self.video_stream_info.time_base,
);
let last_pts = self.last_pts.unwrap_or(0);
if target_pts < last_pts || target_pts - last_pts > seek_ahead {
self.seek_to_pts(target_pts)?;
}
self.last_pts = Some(target_pts);

if (*self.frame_buf.latest_av_frame).pts >= target_pts {
return Ok(true);
Expand Down Expand Up @@ -986,8 +1017,10 @@ impl FFmpegDecoder {
// The target lies after the last frame's timestamp but before the end of
// the stream (e.g. a 24 fps clip sampled at 30 fps): the last frame is
// still the one on screen, so return it instead of reporting the end.
let shows_last_frame =
self.has_decoded_frame && offset < self.duration_in_frames;
let shows_last_frame = self.has_decoded_frame
&& self
.stream_duration()
.is_some_and(|duration| target_pts < duration);
if shows_last_frame {
self.transfer_hardware_surface_data(target_frame)?;
}
Expand Down
46 changes: 46 additions & 0 deletions fframes-media/src/video_types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -27,3 +27,49 @@ pub struct VideoMedia {
pub metadata: Option<GeneralVideoFileMetadata>,
pub path: PathBuf,
}

/// Playback speed of a video source, kept to 1/1000. Speeds that round to the same
/// value are equal, so they share a decoder.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct PlaybackSpeed(i32);

impl PlaybackSpeed {
/// The denominator of `thousandths`.
pub const SCALE: i32 = 1000;
pub const NORMAL: Self = Self(Self::SCALE);

/// A speed that is not positive and finite plays at normal speed.
pub fn new(speed: f32) -> Self {
if speed.is_finite() && speed > 0.0 {
Self(((speed * Self::SCALE as f32).round() as i32).max(1))
} else {
Self::NORMAL
}
}

/// The speed in 1/1000.
pub fn thousandths(self) -> i32 {
self.0
}
}

impl Default for PlaybackSpeed {
fn default() -> Self {
Self::NORMAL
}
}

#[cfg(test)]
mod playback_speed_tests {
use super::PlaybackSpeed;

#[test]
fn speeds_are_kept_to_a_thousandth() {
assert_eq!(PlaybackSpeed::new(1.000_099), PlaybackSpeed::NORMAL);
assert_eq!(PlaybackSpeed::new(1.4).thousandths(), 1400);
assert_eq!(PlaybackSpeed::new(0.0001).thousandths(), 1);
for invalid in [0.0, -2.0, f32::NAN, f32::INFINITY] {
assert_eq!(PlaybackSpeed::new(invalid), PlaybackSpeed::NORMAL);
}
}
}
Loading
Loading