diff --git a/Cargo.lock b/Cargo.lock index 98da069a..e9a46c06 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1294,6 +1294,7 @@ dependencies = [ "fframes_skia_renderer", "hello-world-example", "raw-window-metal", + "rtrb", "softbuffer", "winit", ] @@ -3419,6 +3420,12 @@ version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c20b6793b5c2fa6553b250154b78d6d0db37e72700ae35fad9387a46f487c97" +[[package]] +name = "rtrb" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5974e397833d890f2f74963408e0d4a6cd8eda2f6a445ae06750291b5d7a997e" + [[package]] name = "rust-fuzzy-search" version = "0.1.1" diff --git a/README.md b/README.md index 1ede72cf..54f25130 100644 --- a/README.md +++ b/README.md @@ -339,6 +339,19 @@ cargo fframes new my-video cd my-video && cargo run --release -- preview ``` +The preview timeline sits below landscape and square videos and to the left of portrait videos. +Drag the filmstrip to scrub, or click a scene marker to pause on its first frame. Use `[` and `]` +to jump between scenes and `,` and `.` to step one frame. Press `Space` to play or pause, or `T` +to hide the timeline and expand the video. + +On macOS the studio tries VideoToolbox for supported source videos and falls back to software +for unsupported formats. `preview --software-decode` uses the CPU for decoding while keeping +the selected drawing backend. Playback and filmstrip workers release idle source decoders. +The GPU preview draws each scene through Skia directly into the window drawable or swapchain. +The default buffer holds up to four prepared scene trees, and only the frame under the pointer +is prepared while you drag the timeline. Decoders reuse the pixel buffers of frames that no +scene tree holds any more. The CPU backend draws directly into the window buffer. + `cargo fframes new` takes `--template single-scene|multi-scene`, `--format landscape|portrait|square|uhd`, `--fps`, `--title` and `--backend`. On macOS and Linux (arm64, x86_64) the first build downloads prebuilt Skia and ffmpeg libraries and takes under a minute on a fast machine; other targets and feature combinations diff --git a/examples/arm/src/main.rs b/examples/arm/src/main.rs index 30fb08d9..49ac5fa1 100644 --- a/examples/arm/src/main.rs +++ b/examples/arm/src/main.rs @@ -59,12 +59,12 @@ fn main() -> ExitCode { ..Default::default() }; - let cpu = args.app.cpu; + let needs_export_gpu = !args.app.cpu && !matches!(args.command, Some(cli::Command::Preview(_))); let command = cli::new(&video, options) .args(args) .preview(fframes_native_player::cli_preview) .default_output("out.mp4"); - if cpu { + if !needs_export_gpu { return command.run(); } diff --git a/fframes-media/Cargo.toml b/fframes-media/Cargo.toml index 4f4b69cf..b9705edf 100644 --- a/fframes-media/Cargo.toml +++ b/fframes-media/Cargo.toml @@ -62,7 +62,9 @@ mp3lame = ["ffmpeg-sys-fframes/build-lib-mp3lame"] opus = ["ffmpeg-sys-fframes/build-lib-opus"] vpx = ["ffmpeg-sys-fframes/build-lib-vpx"] # enable various hardware accelerations -videotoolbox = ["ffmpeg-sys-fframes/build-videotoolbox"] +# The player opts in on macOS without changing decoding policy for exports. +preview-videotoolbox = ["ffmpeg-sys-fframes/build-videotoolbox"] +videotoolbox = ["preview-videotoolbox"] audiotoolbox = ["ffmpeg-sys-fframes/build-audiotoolbox"] vaapi = ["ffmpeg-sys-fframes/build-vaapi"] nvidia = ["ffmpeg-sys-fframes/build-nvidia"] diff --git a/fframes-media/src/video_decoder.rs b/fframes-media/src/video_decoder.rs index fa7525b6..a6d5aadd 100644 --- a/fframes-media/src/video_decoder.rs +++ b/fframes-media/src/video_decoder.rs @@ -3,13 +3,13 @@ use crate::error::Result; use crate::video_types::{FrameConvertOptions, PlaybackSpeed}; use ffmpeg_sys_fframes::SwsFlags::{SWS_BICUBIC, SWS_BILINEAR}; use ffmpeg_sys_fframes::*; -use std::cell::UnsafeCell; +use std::borrow::Cow; use std::collections::VecDeque; use std::ffi::{CStr, CString, c_void}; use std::mem::size_of; use std::path::Path; use std::ptr; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use usvgr::PreloadedImageData; /// The video stream's duration in its own time base. Matroska and `WebM` leave the stream @@ -92,12 +92,10 @@ pub fn decode_video_resource(s: &str) -> Option<(&str, i64)> { struct VideoStreamInfo { hw_pix_fmt: Option, stream_index: i32, - codec_ctx: *mut AVCodecContext, width: i32, height: i32, display_width: i32, display_height: i32, - pixel_format: AVPixelFormat, color_space: AVColorSpace, color_range: AVColorRange, time_base: AVRational, @@ -112,12 +110,16 @@ struct SwsScaler { sws_ctx: *mut SwsContext, options: Option, source_format: Option, + source_has_alpha: bool, source_color: Option<(AVColorSpace, AVColorRange)>, frame_data_len: usize, video_stream_info: VideoStreamInfo, - linesize: [i32; 7], + linesize: [i32; 4], } +// The scaler exclusively owns its SwsContext. All conversions use it under one mutex. +unsafe impl Send for SwsScaler {} + impl Drop for SwsScaler { fn drop(&mut self) { unsafe { @@ -240,8 +242,8 @@ impl SwsScaler { } } - fn calculate_linesize(width: i32) -> [i32; 7] { - let mut linesize = [0; 7]; + fn calculate_linesize(width: i32) -> [i32; 4] { + let mut linesize = [0; 4]; linesize[0] = width * PIX_FMT_SIZE as i32; linesize } @@ -253,7 +255,7 @@ impl SwsScaler { ); let frame_data_len = width as usize * height as usize * PIX_FMT_SIZE; - let linesize: [i32; 7] = Self::calculate_linesize(width); + let linesize: [i32; 4] = Self::calculate_linesize(width); SwsScaler { linesize, frame_data_len, @@ -262,6 +264,7 @@ impl SwsScaler { sws_ctx: std::ptr::null_mut(), options: None, source_format: None, + source_has_alpha: false, source_color: None, video_stream_info, } @@ -290,6 +293,9 @@ impl SwsScaler { sws_freeContext(self.sws_ctx); self.sws_ctx = ctx; self.source_format = Some(pix_fmt); + let desc = unsafe { av_pix_fmt_desc_get(pix_fmt) }; + self.source_has_alpha = + !desc.is_null() && unsafe { (*desc).flags } & AV_PIX_FMT_FLAG_ALPHA as u64 != 0; self.source_color = Some(source_color); self.width = new_width; self.height = new_height; @@ -326,44 +332,64 @@ impl SwsScaler { self.reinit_sws_context(source_pix_fmt, source_color, options)?; } - // A reused buffer may still have the length of another output size. This - // reallocates only when the output grows and does nothing when the size is - // unchanged. `sws_scale` can write past the image, see `SWS_DST_PADDING`. + // Reserve the SIMD tail without zeroing a full frame before swscale fills it. let capacity = self.frame_data_len + SWS_DST_PADDING; rgba_dst.reserve(capacity.saturating_sub(rgba_dst.len())); - rgba_dst.resize(self.frame_data_len, 0); + // sws_scale reads four plane pointers even for packed RGBA. + let mut planes = [ptr::null_mut(); 4]; + planes[0] = rgba_dst.as_mut_ptr(); let ret = sws_scale( self.sws_ctx, (*source_frame).data.as_ptr().cast::<*const u8>(), (*source_frame).linesize.as_ptr(), 0, self.video_stream_info.height, - &rgba_dst.as_mut_ptr(), + planes.as_ptr(), self.linesize.as_ptr(), ); - if ret < 0 { + if ret != self.height { return Err(FFramesMediaError::LibAVAudioDecodingError(( ret, - "Error while converting/scaling frame".to_string(), + format!( + "Expected {} output rows from sws_scale, got {ret}", + self.height + ), ))); } + // SAFETY: sws_scale initialized every RGBA row with width * 4 stride. + // The tail padding lies inside the allocation but outside the image. + unsafe { rgba_dst.set_len(self.frame_data_len) }; Ok(()) } } +/// Selects the decode path without changing encoder or presentation settings. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum DecodeMode { + /// Hardware accelerators explicitly enabled by this project. + #[default] + Configured, + /// Also try the platform accelerator included with the native studio. + Preview, + /// Use the codec's software implementation. + Software, +} + #[derive(Debug)] pub struct FFmpegDecoder { pub current_loop: i64, hw_frame: *mut AVFrame, + map_videotoolbox: bool, /// Frames are received here and moved into the target frame, so the newest decoded frame /// survives `avcodec_receive_frame` returning EOF (which unrefs the frame it is given). recv_frame: *mut AVFrame, frame_buf: Arc, fmt_ctx: *mut AVFormatContext, video_stream_info: VideoStreamInfo, + codec_ctx: *mut AVCodecContext, pkt: *mut AVPacket, /// Offsets are counted in this time base: `speed / target_fps` seconds per offset. custom_time_base: AVRational, @@ -374,81 +400,88 @@ pub struct FFmpegDecoder { draining: bool, /// The target frame holds a frame decoded since the last seek. has_decoded_frame: bool, + /// Frames received since the last seek, oldest first, at most `recent_capacity`. + /// Workers that request frames out of order take one of these instead of seeking back. + recent: VecDeque<*mut AVFrame>, + recent_capacity: usize, + seeks: u64, } +/// Frames the preview keeps for out-of-order requests, see `FFmpegDecoder::recent`. +const RECENT_FRAMES: usize = 8; + unsafe impl Send for FFmpegDecoder {} unsafe impl Sync for FFmpegDecoder {} +#[derive(Debug)] struct ScaledFrameImage { pts: i64, - data: Vec, + options: Option, + image: Arc, +} + +/// Evicted images kept for buffer reuse, see `ConvertedFrames::retired`. +const MAX_RETIRED_IMAGES: usize = 8; + +#[derive(Debug)] +struct ConvertedFrames { + scaler: SwsScaler, + images: VecDeque, + /// Images evicted from the ring that a scene tree or a renderer still holds. Their + /// pixel buffers are reused once the last holder drops them, so playback does not + /// allocate a frame-sized buffer per frame. + retired: Vec>, +} + +impl ConvertedFrames { + /// A pixel buffer no other holder can see, from the ring or the retired images. + fn free_pixels(&mut self) -> Vec { + if self.images.len() == self.images.capacity() + && let Some(evicted) = self.images.pop_front() + { + if self.retired.len() == MAX_RETIRED_IMAGES { + self.retired.remove(0); + } + self.retired.push(evicted.image); + } + let Some(free) = self + .retired + .iter() + .position(|image| Arc::strong_count(image) == 1) + else { + return Vec::new(); + }; + Arc::try_unwrap(self.retired.swap_remove(free)) + .map(|image| image.data.into_owned()) + .unwrap_or_default() + } } #[derive(Debug)] pub struct FFmpegFrameBuf { - resource_name: String, + resource_name: Arc, latest_av_frame: *mut AVFrame, video_stream_info: VideoStreamInfo, - // This is not a safe operation but it gives a dramatic performance improvement - // for the updating the underlying datavec. So we are relying on the constraint that - // the decoder can operate (thus write the datavec) only within the frame, but - // the frame can be sent and read outside the dnecoder - data_buf: Arc>>, - sws_scaler: UnsafeCell, + converted: Arc>, } impl Drop for FFmpegFrameBuf { fn drop(&mut self) { - unsafe { - if !self.latest_av_frame.is_null() { - av_frame_unref(self.latest_av_frame); - av_frame_free(&raw mut self.latest_av_frame); - } - - if let Some(queue) = self.data_buf.get().as_mut() { - queue.clear(); - } - } + unsafe { av_frame_free(&raw mut self.latest_av_frame) }; } } -/// Creates svgr preloaded image data with correctly blended color. -/// -/// `rgba_data` borrows a slot of the decoder's ring buffer (with a fabricated -/// `'static` lifetime) that is recycled by later decodes, so the returned -/// image always owns a copy of the pixels. -fn create_preloaded_image( - source_fmt: AVPixelFormat, - resource_name: String, - width: u32, - height: u32, - rgba_data: &'static [u8], -) -> Arc { - let has_alpha_channel = unsafe { - let desc = av_pix_fmt_desc_get(source_fmt); - - !desc.is_null() && (*desc).flags & AV_PIX_FMT_FLAG_ALPHA as u64 != 0 - }; - - if has_alpha_channel { - // Pre-multiplies alpha and returns an owned copy. - Arc::new(PreloadedImageData::new( - resource_name, - width, - height, - rgba_data, - )) - } else { - // Opaque pixels are already final; only the copy is needed. - Arc::new(PreloadedImageData { - id: resource_name, - data: std::borrow::Cow::Owned(rgba_data.to_vec()), - width, - height, - }) +fn premultiply_alpha(pixels: &mut [u8]) { + for pixel in pixels.as_chunks_mut::<4>().0 { + let alpha = u16::from(pixel[3]); + for channel in &mut pixel[..3] { + *channel = ((u16::from(*channel) * alpha + 127) / 255) as u8; + } } } +// The decoder owns writes to the AVFrame. Its unsafe readers require that it +// stays unchanged while they run. The conversion ring and scaler share a lock. unsafe impl Send for FFmpegFrameBuf {} unsafe impl Sync for FFmpegFrameBuf {} @@ -468,44 +501,16 @@ impl FFmpegFrameBuf { Ok(FFmpegFrameBuf { video_stream_info, latest_av_frame: av_frame, - resource_name, - #[allow(clippy::arc_with_non_send_sync)] - data_buf: Arc::new(UnsafeCell::new(VecDeque::with_capacity(buf_size))), - sws_scaler: UnsafeCell::new(sws_ctx), + resource_name: resource_name.into(), + converted: Arc::new(Mutex::new(ConvertedFrames { + scaler: sws_ctx, + images: VecDeque::with_capacity(buf_size.max(1)), + retired: Vec::with_capacity(MAX_RETIRED_IMAGES), + })), }) } } - unsafe fn write_new_frame(&self) -> Option<&'static mut ScaledFrameImage> { - unsafe { - let queue = self.data_buf.get().as_mut()?; - let latest_pts = (*self.latest_av_frame).pts; - - // fast path for the cpu renderer which will always have capacity 1 - if queue.capacity() == 1 && queue.len() == 1 { - // the pts identifies the image (and its renderer cache entries), it has to - // follow the pixels - let image = queue.get_mut(0)?; - image.pts = latest_pts; - return Some(image); - } - - // `SwsScaler::convert` sizes the buffer once it knows the requested output size - if queue.len() == queue.capacity() { - let mut last_buffer = queue.pop_front()?; - last_buffer.pts = latest_pts; - queue.push_back(last_buffer); - } else { - queue.push_back(ScaledFrameImage { - pts: latest_pts, - data: Vec::new(), - }); - } - - queue.get_mut(queue.len() - 1) - } - } - pub fn get_stream_width(&self) -> u32 { self.video_stream_info.display_width as u32 } @@ -515,67 +520,48 @@ impl FFmpegFrameBuf { } pub fn get_width(&self) -> u32 { - unsafe { self.sws_scaler.get().as_ref() } - .expect("Critical mememroy error: SWS scale is not allocated") - .width as u32 + self.converted.lock().unwrap().scaler.width as u32 } pub fn get_height(&self) -> u32 { - unsafe { self.sws_scaler.get().as_ref() } - .expect("Critical mememroy error: SWS scale is not allocated") - .height as u32 + self.converted.lock().unwrap().scaler.height as u32 } - /// Returns fframe image constructed from the underlying ffmpeg's frame + /// Converts the current frame to an immutable, premultiplied RGBA image. /// # Safety - /// This is a libav based function which involves C ffi cals - /// In addition it is manually transmutes the pointer owned by the decoder to the fframes - /// images so it should not be used outside of the rendering worker. + /// The decoder must not replace the underlying `AVFrame` during conversion. pub unsafe fn convert_last_decoded_frame_into_svg_image( &self, options: Option<&FrameConvertOptions>, ) -> Result> { - unsafe { - let scaler = self.sws_scaler.get().as_mut().unwrap(); - // fast path if we have the image already decoded and converted last time with the - // same options (the scaler keeps them), e.g. the same video frame is requested at - // the same size from multiple scenes - match self.data_buf.get().as_ref().and_then(|q| q.back()) { - Some(image) - if image.pts == (*self.latest_av_frame).pts - && scaler.options == options.copied() => - { - return Ok(create_preloaded_image( - scaler.video_stream_info.pixel_format, - encode_video_resource(&self.resource_name, image.pts), - self.get_width(), - self.get_height(), - &image.data, - )); - } - _ => {} - } - - let image = self.write_new_frame().unwrap(); - if let Err(err) = - scaler.convert(options.copied(), self.latest_av_frame, &mut image.data) - { - // the buffer already has this frame's pts but not its pixels (a reused buffer - // still holds an older frame), so drop it instead of returning it for this pts - if let Some(queue) = self.data_buf.get().as_mut() { - queue.pop_back(); - } - return Err(err); - } + let pts = unsafe { (*self.latest_av_frame).pts }; + let options = options.copied(); + let mut converted = self.converted.lock().unwrap(); + if let Some(last) = converted.images.back() + && last.pts == pts + && last.options == options + { + return Ok(Arc::clone(&last.image)); + } - Ok(create_preloaded_image( - scaler.video_stream_info.pixel_format, - encode_video_resource(&self.resource_name, image.pts), - scaler.width as u32, - scaler.height as u32, - &image.data, - )) + let mut pixels = converted.free_pixels(); + let scaler = &mut converted.scaler; + unsafe { scaler.convert(options, self.latest_av_frame, &mut pixels)? }; + if scaler.source_has_alpha { + premultiply_alpha(&mut pixels); } + let image = Arc::new(PreloadedImageData { + id: encode_video_resource(&self.resource_name, pts), + width: scaler.width as u32, + height: scaler.height as u32, + data: Cow::Owned(pixels), + }); + converted.images.push_back(ScaledFrameImage { + pts, + options, + image: Arc::clone(&image), + }); + Ok(image) } /// Returns the timestamp of the frame in the native frame timebase @@ -617,21 +603,38 @@ impl FFmpegFrameBuf { } impl FFmpegDecoder { + /// Live view of this decoder's cursor. Use [`Self::retained_frame`] for a frame + /// that will outlive the next decode or seek. pub fn get_raw_frame(&self) -> Arc { Arc::clone(&self.frame_buf) } + /// Retains the decoded planes so this frame can be read after the decoder advances + /// or seeks. `FFmpeg` clones the frame header and reference-counts the pixel buffers. + pub fn retained_frame(&self) -> Result> { + let frame = &self.frame_buf; + let latest_av_frame = unsafe { av_frame_clone(frame.latest_av_frame) }; + if latest_av_frame.is_null() { + return Err(FFramesMediaError::LibAVAllocationError("frame")); + } + Ok(Arc::new(FFmpegFrameBuf { + latest_av_frame, + resource_name: Arc::clone(&frame.resource_name), + video_stream_info: self.video_stream_info, + converted: Arc::clone(&frame.converted), + })) + } + pub fn get_decoded_image_in_buf(&self, pts: i64) -> Option> { - let buf = unsafe { self.frame_buf.data_buf.get().as_ref() }?; - let image = buf.iter().find(|i| i.pts == pts)?; - - Some(create_preloaded_image( - self.frame_buf.video_stream_info.pixel_format, - encode_video_resource(&self.frame_buf.resource_name, image.pts), - self.frame_buf.get_width(), - self.frame_buf.get_height(), - &image.data, - )) + self.frame_buf + .converted + .lock() + .unwrap() + .images + .iter() + .rev() + .find(|slot| slot.pts == pts) + .map(|slot| Arc::clone(&slot.image)) } /// Plays the stream `speed` times faster: offset `n` shows the frame at @@ -667,6 +670,18 @@ impl FFmpegDecoder { /// # Safety /// Generally safe but uses libav functions pub unsafe fn new(path: &Path, target_fps: usize, buffer_size: usize) -> Result { + unsafe { Self::new_with_mode(path, target_fps, buffer_size, DecodeMode::Configured) } + } + + /// Opens a decoder using the requested hardware policy. + /// # Safety + /// Has the same libav initialization requirements as [`Self::new`]. + pub unsafe fn new_with_mode( + path: &Path, + target_fps: usize, + buffer_size: usize, + mode: DecodeMode, + ) -> Result { unsafe { let filename = path .file_name() @@ -702,50 +717,31 @@ impl FFmpegDecoder { ))); } - let video_stream_info = Self::open_codec_context(fmt_ctx)?; - let pkt = av_packet_alloc(); - if pkt.is_null() { - avformat_close_input(&raw mut fmt_ctx); - return Err(FFramesMediaError::LibAVAllocationError("packet")); - } - - let hw_frame = if video_stream_info.hw_pix_fmt.is_some() { - let hw_frame = av_frame_alloc(); - if hw_frame.is_null() { - avformat_close_input(&raw mut fmt_ctx); - return Err(FFramesMediaError::LibAVAllocationError("av_frame")); - } - - (*hw_frame).width = video_stream_info.width; - (*hw_frame).height = video_stream_info.height; - (*hw_frame).format = video_stream_info.pixel_format as i32; - - hw_frame - } else { - ptr::null_mut() - }; - - let recv_frame = av_frame_alloc(); - if recv_frame.is_null() { + let (mut codec_ctx, video_stream_info) = Self::open_codec_context(fmt_ctx, mode) + .inspect_err(|_| avformat_close_input(&raw mut fmt_ctx))?; + let frame_buf = FFmpegFrameBuf::new( + filename.to_string_lossy().into_owned(), + video_stream_info, + buffer_size, + ) + .inspect_err(|_| { + avcodec_free_context(&raw mut codec_ctx); avformat_close_input(&raw mut fmt_ctx); - return Err(FFramesMediaError::LibAVAllocationError("av_frame")); - } + })?; let custom_time_base = AVRational { num: 1, den: target_fps as i32, }; - Ok(FFmpegDecoder { - pkt, + let mut decoder = Self { + codec_ctx, fmt_ctx, - hw_frame, - recv_frame, - frame_buf: Arc::new(FFmpegFrameBuf::new( - filename.to_string_lossy().to_string(), - video_stream_info, - buffer_size, - )?), + pkt: ptr::null_mut(), + hw_frame: ptr::null_mut(), + map_videotoolbox: cfg!(target_os = "macos") && mode == DecodeMode::Preview, + recv_frame: ptr::null_mut(), + frame_buf: Arc::new(frame_buf), video_stream_info, custom_time_base, target_fps: target_fps as i32, @@ -753,11 +749,38 @@ impl FFmpegDecoder { last_pts: None, draining: false, has_decoded_frame: false, - }) + recent: VecDeque::new(), + // Export workers own their decoders and read in order. + recent_capacity: if mode == DecodeMode::Configured { + 0 + } else { + RECENT_FRAMES + }, + seeks: 0, + }; + + decoder.pkt = av_packet_alloc(); + if decoder.pkt.is_null() { + return Err(FFramesMediaError::LibAVAllocationError("packet")); + } + decoder.recv_frame = av_frame_alloc(); + if decoder.recv_frame.is_null() { + return Err(FFramesMediaError::LibAVAllocationError("av_frame")); + } + if video_stream_info.hw_pix_fmt.is_some() { + decoder.hw_frame = av_frame_alloc(); + if decoder.hw_frame.is_null() { + return Err(FFramesMediaError::LibAVAllocationError("av_frame")); + } + } + Ok(decoder) } } - unsafe fn open_codec_context(fmt_ctx: *mut AVFormatContext) -> Result { + unsafe fn open_codec_context( + fmt_ctx: *mut AVFormatContext, + mode: DecodeMode, + ) -> Result<(*mut AVCodecContext, VideoStreamInfo)> { unsafe { let ret = av_find_best_stream( fmt_ctx, @@ -778,35 +801,37 @@ impl FFmpegDecoder { let stream = *(*fmt_ctx).streams.offset(video_stream_idx as isize); let dec = avcodec_find_decoder((*(*stream).codecpar).codec_id); - let mut hw_device_ctx: *mut AVBufferRef = ptr::null_mut(); - let hw_pix_fmt = find_hw_accelleleration_for_codec(&mut hw_device_ctx, dec); - if dec.is_null() { return Err(FFramesMediaError::LibAVAudioDecodingError(( - ret, + AVERROR_DECODER_NOT_FOUND, "Could not find decoder".to_string(), ))); } - let video_dec_ctx = avcodec_alloc_context3(dec); + let mut video_dec_ctx = avcodec_alloc_context3(dec); if video_dec_ctx.is_null() { - return Err(FFramesMediaError::LibAVAllocationError("avcocdec_context")); - } - - if let Some(hw_pix_fmt) = hw_pix_fmt { - (*video_dec_ctx).opaque = hw_pix_fmt as i32 as *mut c_void; - (*video_dec_ctx).get_format = Some(get_hw_format); - (*video_dec_ctx).hw_device_ctx = av_buffer_ref(hw_device_ctx); + return Err(FFramesMediaError::LibAVAllocationError("avcodec_context")); } - if avcodec_parameters_to_context(video_dec_ctx, (*stream).codecpar) < 0 { + let ret = avcodec_parameters_to_context(video_dec_ctx, (*stream).codecpar); + if ret < 0 { + avcodec_free_context(&raw mut video_dec_ctx); return Err(FFramesMediaError::LibAVAudioDecodingError(( ret, "Failed to copy video codec parameters to decoder context".to_string(), ))); } - if avcodec_open2(video_dec_ctx, dec, ptr::null_mut()) < 0 { + let hw_pix_fmt = + find_hw_acceleration_for_codec(&mut (*video_dec_ctx).hw_device_ctx, dec, mode); + if let Some(hw_pix_fmt) = hw_pix_fmt { + (*video_dec_ctx).opaque = hw_pix_fmt as i32 as *mut c_void; + (*video_dec_ctx).get_format = Some(get_hw_format); + } + + let ret = avcodec_open2(video_dec_ctx, dec, ptr::null_mut()); + if ret < 0 { + avcodec_free_context(&raw mut video_dec_ctx); return Err(FFramesMediaError::LibAVAudioDecodingError(( ret, "Failed to open video codec".to_string(), @@ -835,21 +860,22 @@ impl FFmpegDecoder { } None => (width, height), }; - Ok(VideoStreamInfo { - hw_pix_fmt, - stream_index: video_stream_idx, - codec_ctx: video_dec_ctx, - width, - height, - display_width, - display_height, - pixel_format: (*video_dec_ctx).pix_fmt, - color_space: (*video_dec_ctx).colorspace, - color_range: (*video_dec_ctx).color_range, - time_base: (*stream).time_base, - duration: stream_duration(fmt_ctx, stream), - frame_rate: (*stream).r_frame_rate, - }) + Ok(( + video_dec_ctx, + VideoStreamInfo { + hw_pix_fmt, + stream_index: video_stream_idx, + width, + height, + display_width, + display_height, + color_space: (*video_dec_ctx).colorspace, + color_range: (*video_dec_ctx).color_range, + time_base: (*stream).time_base, + duration: stream_duration(fmt_ctx, stream), + frame_rate: (*stream).r_frame_rate, + }, + )) } } @@ -877,16 +903,62 @@ impl FFmpegDecoder { } (*self.frame_buf.latest_av_frame).pts = -1; - avcodec_flush_buffers(self.video_stream_info.codec_ctx); + avcodec_flush_buffers(self.codec_ctx); + self.last_pts = Some(timestamp); self.draining = false; self.has_decoded_frame = false; - self.last_pts = Some(timestamp); + self.seeks += 1; + self.forget_recent(); av_packet_unref(self.pkt); Ok(()) } } + /// How many times the decoder repositioned the stream. + pub fn seeks(&self) -> u64 { + self.seeks + } + + /// Keeps a reference to the frame that was just received. + unsafe fn remember(&mut self, frame: *mut AVFrame) { + if self.recent_capacity == 0 { + return; + } + let clone = unsafe { av_frame_clone(frame) }; + if clone.is_null() { + return; + } + if self.recent.len() == self.recent_capacity + && let Some(mut oldest) = self.recent.pop_front() + { + unsafe { av_frame_free(&raw mut oldest) }; + } + self.recent.push_back(clone); + } + + fn forget_recent(&mut self) { + for mut frame in self.recent.drain(..) { + unsafe { av_frame_free(&raw mut frame) }; + } + } + + /// The frame the stream shows at `target_pts`, if the recent frames cover it. They + /// are consecutive, so the first one at or after the target is the right one. + fn recent_frame(&self, target_pts: i64) -> Option<*mut AVFrame> { + let covered = self + .recent + .front() + .is_some_and(|&oldest| unsafe { (*oldest).pts } <= target_pts); + if !covered { + return None; + } + self.recent + .iter() + .copied() + .find(|&frame| unsafe { (*frame).pts } >= target_pts) + } + /// Decodes the frame at the offset of the video played in a loop. The position in /// the loop comes from the exact stream duration, so loops at a fractional speed /// do not drift and do not end early. @@ -920,9 +992,29 @@ impl FFmpegDecoder { /// If we are actually using hardware accelerated frames it is required to correctly /// move the data from the hardware surface to the software frame - unsafe fn transfer_hardware_surface_data(&self, target_frame: *mut AVFrame) -> Result<()> { + unsafe fn transfer_hardware_surface_data(&mut self, target_frame: *mut AVFrame) -> Result<()> { match self.video_stream_info.hw_pix_fmt { Some(hw_pix_fmt) if hw_pix_fmt as i32 == (*target_frame).format => { + // A retained frame can still own the last transfer's software planes. + let output = self.frame_buf.latest_av_frame; + if self.map_videotoolbox && hw_pix_fmt == AVPixelFormat::AV_PIX_FMT_VIDEOTOOLBOX { + // FFmpeg holds a read lock on the CVPixelBuffer until the last + // mapped frame reference drops. swscale can read its planes directly. + av_frame_unref(output); + if av_hwframe_map(output, target_frame, AV_HWFRAME_MAP_READ as i32) == 0 { + return Ok(()); + } + av_frame_unref(output); + self.map_videotoolbox = false; + } + // copy_props appends side data and merges dictionaries. Retire those + // along with the pixels if this destination already has frame metadata. + if (!(*output).buf[0].is_null() && av_frame_is_writable(output) == 0) + || (*output).nb_side_data != 0 + || !(*output).metadata.is_null() + { + av_frame_unref(output); + } if (*self.frame_buf.latest_av_frame).format == (AVPixelFormat::AV_PIX_FMT_NONE as i32) { @@ -943,7 +1035,13 @@ impl FFmpegDecoder { ))); } - av_frame_copy_props(self.frame_buf.latest_av_frame, self.hw_frame); + let ret = av_frame_copy_props(output, target_frame); + if ret < 0 { + return Err(FFramesMediaError::LibAVAudioDecodingError(( + ret, + "Error copying hardware frame properties".to_string(), + ))); + } } // The decoder fell back to software decoding: the frame it returned already // holds the pixels. Keep a reference in the target too, since EOF may need @@ -973,6 +1071,23 @@ impl FFmpegDecoder { unsafe fn decode_up_to_pts(&mut self, target_pts: i64) -> Result { unsafe { + if (*self.frame_buf.latest_av_frame).pts == target_pts { + return Ok(true); + } + // A recent frame leaves the stream position alone, so the worker that + // reads in order continues without a seek. + if let Some(frame) = self.recent_frame(target_pts) { + av_frame_unref(self.frame_buf.latest_av_frame); + let ret = av_frame_ref(self.frame_buf.latest_av_frame, frame); + if ret < 0 { + return Err(FFramesMediaError::LibAVAudioDecodingError(( + ret, + "Error reusing a decoded frame".to_string(), + ))); + } + return Ok(true); + } + // Going back needs a seek, otherwise the newer frame that was already decoded // would be returned. Far jumps forward (more than 2 seconds of the stream) seek // to the closest keyframe instead of decoding every frame in between (frames can @@ -1001,14 +1116,19 @@ impl FFmpegDecoder { loop { // A previous call may have returned with more decoded frames queued. // Consume them before sending another packet (which could return EAGAIN). - let ret = avcodec_receive_frame(self.video_stream_info.codec_ctx, self.recv_frame); + let ret = avcodec_receive_frame(self.codec_ctx, self.recv_frame); match ret { 0 => { av_frame_unref(target_frame); av_frame_move_ref(target_frame, self.recv_frame); self.has_decoded_frame = true; - if (*target_frame).pts >= target_pts { + let due = (*target_frame).pts >= target_pts; + // Skipped frames are kept too, so the recent frames stay consecutive. + if due || self.recent_capacity > 0 { self.transfer_hardware_surface_data(target_frame)?; + self.remember(self.frame_buf.latest_av_frame); + } + if due { return Ok(true); } continue; @@ -1057,7 +1177,7 @@ impl FFmpegDecoder { } else { self.pkt.cast_const() }; - let ret = avcodec_send_packet(self.video_stream_info.codec_ctx, packet); + let ret = avcodec_send_packet(self.codec_ctx, packet); av_packet_unref(self.pkt); if ret < 0 { return Err(FFramesMediaError::LibAVAudioDecodingError(( @@ -1076,17 +1196,22 @@ impl Drop for FFmpegDecoder { unsafe { av_frame_free(&raw mut self.recv_frame); av_frame_free(&raw mut self.hw_frame); - avcodec_free_context(&raw mut self.video_stream_info.codec_ctx); + avcodec_free_context(&raw mut self.codec_ctx); avformat_close_input(&raw mut self.fmt_ctx); av_packet_free(&raw mut self.pkt); } + self.forget_recent(); } } -unsafe fn find_hw_accelleleration_for_codec( +unsafe fn find_hw_acceleration_for_codec( hw_device_ctx: &mut *mut AVBufferRef, codec: *const AVCodec, + mode: DecodeMode, ) -> Option { + if mode == DecodeMode::Software { + return None; + } unsafe { let hw_types = [ #[cfg(feature = "vaapi")] @@ -1095,7 +1220,7 @@ unsafe fn find_hw_accelleleration_for_codec( AVHWDeviceType::AV_HWDEVICE_TYPE_D3D11VA, #[cfg(feature = "direct3d9")] AVHWDeviceType::AV_HWDEVICE_TYPE_DXVA2, - #[cfg(feature = "videotoolbox")] + #[cfg(any(feature = "preview-videotoolbox", feature = "videotoolbox"))] AVHWDeviceType::AV_HWDEVICE_TYPE_VIDEOTOOLBOX, #[cfg(feature = "nvidia")] AVHWDeviceType::AV_HWDEVICE_TYPE_CUDA, @@ -1108,18 +1233,16 @@ unsafe fn find_hw_accelleleration_for_codec( ]; for &hw_type in &hw_types { - let create_result = - av_hwdevice_ctx_create(hw_device_ctx, hw_type, ptr::null(), ptr::null_mut(), 0); - - if create_result < 0 { + #[cfg(all(feature = "preview-videotoolbox", not(feature = "videotoolbox")))] + if hw_type == AVHWDeviceType::AV_HWDEVICE_TYPE_VIDEOTOOLBOX + && mode != DecodeMode::Preview + { continue; } - let mut hw_pix_fmt = AVPixelFormat::AV_PIX_FMT_NONE; let mut i = 0; loop { - // Check if decoder supports this hardware if yes we return the available pix fmt let config = avcodec_get_hw_config(codec, i); if config.is_null() { break; @@ -1135,11 +1258,13 @@ unsafe fn find_hw_accelleleration_for_codec( i += 1; } - if hw_pix_fmt != AVPixelFormat::AV_PIX_FMT_NONE { + // The codec context owns the reference returned on success. + if hw_pix_fmt != AVPixelFormat::AV_PIX_FMT_NONE + && av_hwdevice_ctx_create(hw_device_ctx, hw_type, ptr::null(), ptr::null_mut(), 0) + >= 0 + { return Some(hw_pix_fmt); } - - av_buffer_unref(hw_device_ctx); } } @@ -1226,6 +1351,19 @@ mod color_tests { const WIDTH: i32 = 32; const HEIGHT: i32 = 16; + #[test] + fn in_place_alpha_matches_the_renderer_for_every_channel_and_alpha() { + let mut pixels = Vec::new(); + for alpha in 0..=255u8 { + for color in 0..=255u8 { + pixels.extend_from_slice(&[color, 255 - color, color, alpha]); + } + } + let expected = PreloadedImageData::blend_rgba_slice(&pixels); + premultiply_alpha(&mut pixels); + assert_eq!(pixels, expected); + } + struct TestFrame(*mut AVFrame); impl TestFrame { @@ -1269,12 +1407,10 @@ mod color_tests { SwsScaler::new(VideoStreamInfo { hw_pix_fmt: None, stream_index: 0, - codec_ctx: ptr::null_mut(), width: WIDTH, height: HEIGHT, display_width: WIDTH, display_height: HEIGHT, - pixel_format: AVPixelFormat::AV_PIX_FMT_YUV444P, color_space: space, color_range: range, time_base: AVRational { num: 1, den: 30 }, diff --git a/fframes-media/tests/video_decoder_eof.rs b/fframes-media/tests/video_decoder_eof.rs index ecca2a30..4e028faf 100644 --- a/fframes-media/tests/video_decoder_eof.rs +++ b/fframes-media/tests/video_decoder_eof.rs @@ -3,6 +3,7 @@ use std::collections::HashMap; use std::path::PathBuf; use std::process::Command; +use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; use fframes_media::{FFmpegDecoder, FrameConvertOptions, PlaybackSpeed, ResizeVideoFrame}; @@ -97,6 +98,36 @@ fn assert_frame(decoder: &mut FFmpegDecoder, index: i64) { } } +#[test] +fn retained_frame_keeps_its_pixels_and_timestamp_across_decode_and_seek() { + let video = VideoFixture::new(3); + let mut decoder = unsafe { FFmpegDecoder::new(&video.path(), FPS, 1) }.unwrap(); + unsafe { + assert!(decoder.decode_up_to(3).unwrap()); + let held = decoder.retained_frame().unwrap(); + let pts = held.get_pts(); + let expected = held + .convert_last_decoded_frame_into_svg_image(None) + .unwrap(); + + std::thread::scope(|scope| { + let decode = scope.spawn(move || { + assert_frame(&mut decoder, 5); + decoder.seek_to_offset(0).unwrap(); + assert_frame(&mut decoder, 0); + }); + assert_eq!(held.get_pts(), pts); + decode.join().unwrap(); + }); + + assert_eq!(held.get_pts(), pts); + let actual = held + .convert_last_decoded_frame_into_svg_image(None) + .unwrap(); + assert_eq!(actual.data, expected.data); + } +} + #[test] fn b_frames_are_drained_and_decoding_can_restart_after_eof() { let video = VideoFixture::new(3); @@ -173,6 +204,74 @@ fn videos_without_b_frames_stop_at_eof() { } } +#[test] +fn converted_images_survive_single_buffer_reuse_and_decoder_drop() { + let video = VideoFixture::new(3); + let mut decoder = unsafe { FFmpegDecoder::new(&video.path(), FPS, 1) }.unwrap(); + let mut images = Vec::new(); + for index in [0, 17, 2, 90, 299] { + assert!(unsafe { decoder.decode_up_to(index) }.unwrap()); + let image = unsafe { + decoder + .get_raw_frame() + .convert_last_decoded_frame_into_svg_image(None) + } + .unwrap(); + let expected = image.data.to_vec(); + images.push((image, expected)); + } + drop(decoder); + for (image, expected) in images { + assert_eq!(image.data.as_ref(), expected); + } +} + +#[test] +fn released_frames_lend_their_pixel_buffers_to_later_conversions() { + let video = VideoFixture::new(0); + let mut decoder = unsafe { FFmpegDecoder::new(&video.path(), FPS, 1) }.unwrap(); + let mut convert = |index| { + assert!(unsafe { decoder.decode_up_to(index) }.unwrap()); + unsafe { + decoder + .get_raw_frame() + .convert_last_decoded_frame_into_svg_image(None) + } + .unwrap() + }; + let first = convert(0); + let first_pixels = first.data.as_ptr(); + // The first image is still held, so the second one needs its own buffer. + let second = convert(30); + assert_ne!(second.data.as_ptr(), first_pixels); + drop(first); + // Once released, the buffer of the first image is reused. + let third = convert(60); + assert_eq!(third.data.as_ptr(), first_pixels); + assert_ne!(third.data.as_ref(), second.data.as_ref()); + assert_eq!(third.data.len(), 32 * 32 * 4); +} + +#[test] +fn preview_decoders_serve_recent_frames_out_of_order_without_seeking() { + use fframes_media::DecodeMode; + + let video = VideoFixture::new(2); + let mut decoder = + unsafe { FFmpegDecoder::new_with_mode(&video.path(), FPS, 1, DecodeMode::Preview) } + .unwrap(); + // Parallel workers reach the decoder out of order within a few frames. + for index in [100, 102, 101, 104, 103, 100, 105, 107, 106] { + assert_frame(&mut decoder, index); + } + assert_eq!(decoder.seeks(), 1, "only the jump from the start seeks"); + // Beyond the recent frames the decoder seeks back as before. + assert_frame(&mut decoder, 90); + assert_eq!(decoder.seeks(), 2); + assert_frame(&mut decoder, 91); + assert_eq!(decoder.seeks(), 2); +} + #[test] fn a_frame_requested_again_at_another_size_is_converted_to_that_size() { let video = VideoFixture::new(0); @@ -201,22 +300,33 @@ fn a_frame_requested_again_at_another_size_is_converted_to_that_size() { .convert_last_decoded_frame_into_svg_image(None) .unwrap(); assert_eq!((image.width, image.height), (32, 32)); - // The second thumbnail request is served from the cache - for _ in 0..2 { - let image = frame - .convert_last_decoded_frame_into_svg_image(Some(&thumbnail)) - .unwrap(); - assert_eq!( - (image.width, image.height), - (16, 16), - "buffer size {buffer_size}" - ); - // A buffer last sized for a larger output can be longer than the image - assert!( - image.data[..thumbnail_len] == expected.data[..thumbnail_len], - "the thumbnail differs from a decoder that only converts to 16x16 \ - (buffer size {buffer_size})" - ); + let thumb = frame + .convert_last_decoded_frame_into_svg_image(Some(&thumbnail)) + .unwrap(); + assert_eq!((thumb.width, thumb.height), (16, 16)); + assert_eq!(thumb.data.len(), thumbnail_len); + assert_eq!(thumb.data, expected.data); + let again = frame + .convert_last_decoded_frame_into_svg_image(Some(&thumbnail)) + .unwrap(); + assert!(Arc::ptr_eq(&thumb, &again)); + let pts = frame.get_pts(); + assert!(Arc::ptr_eq( + &thumb, + &decoder.get_decoded_image_in_buf(pts).unwrap() + )); + assert!(decoder.decode_up_to(6).unwrap()); + let _full_size = frame + .convert_last_decoded_frame_into_svg_image(None) + .unwrap(); + if buffer_size == 1 { + assert!(decoder.get_decoded_image_in_buf(pts).is_none()); + } else { + // The cached thumbnail retains its own dimensions after a full-size conversion. + assert!(Arc::ptr_eq( + &thumb, + &decoder.get_decoded_image_in_buf(pts).unwrap() + )); } } } @@ -325,6 +435,84 @@ fn frames_can_be_converted_at_widths_that_are_not_a_multiple_of_16() { } } +#[test] +fn preview_decoding_preserves_frames_and_color_across_seeks() { + use fframes_media::DecodeMode; + let fixture = VideoFixture::with_size(3, FPS, 56, (320, 192)); + let file = fixture.path(); + let mut preview = + unsafe { FFmpegDecoder::new_with_mode(&file, FPS, 1, DecodeMode::Preview) }.unwrap(); + let mut software = + unsafe { FFmpegDecoder::new_with_mode(&file, FPS, 1, DecodeMode::Software) }.unwrap(); + for i in [0, 1, 16, 9, 55, 2] { + unsafe { + assert!(preview.decode_up_to(i).unwrap() && software.decode_up_to(i).unwrap()); + let held = preview.retained_frame().unwrap(); + let reference = software.retained_frame().unwrap(); + assert_eq!(held.get_pts(), reference.get_pts()); + // Changing the cursor must not invalidate a mapped VideoToolbox surface. + assert!(preview.decode_up_to((i + 1) % 56).unwrap()); + let a = held + .convert_last_decoded_frame_into_svg_image(None) + .unwrap(); + let b = reference + .convert_last_decoded_frame_into_svg_image(None) + .unwrap(); + assert_eq!((a.width, a.height), (320, 192)); + assert_eq!(a.data.len(), b.data.len()); + // Hardware can reconstruct a channel a little differently than libavcodec. + let error: usize = a + .data + .iter() + .zip(b.data.iter()) + .map(|(x, y)| x.abs_diff(*y) as usize) + .sum(); + assert!( + error < a.data.len() * 2, + "frame {i}: mean channel difference exceeds two" + ); + } + } +} + +#[test] +fn preview_falls_back_for_lossless_alpha_video() { + use fframes_media::DecodeMode; + let fixture = VideoFixture::with_rate(0, FPS, 8); + let file = fixture.path().with_extension("mkv"); + let output = Command::new("ffmpeg") + .args([ + "-v", + "error", + "-y", + "-f", + "lavfi", + "-i", + "testsrc2=size=32x32:rate=30,format=bgra,colorchannelmixer=aa=0.5", + "-frames:v", + "8", + "-c:v", + "ffv1", + ]) + .arg(&file) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let mut preview = + unsafe { FFmpegDecoder::new_with_mode(&file, FPS, 1, DecodeMode::Preview) }.unwrap(); + let mut software = + unsafe { FFmpegDecoder::new_with_mode(&file, FPS, 1, DecodeMode::Software) }.unwrap(); + for i in [0, 6, 3] { + let pixels = convert_frame(&mut preview, i, None); + assert_eq!(pixels, convert_frame(&mut software, i, None)); + assert!((127..=128).contains(&pixels[3])); + } +} + /// The source frame that offset `n` shows at `num / den` speed: the first frame at or /// after `n * num / den` (the last frame until the stream ends), in a loop if `looping`. fn expected_frame(offset: i64, num: i64, den: i64, looping: bool) -> i64 { diff --git a/fframes-native-player/Cargo.toml b/fframes-native-player/Cargo.toml index ba65e10b..b691bc13 100644 --- a/fframes-native-player/Cargo.toml +++ b/fframes-native-player/Cargo.toml @@ -17,9 +17,11 @@ fframes_skia_renderer.workspace = true winit = "0.30" softbuffer = "0.4" cpal = { version = "0.18", optional = true } +rtrb = { version = "0.4.1", optional = true } # The CAMetalLayer of the window, presented to by Metal and by Vulkan (MoltenVK). [target.'cfg(target_os = "macos")'.dependencies] +fframes = { workspace = true, features = ["preview-videotoolbox"] } raw-window-metal = { version = "1.1", optional = true } core-graphics-types = { version = "0.2", optional = true, default-features = false } @@ -29,7 +31,7 @@ hello-world-example = { path = "../examples/hello-world" } [features] default = ["vulkan", "audio"] -audio = ["cpal"] +audio = ["cpal", "dep:rtrb"] # At least one GPU backend is required to build Skia with Ganesh. The player falls back # to the Skia CPU raster backend at runtime if the GPU backend can not be initialized. vulkan = ["fframes_skia_renderer/vulkan", "dep:raw-window-metal"] diff --git a/fframes-native-player/src/app.rs b/fframes-native-player/src/app.rs index 94ddc630..977369fb 100644 --- a/fframes-native-player/src/app.rs +++ b/fframes-native-player/src/app.rs @@ -2,10 +2,9 @@ use std::fmt::Write as _; use std::rc::Rc; use std::time::{Duration, Instant}; -use fframes::usvgr; use winit::application::ApplicationHandler; use winit::dpi::{LogicalSize, PhysicalPosition}; -use winit::event::{ElementState, KeyEvent, MouseButton, WindowEvent}; +use winit::event::{ElementState, KeyEvent, MouseButton, MouseScrollDelta, WindowEvent}; use winit::event_loop::{ActiveEventLoop, ControlFlow}; use winit::keyboard::{Key, NamedKey}; use winit::window::{CursorIcon, Fullscreen, Window, WindowId}; @@ -15,13 +14,18 @@ use crate::error::PlayerError; use crate::options::PlayerBackend; use crate::presenter::Presenter; use crate::scheduler::Scheduler; +use crate::studio::{Studio, StudioLayout, StudioScroll}; +use fframes_skia_renderer::skia_safe::Point; /// The control bar hides after this long without pointer movement while playing. const CONTROLS_IDLE_TIMEOUT: Duration = Duration::from_millis(2500); -/// Sent by the frame workers whenever a frame becomes available. -#[derive(Debug, Clone, Copy)] -pub(crate) struct FrameReady; +/// A frame worker finished a playback or filmstrip job. +pub(crate) enum FrameReady { + Playback, + Failed { position: u64, epoch: u64 }, + Thumbnail(usize, fframes_skia_renderer::skia_safe::Image), +} /// Playback clock in player positions (see `scheduler`), driven by wall time. struct Clock { @@ -61,8 +65,10 @@ impl Clock { } pub(crate) struct AppConfig<'a> { + pub studio: Studio, pub title: &'a str, pub window_size: LogicalSize, + pub fit_portrait_to_screen: bool, pub backend: PlayerBackend, pub vsync: bool, pub fps: usize, @@ -80,12 +86,14 @@ pub(crate) struct App<'s> { audio: Option<&'s crate::audio::AudioOutput>, window: Option>, + /// The tree on screen. Skia copies its video frames to the GPU during the flush. + current: Option<(u64, fframes::usvgr::Tree)>, presenter: Option, + redraw_pending: bool, pub(crate) error: Option, clock: Clock, looping: bool, - current: Option<(u64, usvgr::Tree)>, /// Toggled with `b`. controls_enabled: bool, controls_visible: bool, @@ -95,7 +103,12 @@ pub(crate) struct App<'s> { cursor_inside: bool, last_pointer_activity: Instant, hovered: Option, - seeking_with_mouse: bool, + drag: Option, + scene_press_axis: f32, + /// The newest destination under the pointer while a native frame is being prepared. + scrub_target: Option, + studio_open: bool, + scroll: StudioScroll, stats_since: Instant, /// New frames drawn into the window since `stats_since`. Frames taken from the @@ -127,6 +140,7 @@ impl<'s> App<'s> { audio, window: None, presenter: None, + redraw_pending: false, error: None, clock, current: None, @@ -137,7 +151,11 @@ impl<'s> App<'s> { cursor_inside: false, last_pointer_activity: now, hovered: None, - seeking_with_mouse: false, + drag: None, + scene_press_axis: 0., + scrub_target: None, + studio_open: true, + scroll: StudioScroll::default(), stats_since: now, frames_shown: 0, presented: None, @@ -164,30 +182,45 @@ impl<'s> App<'s> { self.scheduler.frame_of(position, self.looping) } - /// Restarts playback at `position`, all the prepared frames are discarded. + /// Seeks to `position`, reusing prepared scenes when possible. fn seek_to(&mut self, position: u64, playing: bool) { - self.clock.set(position, playing); - self.scheduler.seek(position, self.looping); + self.scrub_target = None; + if !self.scheduler.seek_buffered(position, self.looping) { + self.scheduler.seek(position, self.looping); + } + self.set_playback(position, playing); + } + fn set_playback(&mut self, position: u64, playing: bool) { + self.clock.set(position, playing); #[cfg(feature = "audio")] if let Some(audio) = self.audio { audio.reset(position, self.config.fps, playing, self.looping); } } + fn playhead(&self) -> usize { + self.scrub_target.unwrap_or_else(|| self.current_frame()) + } + fn seek_to_frame(&mut self, frame: i64) { let frame = frame.clamp(0, self.last_frame() as i64) as u64; self.seek_to(frame, self.clock.is_playing()); } fn toggle_play(&mut self) { - let frame = self.current_frame(); - if self.clock.is_playing() { - self.seek_to(frame as u64, false); - } else if !self.looping && frame >= self.last_frame() { + let frame = self.playhead(); + let playing = !self.clock.is_playing(); + let position = self.clock.position(); + if playing && !self.looping && frame >= self.last_frame() { self.seek_to(0, true); + } else if self.scrub_target.is_none() + && self.scheduler.frame_of(position, self.looping) == frame + { + // The workers have already prepared frames ahead of this position. + self.set_playback(position, playing); } else { - self.seek_to(frame as u64, true); + self.seek_to(frame as u64, playing); } } @@ -203,8 +236,10 @@ impl<'s> App<'s> { } if let Some(frame) = self.scheduler.take(position) { + let next_frame = self.scheduler.frame_of(frame.0, self.looping); self.current = Some(frame); self.needs_redraw = true; + self.finish_scrub(next_frame); } let controls_visible = self.controls_should_be_visible(); @@ -216,6 +251,7 @@ impl<'s> App<'s> { if self.needs_redraw { self.needs_redraw = false; if let Some(window) = &self.window { + self.redraw_pending = true; window.request_redraw(); } } @@ -223,6 +259,31 @@ impl<'s> App<'s> { self.update_title(); } + fn finish_scrub(&mut self, frame: usize) { + if let Some(target) = self.scrub_target.take() { + if target != frame { + self.seek_to(target as u64, false); + self.scrub_target = Some(target); + } + self.needs_redraw = true; + } + } + + fn frame_ready(&mut self, event: FrameReady) { + match event { + FrameReady::Thumbnail(slot, image) => { + self.config.studio.set_thumbnail(slot, image); + self.needs_redraw = true; + } + FrameReady::Failed { position, epoch } + if self.scheduler.is_current_seek(position, epoch) => + { + self.finish_scrub(self.scheduler.frame_of(position, self.looping)); + } + FrameReady::Playback | FrameReady::Failed { .. } => {} + } + } + fn update_title(&mut self) { let now = Instant::now(); let stats_elapsed = now.duration_since(self.stats_since).as_secs_f64(); @@ -269,7 +330,7 @@ impl<'s> App<'s> { return; } - let frame = self.current_frame() as i64; + let frame = self.playhead() as i64; let second = self.config.fps as i64; match event.logical_key.as_ref() { Key::Named(NamedKey::Space) => self.toggle_play(), @@ -295,6 +356,13 @@ impl<'s> App<'s> { self.controls_enabled = !self.controls_enabled; self.needs_redraw = true; } + Key::Character("t") => self.toggle_studio(), + Key::Character("[" | "]") => { + let next = event.logical_key == Key::Character("]".into()); + if let Some(target) = self.config.studio.adjacent_scene(frame as usize, next) { + self.seek_to_frame(target as i64); + } + } Key::Character("f") => self.toggle_fullscreen(), Key::Character(digit) if digit.len() == 1 && digit.as_bytes()[0].is_ascii_digit() => { let tenth = i64::from(digit.as_bytes()[0] - b'0'); @@ -310,6 +378,37 @@ impl<'s> App<'s> { self.seek_to(frame, false); } + fn studio_layout(&self) -> Option<&StudioLayout> { + self.presenter.as_ref()?.studio_layout.as_ref() + } + + fn pointer(&self) -> Point { + let scale = self.window.as_ref().map_or(1., |w| w.scale_factor()); + let pos = self.cursor.to_logical::(scale); + Point::new(pos.x, pos.y) + } + + fn invalidate_view(&mut self) { + self.needs_redraw = true; + } + + /// A drag along the timeline or the seek slider turns lookahead off: the workers + /// prepare only the frame under the pointer until the button is released. + fn set_drag(&mut self, drag: Option) { + self.drag = drag; + self.scheduler + .set_scrubbing(matches!(drag, Some(Control::Seek | Control::Timeline))); + } + + fn toggle_studio(&mut self) { + self.invalidate_view(); + self.studio_open = !self.studio_open; + self.set_drag(None); + self.hovered = None; + self.last_pointer_activity = Instant::now(); + self.needs_redraw = true; + } + fn toggle_fullscreen(&mut self) { let Some(window) = &self.window else { return }; let fullscreen = match window.fullscreen() { @@ -329,27 +428,56 @@ impl<'s> App<'s> { fn controls_should_be_visible(&self) -> bool { self.controls_enabled && (!self.clock.is_playing() - || self.seeking_with_mouse + || self.drag.is_some() || (self.cursor_inside && self.hovered.is_some()) || self.last_pointer_activity.elapsed() < CONTROLS_IDLE_TIMEOUT) } fn control_under_cursor(&self) -> Option { - if !self.controls_visible { - return None; + let point = self.pointer(); + if self.studio_open + && let Some(layout) = self.studio_layout() + { + return layout.hit(point, &self.config.studio); } - - self.presenter.as_ref().and_then(|presenter| { - presenter + if self.controls_visible + && let Some(control) = self + .presenter + .as_ref()? .controls_layout() .hit(self.cursor.x as f32, self.cursor.y as f32) - }) + { + return Some(control); + } + let layout = self.studio_layout()?; + (self.controls_visible && layout.focus_hit(point)).then_some(Control::Focus) } fn seek_to_cursor(&mut self) { let Some(presenter) = &self.presenter else { return; }; + if self.drag == Some(Control::Timeline) { + if let Some(ui) = self.studio_layout() { + let frame = + ui.frame_at(ui.time_axis(self.pointer()), self.config.duration_in_frames); + // Finish the in-flight frame before requesting the newest pointer + // position. Starting a fresh generation on every move can starve it. + if self.scrub_target.is_none() { + if !self.clock.is_playing() + && self.current.is_some() + && frame == self.current_frame() + && self.clock.position() == frame as u64 + { + return; + } + self.seek_to(frame as u64, false); + } + self.scrub_target = Some(frame); + self.needs_redraw = true; + } + return; + } let progress = f64::from( presenter .controls_layout() @@ -364,7 +492,14 @@ impl<'s> App<'s> { self.cursor_inside = true; self.last_pointer_activity = Instant::now(); - if self.seeking_with_mouse { + if matches!(self.drag, Some(Control::Scene(_))) + && self.studio_layout().is_some_and(|ui| { + (ui.time_axis(self.pointer()) - self.scene_press_axis).abs() >= 3. + }) + { + self.set_drag(Some(Control::Timeline)); + } + if matches!(self.drag, Some(Control::Seek | Control::Timeline)) { self.seek_to_cursor(); } @@ -376,6 +511,12 @@ impl<'s> App<'s> { if let Some(window) = &self.window { window.set_cursor(match hovered { None | Some(Control::Bar) => CursorIcon::Default, + Some(Control::Timeline) + if self.studio_layout().is_some_and(|ui| ui.portrait) => + { + CursorIcon::NsResize + } + Some(Control::Timeline) => CursorIcon::EwResize, Some(_) => CursorIcon::Pointer, }); } @@ -394,16 +535,23 @@ impl<'s> App<'s> { } fn on_pointer_pressed(&mut self) { - let frame = self.current_frame() as i64; + let frame = self.playhead() as i64; let second = self.config.fps as i64; match self.control_under_cursor() { Some(Control::Back) => self.seek_to_frame(frame - second), Some(Control::Forward) => self.seek_to_frame(frame + second), - Some(Control::Seek) => { - self.seeking_with_mouse = true; + Some(control @ (Control::Seek | Control::Timeline)) => { + self.set_drag(Some(control)); self.seek_to_cursor(); } + Some(control @ Control::Scene(_)) => { + if let Some(ui) = self.studio_layout() { + self.scene_press_axis = ui.time_axis(self.pointer()); + } + self.set_drag(Some(control)); + } + Some(Control::Focus) => self.toggle_studio(), Some(Control::Loop) => { self.looping = !self.looping; let playing = self.clock.is_playing(); @@ -417,10 +565,21 @@ impl<'s> App<'s> { self.needs_redraw = true; } + fn on_pointer_released(&mut self) { + let drag = self.drag; + self.set_drag(None); + if let Some(control @ Control::Scene(start)) = drag + && self.control_under_cursor() == Some(control) + { + self.step(start as i64); + } + } + fn redraw(&mut self) -> Result<(), PlayerError> { + self.redraw_pending = false; let controls = ControlsState { visible: self.controls_visible, - frame: self.current_frame(), + frame: self.playhead(), duration_in_frames: self.config.duration_in_frames, fps: self.config.fps, playing: self.clock.is_playing(), @@ -430,7 +589,13 @@ impl<'s> App<'s> { }; if let Some(presenter) = self.presenter.as_mut() { - presenter.present(self.current.as_ref().map(|(_, tree)| tree), &controls)?; + presenter.present( + self.current.as_ref().map(|(_, tree)| tree), + &controls, + &self.config.studio, + &self.scroll, + self.studio_open, + )?; let position = self.current.as_ref().map(|(position, _)| *position); if position.is_some() && position != self.presented { @@ -454,6 +619,24 @@ impl ApplicationHandler for App<'_> { return; } + if self.config.fit_portrait_to_screen + && let Some(monitor) = event_loop + .primary_monitor() + .or_else(|| event_loop.available_monitors().next()) + { + let screen = monitor.size().to_logical::(monitor.scale_factor()); + // Leave room for the window decorations and desktop controls. + let height = f64::from(self.config.window_size.height) + .min(screen.height - 72.) + .max(1.); + let width = height * f64::from(self.config.studio.video_aspect) + + f64::from(crate::studio::PORTRAIT_SIDEBAR_WIDTH); + self.config.window_size = LogicalSize::new( + width.min(screen.width - 24.).max(1.).round() as u32, + height.round() as u32, + ); + } + let attributes = Window::default_attributes() .with_title(self.config.title) .with_inner_size(self.config.window_size); @@ -463,14 +646,7 @@ impl ApplicationHandler for App<'_> { Err(err) => return self.fail(event_loop, err.into()), }; - let duration_label = format_time(self.config.duration_in_frames, self.config.fps); - match Presenter::new( - window.clone(), - self.config.backend, - self.config.vsync, - self.config.background, - duration_label, - ) { + match Presenter::new(window.clone(), &self.config) { Ok(presenter) => self.presenter = Some(presenter), Err(err) => return self.fail(event_loop, err), } @@ -479,7 +655,8 @@ impl ApplicationHandler for App<'_> { self.needs_redraw = true; } - fn user_event(&mut self, _event_loop: &ActiveEventLoop, _event: FrameReady) { + fn user_event(&mut self, _event_loop: &ActiveEventLoop, event: FrameReady) { + self.frame_ready(event); self.tick(); } @@ -490,7 +667,7 @@ impl ApplicationHandler for App<'_> { if let Some(Err(err)) = self.presenter.as_mut().map(Presenter::resize) { return self.fail(event_loop, err); } - self.needs_redraw = true; + self.invalidate_view(); } WindowEvent::RedrawRequested => { if let Err(err) = self.redraw() { @@ -499,6 +676,28 @@ impl ApplicationHandler for App<'_> { } WindowEvent::KeyboardInput { event, .. } => self.handle_key(event_loop, event), WindowEvent::CursorMoved { position, .. } => self.on_pointer_moved(position), + WindowEvent::MouseWheel { delta, .. } if self.studio_open && self.cursor_inside => { + let point = self.pointer(); + let dy = match delta { + MouseScrollDelta::LineDelta(_, dy) => f64::from(dy), + MouseScrollDelta::PixelDelta(delta) => delta.y, + }; + if let Some(ui) = self + .presenter + .as_ref() + .and_then(|p| p.studio_layout.as_ref()) + { + ui.scroll( + &self.config.studio, + &mut self.scroll, + point, + isize::from(dy < 0.) - isize::from(dy > 0.), + ); + self.hovered = self.control_under_cursor(); + self.needs_redraw = true; + } + } + WindowEvent::Focused(false) => self.set_drag(None), WindowEvent::CursorLeft { .. } => { self.cursor_inside = false; if self.hovered.take().is_some() { @@ -511,7 +710,7 @@ impl ApplicationHandler for App<'_> { .. } => match state { ElementState::Pressed => self.on_pointer_pressed(), - ElementState::Released => self.seeking_with_mouse = false, + ElementState::Released => self.on_pointer_released(), }, _ => {} } @@ -527,3 +726,93 @@ impl ApplicationHandler for App<'_> { }); } } + +#[cfg(test)] +mod tests { + use super::*; + + fn paused_app(scheduler: &Scheduler) -> App<'_> { + let timeline = fframes::ResolvedRenderingTimeline { + audio_map: None, + scenes: None, + duration_in_frames: 100, + }; + let time = fframes::TimeBase { + fps: 30, + sample_rate: 44100, + }; + App::new( + AppConfig { + studio: Studio::new(time, &timeline, None, 0, 1.), + title: "test", + window_size: LogicalSize::new(640, 480), + fit_portrait_to_screen: false, + backend: PlayerBackend::Cpu, + vsync: false, + fps: time.fps, + duration_in_frames: timeline.duration_in_frames, + background: fframes::Color::BLACK, + looping: false, + autoplay: false, + start_frame: 0, + }, + scheduler, + #[cfg(feature = "audio")] + None, + ) + } + + #[test] + fn resuming_keeps_prefetched_frames_unless_playback_restarts() { + let scheduler = Scheduler::new(100, 3, false, 0, false); + let mut app = paused_app(&scheduler); + app.seek_to(10, false); + app.toggle_play(); + assert!(app.clock.is_playing()); + assert!(scheduler.is_current_seek(10, 2)); + + app.seek_to(99, false); + app.toggle_play(); + assert!(scheduler.is_current_seek(0, 4)); + } + + #[test] + fn failed_scrub_continues_at_the_latest_pointer_position() { + let scheduler = Scheduler::new(100, 3, false, 0, false); + let mut app = paused_app(&scheduler); + app.seek_to(10, false); + app.scrub_target = Some(20); + // A stale completion (even at the same position) and a failed lookahead + // frame must not advance the pending scrub. + for (position, epoch) in [(10, 1), (11, 2)] { + app.frame_ready(FrameReady::Failed { position, epoch }); + assert_eq!(app.clock.position(), 10); + assert_eq!(app.scrub_target, Some(20)); + } + app.frame_ready(FrameReady::Failed { + position: 10, + epoch: 2, + }); + assert_eq!(app.clock.position(), 20); + assert_eq!(app.scrub_target, Some(20)); + app.frame_ready(FrameReady::Failed { + position: 20, + epoch: 3, + }); + assert_eq!(app.scrub_target, None); + assert!(scheduler.is_current_seek(20, 3)); + } + + #[test] + fn a_timeline_drag_turns_lookahead_off_until_the_button_is_released() { + let scheduler = Scheduler::new(100, 3, false, 0, false); + let mut app = paused_app(&scheduler); + app.set_drag(Some(Control::Scene(0))); + assert!(!scheduler.is_scrubbing()); + app.set_drag(Some(Control::Timeline)); + assert!(scheduler.is_scrubbing()); + app.on_pointer_released(); + assert!(!scheduler.is_scrubbing()); + assert_eq!(app.drag, None); + } +} diff --git a/fframes-native-player/src/audio.rs b/fframes-native-player/src/audio.rs index 1013bccd..979a5198 100644 --- a/fframes-native-player/src/audio.rs +++ b/fframes-native-player/src/audio.rs @@ -4,7 +4,8 @@ use fframes::{ AudioMixOptions, AudioMixer, AudioTimelineSamples, AudioTimelineUnit, FFramesContext, ResolvedAudioMap, }; -use std::collections::VecDeque; +use rtrb::{Consumer, Producer, RingBuffer}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, Condvar, Mutex, MutexGuard}; /// Samples mixed per step, same order of magnitude as an encoder audio frame. @@ -14,30 +15,40 @@ pub(crate) struct AudioOutput { // Not `Send` on every platform, lives on the main thread. _stream: cpal::Stream, shared: Arc, + /// The write end of the sample ring, used by the feeder thread only. + producer: Mutex>, pub(crate) sample_rate: usize, } +/// A mixed stereo sample, tagged with the playback it was mixed for. +#[derive(Clone, Copy)] +struct Sample { + left: f32, + right: f32, + epoch: u64, +} + struct Shared { - queue: Mutex, + control: Mutex, feeder_wakeup: Condvar, + /// Read by the audio callback without a lock. Both change under `control`. + playing: AtomicBool, + /// Bumped on every seek. The callback discards samples of earlier epochs. + epoch: AtomicU64, /// How many mixed samples to keep queued (~100ms). target_len: usize, } -struct Queue { - /// Stereo frames `(left, right)`. - samples: VecDeque<(f32, f32)>, - playing: bool, +struct Control { looping: bool, /// Next position to mix, in samples since the start of the (possibly looped) playback. feed_position: u64, - epoch: u64, shutdown: bool, } impl Shared { - fn lock(&self) -> MutexGuard<'_, Queue> { - self.queue + fn lock(&self) -> MutexGuard<'_, Control> { + self.control .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) } @@ -52,26 +63,37 @@ impl AudioOutput { let sample_rate = config.sample_rate() as usize; let channels = config.channels() as usize; let sample_format = config.sample_format(); + let target_len = sample_rate / 10; let shared = Arc::new(Shared { - queue: Mutex::new(Queue { - samples: VecDeque::new(), - playing: false, + control: Mutex::new(Control { looping: false, feed_position: 0, - epoch: 0, shutdown: false, }), feeder_wakeup: Condvar::new(), - target_len: sample_rate / 10, + playing: AtomicBool::new(false), + epoch: AtomicU64::new(0), + target_len, }); + // Room for the samples of the current playback plus the ones of the playback + // before a seek, which the callback discards. + let (producer, consumer) = RingBuffer::new(2 * target_len + CHUNK); let config = config.config(); let stream = match sample_format { - cpal::SampleFormat::F32 => build_stream::(&device, config, channels, &shared), - cpal::SampleFormat::I16 => build_stream::(&device, config, channels, &shared), - cpal::SampleFormat::U16 => build_stream::(&device, config, channels, &shared), - cpal::SampleFormat::I32 => build_stream::(&device, config, channels, &shared), + cpal::SampleFormat::F32 => { + build_stream::(&device, config, channels, &shared, consumer) + } + cpal::SampleFormat::I16 => { + build_stream::(&device, config, channels, &shared, consumer) + } + cpal::SampleFormat::U16 => { + build_stream::(&device, config, channels, &shared, consumer) + } + cpal::SampleFormat::I32 => { + build_stream::(&device, config, channels, &shared, consumer) + } format => return Err(format!("unsupported sample format {format}")), }?; stream.play().map_err(|e| e.to_string())?; @@ -79,19 +101,19 @@ impl AudioOutput { Ok(Self { _stream: stream, shared, + producer: Mutex::new(producer), sample_rate, }) } /// Restarts the audio at `position` (in frames of the player position space). pub(crate) fn reset(&self, position: u64, fps: usize, playing: bool, looping: bool) { - let mut queue = self.shared.lock(); - queue.epoch += 1; - queue.samples.clear(); - queue.feed_position = position * self.sample_rate as u64 / fps as u64; - queue.playing = playing; - queue.looping = looping; - drop(queue); + let mut control = self.shared.lock(); + control.feed_position = position * self.sample_rate as u64 / fps as u64; + control.looping = looping; + self.shared.epoch.fetch_add(1, Ordering::AcqRel); + self.shared.playing.store(playing, Ordering::Release); + drop(control); self.shared.feeder_wakeup.notify_all(); } @@ -100,7 +122,7 @@ impl AudioOutput { self.shared.feeder_wakeup.notify_all(); } - /// Mixes audio into the queue until `shutdown`. Runs on a scoped thread. + /// Mixes audio into the ring until `shutdown`. Runs on a scoped thread. pub(crate) fn run_feeder( &self, ctx: &FFramesContext<'_, '_>, @@ -125,25 +147,38 @@ impl AudioOutput { } let mut left = vec![0.; CHUNK]; let mut right = vec![0.; CHUNK]; + let mut producer = self + .producer + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let capacity = producer.buffer().capacity(); loop { let (position, looping, epoch) = { - let mut queue = self.shared.lock(); + let mut control = self.shared.lock(); loop { - if queue.shutdown { + if control.shutdown { return; } - let has_more = queue.looping || queue.feed_position < total_samples; - if queue.playing && has_more && queue.samples.len() < self.shared.target_len { + let has_more = control.looping || control.feed_position < total_samples; + let queued = capacity - producer.slots(); + if self.shared.playing.load(Ordering::Acquire) + && has_more + && queued < self.shared.target_len + { break; } - queue = self + control = self .shared .feeder_wakeup - .wait(queue) + .wait(control) .unwrap_or_else(std::sync::PoisonError::into_inner); } - (queue.feed_position, queue.looping, queue.epoch) + ( + control.feed_position, + control.looping, + self.shared.epoch.load(Ordering::Acquire), + ) }; let timeline_sample = if looping { @@ -161,15 +196,16 @@ impl AudioOutput { &mut right[..len], ); - let mut queue = self.shared.lock(); - if queue.epoch == epoch { - queue.samples.extend( - left[..len] - .iter() - .copied() - .zip(right[..len].iter().copied()), - ); - queue.feed_position += len as u64; + let mut control = self.shared.lock(); + // The wait above leaves at least a chunk of room in the ring. + if self.shared.epoch.load(Ordering::Acquire) == epoch + && let Ok(chunk) = producer.write_chunk_uninit(len) + { + let samples = left[..len] + .iter() + .zip(&right[..len]) + .map(|(&left, &right)| Sample { left, right, epoch }); + control.feed_position += chunk.fill_from_iter(samples) as u64; } } } @@ -180,19 +216,27 @@ fn build_stream>( config: cpal::StreamConfig, channels: usize, shared: &Arc, + mut consumer: Consumer, ) -> Result { let shared = Arc::clone(shared); device .build_output_stream( config, + // The callback never blocks: it reads the ring and two atomics. move |data: &mut [T], _: &cpal::OutputCallbackInfo| { - let mut queue = shared.lock(); - let playing = queue.playing; + let epoch = shared.epoch.load(Ordering::Acquire); + let playing = shared.playing.load(Ordering::Acquire); for frame in data.chunks_mut(channels) { + // Samples mixed before the last seek are discarded, also while paused. + while consumer.peek().is_ok_and(|sample| sample.epoch != epoch) { + let _ = consumer.pop(); + } let (left, right) = if playing { - queue.samples.pop_front().unwrap_or((0.0, 0.0)) + consumer + .pop() + .map_or((0.0, 0.0), |sample| (sample.left, sample.right)) } else { (0.0, 0.0) }; @@ -208,9 +252,7 @@ fn build_stream>( } } - let needs_more = playing && queue.samples.len() < shared.target_len; - drop(queue); - if needs_more { + if playing && consumer.slots() < shared.target_len { shared.feeder_wakeup.notify_one(); } }, diff --git a/fframes-native-player/src/controls.rs b/fframes-native-player/src/controls.rs index bd026f95..d0af0c54 100644 --- a/fframes-native-player/src/controls.rs +++ b/fframes-native-player/src/controls.rs @@ -16,6 +16,9 @@ pub(crate) enum Control { PlayPause, Forward, Seek, + Timeline, + Scene(usize), + Focus, Loop, Fullscreen, /// The bar background, swallows clicks so they do not toggle playback. @@ -86,6 +89,7 @@ pub(crate) struct Controls { regular: Typeface, medium: Typeface, icons: IconPaths, + pub(crate) frame_mark: skia_safe::svg::Dom, } struct IconPaths { @@ -120,6 +124,8 @@ impl ControlsLayout { height: f32, scale: f32, ) -> Self { + // Keep both time labels and icon hitboxes apart in a narrow window. + let scale = scale.min(width / 400.); let px = |logical: f32| logical * scale; let font = controls.font(true, scale); let time_width = tabular_width(&font, duration_label); @@ -221,9 +227,14 @@ impl Controls { } }; + let frame_mark = + skia_safe::svg::Dom::from_str(include_str!("frame-f.svg"), FontMgr::empty()) + .expect("valid fframes mark SVG"); + Self { regular: typeface(font_style::Weight::NORMAL), medium: typeface(font_style::Weight::MEDIUM), + frame_mark, icons: IconPaths { play: icon(icons::PLAY), pause: icon(icons::PAUSE), @@ -236,7 +247,7 @@ impl Controls { } } - fn font(&self, medium: bool, scale: f32) -> Font { + pub(crate) fn font(&self, medium: bool, scale: f32) -> Font { let typeface = if medium { &self.medium } else { &self.regular }; let mut font = Font::from_typeface(typeface, FONT_SIZE * scale); font.set_subpixel(true); @@ -272,53 +283,15 @@ impl Controls { let hovered = |control| state.hovered == Some(control); - self.ghost_button( - canvas, - layout.back, - &self.icons.back, - hovered(Control::Back), - false, - scale, - ); - self.primary_button( - canvas, - layout.play, - if state.playing { - &self.icons.pause - } else { - &self.icons.play - }, - hovered(Control::PlayPause), - scale, - ); - self.ghost_button( - canvas, - layout.forward, - &self.icons.forward, - hovered(Control::Forward), - false, - scale, - ); - self.ghost_button( - canvas, - layout.looping, - &self.icons.looping, - hovered(Control::Loop), - state.looping, - scale, - ); - self.ghost_button( - canvas, - layout.fullscreen, - if state.fullscreen { - &self.icons.collapse - } else { - &self.icons.expand - }, - hovered(Control::Fullscreen), - false, - scale, - ); + for (control, rect) in [ + (Control::Back, layout.back), + (Control::PlayPause, layout.play), + (Control::Forward, layout.forward), + (Control::Loop, layout.looping), + (Control::Fullscreen, layout.fullscreen), + ] { + self.draw_button(canvas, control, rect, state, scale); + } let mut text = paint.clone(); text.set_color(TEXT); @@ -369,6 +342,46 @@ impl Controls { } } + pub(crate) fn draw_button( + &self, + canvas: &Canvas, + control: Control, + rect: Rect, + state: &ControlsState, + scale: f32, + ) { + let hovered = state.hovered == Some(control); + let icon = match control { + Control::PlayPause => { + let icon = if state.playing { + &self.icons.pause + } else { + &self.icons.play + }; + return self.primary_button(canvas, rect, icon, hovered, scale); + } + Control::Back => &self.icons.back, + Control::Forward => &self.icons.forward, + Control::Loop => &self.icons.looping, + Control::Fullscreen => { + if state.fullscreen { + &self.icons.collapse + } else { + &self.icons.expand + } + } + _ => return, + }; + self.ghost_button( + canvas, + rect, + icon, + hovered, + control == Control::Loop && state.looping, + scale, + ); + } + fn ghost_button( &self, canvas: &Canvas, @@ -427,7 +440,7 @@ fn digit_width(font: &Font) -> f32 { .fold(0., f32::max) } -fn tabular_width(font: &Font, text: &str) -> f32 { +pub(crate) fn tabular_width(font: &Font, text: &str) -> f32 { let digit = digit_width(font); text.chars() .map(|char| { @@ -440,7 +453,13 @@ fn tabular_width(font: &Font, text: &str) -> f32 { .sum() } -fn draw_tabular(canvas: &Canvas, text: &str, origin: Point, font: &Font, paint: &Paint) { +pub(crate) fn draw_tabular( + canvas: &Canvas, + text: &str, + origin: Point, + font: &Font, + paint: &Paint, +) -> f32 { let digit = digit_width(font); let mut x = origin.x; for char in text.chars() { @@ -454,6 +473,7 @@ fn draw_tabular(canvas: &Canvas, text: &str, origin: Point, font: &Font, paint: x += width; } } + x } /// `mm:ss`, or `h:mm:ss` from an hour on, like the web editor. diff --git a/fframes-native-player/src/frame-f.svg b/fframes-native-player/src/frame-f.svg new file mode 100644 index 00000000..c2375eae --- /dev/null +++ b/fframes-native-player/src/frame-f.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/fframes-native-player/src/lib.rs b/fframes-native-player/src/lib.rs index 9de36745..227b4c3d 100644 --- a/fframes-native-player/src/lib.rs +++ b/fframes-native-player/src/lib.rs @@ -20,8 +20,11 @@ //! - `h`/`l` or `←`/`→`: seek one second back/forward //! - `j`/`k` or `↓`/`↑` (also `,`/`.`): step one frame back/forward, pauses //! - `g`/`G` or `Home`/`End`: first/last frame, `0`-`9`: jump to 0-90% -//! - the control bar (shown while paused or when the pointer moves): back, play/pause, -//! forward, a seek slider, looping and full screen, `b` hides it +//! - Studio keeps playback controls above the timeline. Focus mode shows a floating +//! dock with a seek slider while paused or on pointer activity; `b` hides the dock. +//! - click a scene to jump to its start, or drag across the timeline to scrub +//! - `[` / `]`: previous/next scene start, `t`: switch between the studio and video-only views +//! - scroll the audio lanes to browse overlapping clips //! - `r`: toggle looping, `f`: full screen, `q`/`Esc`: quit //! //! [`play`] must be called from the main thread and only once per process (a winit @@ -36,6 +39,7 @@ mod icons; mod options; mod presenter; mod scheduler; +mod studio; pub use error::*; pub use options::*; @@ -43,14 +47,12 @@ pub use options::*; use std::collections::HashMap; use std::sync::Mutex; -use fframes::{ - FFramesContext, FFramesRendererRuntime, TimeBase, Video, VideoDecodersWorker, VideoSize, usvgr, -}; +use fframes::{FFramesContext, FFramesRendererRuntime, TimeBase, Video, VideoSize, usvgr}; use winit::dpi::LogicalSize; use winit::event_loop::EventLoop; use crate::app::{App, AppConfig, FrameReady}; -use crate::scheduler::Scheduler; +use crate::scheduler::{Scheduler, WorkerKind}; /// Opens a window and plays `video` in real time until the window is closed. /// @@ -63,32 +65,33 @@ pub fn play<'a, 'media: 'a, TVideo: Video + Sync>( let event_loop = EventLoop::::with_user_event().build()?; #[cfg(feature = "audio")] - let audio = options - .audio - .then(|| { - crate::audio::AudioOutput::new() + let mut audio = None; + let audio_sample_rate = || { + #[cfg(feature = "audio")] + if options.audio { + audio = crate::audio::AudioOutput::new() .inspect_err(|err| eprintln!("fframes player: audio disabled ({err})")) - .ok() - }) - .flatten(); - #[cfg(feature = "audio")] - let sample_rate = audio.as_ref().map_or(44100, |audio| audio.sample_rate); - #[cfg(not(feature = "audio"))] - let sample_rate = 44100; - + .ok(); + if let Some(output) = &audio { + return output.sample_rate; + } + } + 44100 + }; let scenes = video.define_scenes(); let FFramesRendererRuntime { time_base, timeline, mut font_source, - } = FFramesRendererRuntime::new( + } = FFramesRendererRuntime::with_audio_sample_rate( TimeBase { fps: TVideo::FPS, - sample_rate, + sample_rate: 44100, }, video, &scenes, options.media, + audio_sample_rate, )?; if options.load_system_fonts { @@ -125,25 +128,45 @@ pub fn play<'a, 'media: 'a, TVideo: Video + Sync>( // The window owns the Skia context, so the trees of `render_svgr` render on the CPU. let svgr_images = fframes::SvgrImages::new(Some(Box::new(fframes::cpu::CpuSvgrRasterizer))); + let window_size = initial_window_size::(options); + let aspect = TVideo::WIDTH as f32 / TVideo::HEIGHT.max(1) as f32; + let tiles = if aspect < 1. { + window_size.height as f32 / 60. + } else { + window_size.width as f32 / (60. * aspect) + }; + let thumbnail_count = (tiles.ceil() as usize) + .clamp(1, 32) + .min(timeline.duration_in_frames); let scheduler = Scheduler::new( timeline.duration_in_frames, - options.prefetch_frames, + options.lookahead(), options.looping, + thumbnail_count, + options.software_decode, ); let proxy = Mutex::new(event_loop.create_proxy()); - let notify_frame_ready = || { + let notify_frame_ready = |event| { let _ = proxy .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) - .send_event(FrameReady); + .send_event(event); }; let mut app = App::new( AppConfig { title: options.title, - window_size: initial_window_size::(options), + window_size, + fit_portrait_to_screen: aspect < 1. && options.window_size.is_none(), backend: options.backend, vsync: options.vsync, + studio: studio::Studio::new( + time_base, + &timeline, + options.media, + thumbnail_count, + aspect, + ), fps: TVideo::FPS, duration_in_frames: timeline.duration_in_frames, background: TVideo::BACKGROUND_COLOR, @@ -156,24 +179,17 @@ pub fn play<'a, 'media: 'a, TVideo: Video + Sync>( audio.as_ref(), ); + let run_worker = |kind| { + let db = font_source.as_db_ref(); + svgr_images.enter(&usvg_options, db, || { + scheduler.run_worker(video, &ctx, &usvg_options, db, kind, ¬ify_frame_ready); + }); + }; let result = std::thread::scope(|scope| { for _ in 0..options.render_threads() { - scope.spawn(|| { - let db = font_source.as_db_ref(); - svgr_images.enter(&usvg_options, db, || { - scheduler.run_worker( - video, - &ctx, - &usvg_options, - db, - // Decoders hand out their current frame buffer, which is recycled by - // the next decode, so they must not be shared between threads. - VideoDecodersWorker::new(options.prefetch_frames.max(1) * 2), - ¬ify_frame_ready, - ); - }); - }); + scope.spawn(|| run_worker(WorkerKind::Playback)); } + scope.spawn(|| run_worker(WorkerKind::Thumbnails)); #[cfg(feature = "audio")] if let (Some(audio), Some(audio_map)) = (audio.as_ref(), timeline.audio_map.as_ref()) { @@ -253,6 +269,7 @@ pub fn cli_preview<'a, 'media: 'a, TVideo: Video + Sync>( looping: request.looping, start_frame: request.start_frame, audio: request.audio, + software_decode: request.software_decode, audio_mix: options.audio_mix, ..Default::default() }, @@ -266,6 +283,15 @@ fn initial_window_size(options: &PlayerOptions) -> LogicalSize { pub default_font: &'a str, /// Window title, defaults to "fframes". pub title: &'a str, - /// Initial window size in logical pixels. Defaults to the video size, scaled down to - /// fit 1280x720. + /// Initial window size in logical pixels. Landscape defaults fit within 1280×720. + /// Portrait defaults fit up to 960 pixels tall with a timeline on the left, subject + /// to the size of the display. pub window_size: Option<(u32, u32)>, pub backend: PlayerBackend, + /// Decode source videos in software even when a hardware decoder is available. + pub software_decode: bool, /// Wait for the display refresh to show a frame. With `false` a frame is shown as soon /// as it is drawn, up to the frame rate of the video: the picture can tear and the GPU /// stays busy. Only the GPU backends wait for the display. @@ -40,10 +43,14 @@ pub struct PlayerOptions<'a, 'media> { pub start_frame: usize, /// Play the video's audio map on the default output device. pub audio: bool, - /// Threads building frame trees (`Video::render_frame`) in parallel. - /// Defaults to half of the available cores, at most 8. + /// Maximum number of threads building frame trees (`Video::render_frame`) in parallel. + /// Defaults to half the available cores: up to four workers for video sources, which + /// share one decoder per file, and up to eight otherwise. Source video workers are + /// limited to the default buffer size plus one live frame. pub render_threads: Option, - /// How many frames ahead of the playhead may be prepared. + /// Maximum number of future frame trees to prepare. The default (`usize::MAX`) + /// buffers up to four trees. Zero prepares only the currently due frame. + /// Skia draws each tree directly into the window when it is due. pub prefetch_frames: usize, /// Master bus of the audio mix, see `RenderOptions::audio_mix`. pub audio_mix: fframes::AudioMixOptions, @@ -58,22 +65,102 @@ impl Default for PlayerOptions<'_, '_> { title: "fframes", window_size: None, backend: PlayerBackend::Auto, + software_decode: false, vsync: true, autoplay: true, looping: true, start_frame: 0, audio: true, render_threads: None, - prefetch_frames: 12, + prefetch_frames: usize::MAX, audio_mix: fframes::AudioMixOptions::default(), } } } impl PlayerOptions<'_, '_> { + pub(crate) fn lookahead(&self) -> usize { + if self.prefetch_frames == usize::MAX { + crate::scheduler::MAX_STAGED_TREES + } else { + self.prefetch_frames + } + } + pub(crate) fn render_threads(&self) -> usize { + // Source decoders are shared and keep recent frames, so a few workers can + // prepare trees in parallel. More workers than staged trees would idle. + if self.media.is_some_and(MediaProvider::has_video_sources) { + return self + .render_threads + .unwrap_or_else(|| (cores() / 2).min(crate::scheduler::MAX_STAGED_TREES)) + .clamp(1, crate::scheduler::MAX_STAGED_TREES + 1); + } self.render_threads - .unwrap_or_else(|| (fframes::get_thread_count() / 2).clamp(1, 8)) + .unwrap_or_else(|| (cores() / 2).clamp(1, 8)) .max(1) } } + +/// The preview owns its workers. Querying Rayon here starts a second idle pool. +fn cores() -> usize { + std::env::var("FFRAMES_NUM_THREADS") + .ok() + .and_then(|value| value.parse::().ok()) + .or_else(|| { + std::env::var("RAYON_NUM_THREADS") + .ok() + .and_then(|value| value.parse::().ok()) + .filter(|&threads| threads > 0) + }) + .unwrap_or_else(|| std::thread::available_parallelism().map_or(1, usize::from)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn lookahead_defaults_to_a_small_tree_buffer() { + let mut options = PlayerOptions::default(); + assert_eq!(options.lookahead(), 4); + options.prefetch_frames = 8; + assert_eq!(options.lookahead(), 8); + options.prefetch_frames = 0; + assert_eq!(options.lookahead(), 0); + } + + #[test] + fn video_sources_default_to_a_few_workers() { + let media = fframes::DynamicMediaProvider::default().with_video( + "clip.mp4", + fframes::media::VideoMedia { + path: "clip.mp4".into(), + metadata: None, + }, + ); + let media = fframes::CombinedMediaProvider::from([&() as &dyn MediaProvider, &media]); + let mut options = PlayerOptions { + media: Some(&media), + ..Default::default() + }; + assert_eq!( + options.render_threads(), + (fframes::get_thread_count() / 2).clamp(1, 4) + ); + options.render_threads = Some(4); + assert_eq!(options.render_threads(), 4); + options.render_threads = Some(16); + assert_eq!(options.render_threads(), 5); + options.render_threads = Some(0); + assert_eq!(options.render_threads(), 1); + options.media = None; + options.render_threads = Some(16); + assert_eq!(options.render_threads(), 16); + options.render_threads = None; + assert_eq!( + options.render_threads(), + (fframes::get_thread_count() / 2).clamp(1, 8) + ); + } +} diff --git a/fframes-native-player/src/presenter/metal.rs b/fframes-native-player/src/presenter/metal.rs index 66e4dcaf..c97a1589 100644 --- a/fframes-native-player/src/presenter/metal.rs +++ b/fframes-native-player/src/presenter/metal.rs @@ -1,4 +1,5 @@ use std::ffi::c_void; +use std::time::{Duration, Instant}; use core_graphics_types::geometry::CGSize; use fframes_skia_renderer::metal::SkiaMetalCtx; @@ -17,9 +18,12 @@ use winit::window::Window; use super::WindowTarget; use crate::error::{PlayerError, PlayerResult}; +const GPU_CACHE_IDLE: Duration = Duration::from_secs(1); + /// Skia draws into the drawables of a `CAMetalLayer` of the window. pub(super) struct MetalTarget { gpu: DirectContext, + last_cache_cleanup: Instant, layer: MetalLayer, backend: SkiaMetalCtx, } @@ -50,6 +54,7 @@ impl MetalTarget { Ok(Self { gpu, + last_cache_cleanup: Instant::now(), layer, backend, }) @@ -76,6 +81,12 @@ impl WindowTarget for MetalTarget { }; let texture = drawable.texture(); + // Release unlocked Skia resources from earlier scenes while retaining active frames. + if self.last_cache_cleanup.elapsed() >= GPU_CACHE_IDLE { + self.gpu.perform_deferred_cleanup(GPU_CACHE_IDLE, None); + self.last_cache_cleanup = Instant::now(); + } + // SAFETY: the texture lives as long as the drawable, which outlives the surface. let texture_info = unsafe { mtl::TextureInfo::new(texture.as_ptr() as mtl::Handle) }; let render_target = backend_render_targets::make_mtl( @@ -93,8 +104,9 @@ impl WindowTarget for MetalTarget { .ok_or_else(|| PlayerError::Skia("can not draw into the drawable".to_owned()))?; draw(surface.canvas()); - self.gpu - .flush_and_submit_surface(&mut surface, SyncCpu::Yes); + // The flush copies image pixels into transfer buffers, so the CPU does not + // wait for the GPU. The layer limits the drawables in flight. + self.gpu.flush_and_submit_surface(&mut surface, SyncCpu::No); drop(surface); // Skia submitted to the same queue, the drawable is presented after its work. diff --git a/fframes-native-player/src/presenter/mod.rs b/fframes-native-player/src/presenter/mod.rs index ece7f6c1..6cf45092 100644 --- a/fframes-native-player/src/presenter/mod.rs +++ b/fframes-native-player/src/presenter/mod.rs @@ -18,6 +18,7 @@ use winit::window::Window; use crate::controls::{Controls, ControlsLayout, ControlsState}; use crate::error::PlayerResult; use crate::options::PlayerBackend; +use crate::studio::{Studio, StudioLayout, StudioScroll}; /// The images of the window that Skia draws into. trait WindowTarget { @@ -32,15 +33,16 @@ trait WindowTarget { /// The window has a new size in physical pixels, neither is zero. fn resize(&mut self, width: u32, height: u32) -> PlayerResult<()>; - /// Draws the next image of the window with `draw` and shows it. Returns once the GPU is - /// done: video frame images point into the pixel buffers of the tree, which is dropped - /// as soon as the next frame replaces it. + /// Draws the next image of the window with `draw` and shows it. The GPU backends + /// return before the GPU is done: Skia copies video frame pixels into its staging + /// memory during the flush, so the tree can be dropped once this returns. fn present(&mut self, draw: &mut dyn FnMut(&Canvas)) -> PlayerResult<()>; } // Fields drop in declaration order: the cached Skia objects before the target that owns // the context and the device. pub(crate) struct Presenter { + pub studio_layout: Option, render_cache: RenderCache, target: Box, window: Rc, @@ -49,17 +51,19 @@ pub(crate) struct Presenter { layout: ControlsLayout, /// The longest time label, sizes the time slots of the control bar. duration_label: String, + audio_rows: usize, + video_aspect: f32, } impl Presenter { pub(crate) fn new( window: Rc, - backend: PlayerBackend, - vsync: bool, - background: fframes::Color, - duration_label: String, + config: &crate::app::AppConfig<'_>, ) -> PlayerResult { - let target = create_target(&window, backend, vsync)?; + let target = create_target(&window, config.backend, config.vsync)?; + let duration_label = crate::controls::format_time(config.duration_in_frames, config.fps); + let background = config.background; + let model = &config.studio; let controls = Controls::new(); let size = window.inner_size(); let layout = ControlsLayout::new( @@ -78,6 +82,9 @@ impl Presenter { controls, layout, duration_label, + studio_layout: None, + audio_rows: model.audio_rows(), + video_aspect: model.video_aspect, }; presenter.resize()?; @@ -102,6 +109,13 @@ impl Presenter { } self.target.resize(size.width, size.height)?; + let logical = size.to_logical::(self.window.scale_factor()); + self.studio_layout = StudioLayout::new( + logical.width, + logical.height, + self.audio_rows, + self.video_aspect, + ); self.layout = ControlsLayout::new( &self.controls, &self.duration_label, @@ -113,50 +127,108 @@ impl Presenter { Ok(()) } - /// Renders `tree` letterboxed into the window with the controls on top and presents it. + /// Draws the current frame and overlays the controls before presenting the window. pub(crate) fn present( &mut self, - tree: Option<&usvgr::Tree>, + frame: Option<&usvgr::Tree>, controls: &ControlsState, + model: &Studio, + scroll: &StudioScroll, + studio_open: bool, ) -> PlayerResult<()> { + let pixel_scale = self.window.scale_factor() as f32; let Self { render_cache, target, background, controls: bar, layout, + studio_layout, .. } = self; target.present(&mut |canvas| { - let size = canvas.base_layer_size(); - let (width, height) = (size.width as f32, size.height as f32); - canvas.clear(Color::BLACK); - - if let Some(tree) = tree { - let video = tree.size(); - let scale = (width / video.width()).min(height / video.height()); - let offset_x = (width - video.width() * scale) / 2.; - let offset_y = (height - video.height() * scale) / 2.; + let view = studio_layout.as_ref().filter(|_| studio_open); + canvas.clear(matte(view)); + if let Some(tree) = frame { + draw_video(canvas, tree, view, pixel_scale, *background, render_cache); + } + let show_bar = view.is_none(); + if let Some(ui) = studio_layout.as_mut() { canvas.save(); - canvas.translate((offset_x, offset_y)); - canvas.scale((scale, scale)); - canvas.clip_rect( - Rect::from_wh(video.width(), video.height()), - None, - Some(true), - ); - canvas.draw_color(*background, None); - fframes_skia_renderer::render::render_tree(tree, canvas, render_cache); + canvas.scale((pixel_scale, pixel_scale)); + if studio_open { + ui.draw(canvas, bar, controls, model, scroll); + } else if controls.visible { + ui.draw_restore( + canvas, + bar, + controls.hovered == Some(crate::controls::Control::Focus), + ); + } canvas.restore(); } - - bar.draw(canvas, layout, controls); + if show_bar { + bar.draw(canvas, layout, controls); + } }) } } +fn matte(view: Option<&StudioLayout>) -> Color { + if view.is_some() { + Color::from_rgb(14, 14, 14) + } else { + Color::BLACK + } +} + +fn draw_video( + canvas: &Canvas, + tree: &usvgr::Tree, + view: Option<&StudioLayout>, + pixel_scale: f32, + background: Color, + cache: &mut RenderCache, +) { + let size = canvas.base_layer_size(); + let viewport = view.map_or_else( + || Rect::from_iwh(size.width, size.height), + |ui| { + let stage = ui.stage; + Rect::from_ltrb( + stage.left * pixel_scale, + stage.top * pixel_scale, + stage.right * pixel_scale, + stage.bottom * pixel_scale, + ) + }, + ); + let video = tree.size(); + let scale = (viewport.width() / video.width()).min(viewport.height() / video.height()); + let width = video.width() * scale; + let height = video.height() * scale; + let x = viewport.center_x() - width / 2.; + let y = if view.is_some_and(|ui| ui.portrait) { + viewport.top + } else { + viewport.center_y() - height / 2. + }; + + canvas.save(); + canvas.translate((x, y)); + canvas.scale((scale, scale)); + canvas.clip_rect( + Rect::from_wh(video.width(), video.height()), + None, + Some(true), + ); + canvas.draw_color(background, None); + fframes_skia_renderer::render::render_tree(tree, canvas, cache); + canvas.restore(); +} + fn create_target( window: &Rc, backend: PlayerBackend, @@ -165,8 +237,8 @@ fn create_target( match backend { PlayerBackend::Auto => { #[cfg(feature = "metal")] - match create_target(window, PlayerBackend::Metal, vsync) { - Ok(target) => return Ok(target), + match metal::MetalTarget::new(window, vsync) { + Ok(target) => return Ok(Box::new(target)), Err(err) => eprintln!("fframes player: metal is not available ({err})"), } diff --git a/fframes-native-player/src/presenter/vulkan.rs b/fframes-native-player/src/presenter/vulkan.rs index 73740b6a..182078c8 100644 --- a/fframes-native-player/src/presenter/vulkan.rs +++ b/fframes-native-player/src/presenter/vulkan.rs @@ -1,11 +1,12 @@ use std::ffi::CStr; use std::mem::ManuallyDrop; use std::rc::Rc; +use std::time::{Duration, Instant}; use fframes_skia_renderer::render::RenderCache; use fframes_skia_renderer::skia_safe::gpu::ganesh::vk::backend_render_targets; use fframes_skia_renderer::skia_safe::gpu::{ - self, DirectContext, FlushInfo, SurfaceOrigin, SyncCpu, + self, DirectContext, FlushInfo, SemaphoresSubmitted, SurfaceOrigin, SyncCpu, backend_semaphores, }; use fframes_skia_renderer::skia_safe::{Canvas, ColorType, Surface}; use fframes_skia_renderer::vulkan::ash::{ @@ -27,11 +28,14 @@ type SurfaceLayer = raw_window_metal::Layer; #[cfg(not(target_os = "macos"))] type SurfaceLayer = (); +const GPU_CACHE_IDLE: Duration = Duration::from_secs(1); + /// Skia draws into the images of a swapchain of the window. pub(super) struct VulkanTarget { swapchain: Option, // Dropped in `drop`, after the swapchain images it draws into. gpu: ManuallyDrop, + last_cache_cleanup: Instant, queue: vk::Queue, queue_lock: QueueLock, surface: vk::SurfaceKHR, @@ -51,15 +55,23 @@ struct Swapchain { handle: vk::SwapchainKHR, /// A Skia surface on every image of the swapchain. images: Vec, + /// One per image, signaled once Skia's work on the image is done. The present of + /// the image waits for it, so the CPU does not have to. + rendered: Vec, /// The swapchain no longer matches the window and is recreated before the next frame. outdated: bool, } impl Swapchain { - fn destroy(self, swapchain_fn: &ash::khr::swapchain::Device) { + fn destroy(self, device: &ash::Device, swapchain_fn: &ash::khr::swapchain::Device) { // Skia's views of the images go first. drop(self.images); - unsafe { swapchain_fn.destroy_swapchain(self.handle, None) }; + unsafe { + for semaphore in self.rendered { + device.destroy_semaphore(semaphore, None); + } + swapchain_fn.destroy_swapchain(self.handle, None); + } } } @@ -97,6 +109,7 @@ impl VulkanTarget { let mut target = Self { swapchain: None, gpu: ManuallyDrop::new(gpu), + last_cache_cleanup: Instant::now(), queue, queue_lock, surface, @@ -200,7 +213,7 @@ impl VulkanTarget { }; if extent.width == 0 || extent.height == 0 { if let Some(old) = old { - old.destroy(&self.swapchain_fn); + old.destroy(device, &self.swapchain_fn); } return Ok(()); } @@ -264,7 +277,7 @@ impl VulkanTarget { // Retired by the new swapchain, or of no use if it could not be created. if let Some(old) = old { - old.destroy(&self.swapchain_fn); + old.destroy(device, &self.swapchain_fn); } let handle = handle.map_err(vulkan_error("can not create the swapchain"))?; @@ -311,20 +324,29 @@ impl VulkanTarget { .collect::>>() }); - match images { - Ok(images) => { - self.swapchain = Some(Swapchain { - handle, - images, - outdated: false, - }); - Ok(()) - } + let mut swapchain = match images { + Ok(images) => Swapchain { + handle, + images, + rendered: Vec::new(), + outdated: false, + }, Err(err) => { unsafe { self.swapchain_fn.destroy_swapchain(handle, None) }; - Err(err) + return Err(err); + } + }; + for _ in 0..swapchain.images.len() { + match unsafe { device.create_semaphore(&vk::SemaphoreCreateInfo::default(), None) } { + Ok(semaphore) => swapchain.rendered.push(semaphore), + Err(err) => { + swapchain.destroy(device, &self.swapchain_fn); + return Err(vulkan_error("can not create a semaphore")(err)); + } } } + self.swapchain = Some(swapchain); + Ok(()) } /// The index of the next swapchain image, once it can be drawn into. `None` while the @@ -401,10 +423,17 @@ impl WindowTarget for VulkanTarget { return Ok(()); }; let surface = &mut swapchain.images[index as usize]; + let rendered = swapchain.rendered[index as usize]; draw(surface.canvas()); let _queue = lock_queue(Some(&self.queue_lock)); + // Release unlocked Skia resources from earlier scenes while retaining active frames. + if self.last_cache_cleanup.elapsed() >= GPU_CACHE_IDLE { + self.gpu.perform_deferred_cleanup(GPU_CACHE_IDLE, None); + self.last_cache_cleanup = Instant::now(); + } + // Ask Skia for the layout the presentation engine reads. `BackendSurfaceAccess::Present` // changes the layout only when Skia's context lists `VK_KHR_swapchain`, and the // contexts of `SkiaVulkanCtx` do not. @@ -412,17 +441,30 @@ impl WindowTarget for VulkanTarget { gpu::vk::ImageLayout::PRESENT_SRC_KHR, self.backend.queue_family_index(), ); - self.gpu.flush_surface_with_texture_state( - surface, - &FlushInfo::default(), - Some(&present_layout), - ); - self.gpu.submit(SyncCpu::Yes); + // SAFETY: the semaphore lives as long as the swapchain, which is destroyed after + // the device is idle. + let mut signal = [unsafe { backend_semaphores::make_vk(rendered.as_raw() as _) }]; + let mut flush_info = FlushInfo::default(); + // SAFETY: `signal` outlives the flush. + unsafe { flush_info.set_signal_semaphores(&mut signal) }; + let signaled = + self.gpu + .flush_surface_with_texture_state(surface, &flush_info, Some(&present_layout)); + // The flush copied image pixels into staging buffers, so the CPU does not wait for + // the GPU: the present does. Without a signal (nothing was drawn) it waits here. + let wait_semaphores: &[vk::Semaphore] = if signaled == SemaphoresSubmitted::Yes { + self.gpu.submit(SyncCpu::No); + &[rendered] + } else { + self.gpu.submit(SyncCpu::Yes); + &[] + }; let presented = unsafe { self.swapchain_fn.queue_present( self.queue, &vk::PresentInfoKHR::default() + .wait_semaphores(wait_semaphores) .swapchains(&[swapchain.handle]) .image_indices(&[index]), ) @@ -448,7 +490,7 @@ impl Drop for VulkanTarget { } if let Some(swapchain) = self.swapchain.take() { - swapchain.destroy(&self.swapchain_fn); + swapchain.destroy(device, &self.swapchain_fn); } unsafe { diff --git a/fframes-native-player/src/scheduler.rs b/fframes-native-player/src/scheduler.rs index 63aa0bb1..e27a2c2e 100644 --- a/fframes-native-player/src/scheduler.rs +++ b/fframes-native-player/src/scheduler.rs @@ -7,13 +7,19 @@ use std::collections::BTreeMap; use std::sync::{Condvar, Mutex, MutexGuard}; +use crate::{app::FrameReady, studio::thumbnail}; use fframes::{FFramesContext, TextCache, Video, VideoDecodersWorker, usvgr}; +pub(crate) const MAX_STAGED_TREES: usize = 4; + pub(crate) struct Scheduler { state: Mutex, workers_wakeup: Condvar, duration: u64, lookahead: u64, + thumbnails: usize, + playback_decoders: VideoDecodersWorker, + software_decode: bool, } struct State { @@ -26,18 +32,38 @@ struct State { /// Last position handed to the player, anything at or before it is stale. displayed: Option, looping: bool, + /// A timeline drag is in progress: prepare only the frame under the pointer. + scrubbing: bool, ready: BTreeMap, + in_flight: usize, shutdown: bool, + next_thumbnail: usize, } struct Job { - position: u64, - epoch: u64, frame: usize, + kind: JobKind, +} + +enum JobKind { + Playback { position: u64, epoch: u64 }, + Thumbnail(usize), +} + +#[derive(Clone, Copy)] +pub(crate) enum WorkerKind { + Playback, + Thumbnails, } impl Scheduler { - pub(crate) fn new(duration_in_frames: usize, lookahead: usize, looping: bool) -> Self { + pub(crate) fn new( + duration_in_frames: usize, + lookahead: usize, + looping: bool, + thumbnails: usize, + software_decode: bool, + ) -> Self { Self { state: Mutex::new(State { epoch: 0, @@ -45,12 +71,18 @@ impl Scheduler { playhead: 0, displayed: None, looping, + scrubbing: false, ready: BTreeMap::new(), + in_flight: 0, shutdown: false, + next_thumbnail: 0, }), workers_wakeup: Condvar::new(), duration: duration_in_frames.max(1) as u64, - lookahead: lookahead.max(1) as u64, + lookahead: lookahead.min(duration_in_frames) as u64, + thumbnails, + playback_decoders: VideoDecodersWorker::for_preview(software_decode), + software_decode, } } @@ -77,7 +109,39 @@ impl Scheduler { state.playhead = position; state.displayed = None; state.looping = looping; - state.ready.clear(); + // Trees hold video frames. Free them after the workers can continue. + let stale = std::mem::take(&mut state.ready); + drop(state); + self.workers_wakeup.notify_all(); + drop(stale); + } + + /// A forward seek within the finished buffer can show its target immediately. + pub(crate) fn seek_buffered(&self, position: u64, looping: bool) -> bool { + let mut state = self.lock(); + if state.looping != looping + || position < state.playhead + || !state.ready.contains_key(&position) + { + return false; + } + state.playhead = position; + let kept = state.ready.split_off(&position); + let stale = std::mem::replace(&mut state.ready, kept); + drop(state); + self.workers_wakeup.notify_all(); + drop(stale); + true + } + + /// While a drag is in progress only the frame under the pointer is prepared, so a + /// scrub never waits for lookahead frames the next pointer position discards. + pub(crate) fn set_scrubbing(&self, scrubbing: bool) { + let mut state = self.lock(); + if state.scrubbing == scrubbing { + return; + } + state.scrubbing = scrubbing; drop(state); self.workers_wakeup.notify_all(); } @@ -87,17 +151,19 @@ impl Scheduler { /// frame rate instead of freezing. pub(crate) fn take(&self, playhead: u64) -> Option<(u64, usvgr::Tree)> { let mut state = self.lock(); + let advanced = state.playhead != playhead; state.playhead = playhead; - let newer = state.ready.split_off(&(playhead + 1)); - let due = std::mem::replace(&mut state.ready, newer); - let taken = due.into_iter().next_back(); + let (taken, skipped) = take_due(&mut state.ready, playhead); if let Some((position, _)) = &taken { state.displayed = Some(*position); } drop(state); - self.workers_wakeup.notify_all(); + if advanced || taken.is_some() { + self.workers_wakeup.notify_all(); + } + drop(skipped); taken } @@ -113,62 +179,127 @@ impl Scheduler { return None; } - // Skip frames the playhead already passed: this is where frames get dropped - // when rendering can not keep up with real time. - if state.next < state.playhead { - state.next = state.playhead; + if let Some(job) = self.claim_playback(&mut state) { + return Some(job); } - let within_lookahead = state.next < state.playhead + self.lookahead; - let within_video = state.looping || state.next < self.duration; - if within_lookahead && within_video { - let position = state.next; - state.next += 1; + state = self + .workers_wakeup + .wait(state) + .unwrap_or_else(std::sync::PoisonError::into_inner); + } + } - return Some(Job { - position, - epoch: state.epoch, - frame: self.frame_of(position, state.looping), - }); - } + /// The next position a playback worker can prepare, if any. Prepared trees and + /// in-flight work are bounded. A new live position bypasses that bound when the + /// clock jumped ahead of all assigned work. + fn claimable_position(&self, state: &State) -> Option { + let lookahead = if state.scrubbing { 0 } else { self.lookahead }; + let position = state.next.max(state.playhead); + let staged = state.ready.len() + state.in_flight; + if position > state.playhead.saturating_add(lookahead) + || (!state.looping && position >= self.duration) + || (position > state.playhead && staged >= lookahead.max(1) as usize) + { + return None; + } + Some(position) + } + + fn claim_playback(&self, state: &mut State) -> Option { + let position = self.claimable_position(state)?; + state.next = position + 1; + state.in_flight += 1; + Some(Job { + frame: self.frame_of(position, state.looping), + kind: JobKind::Playback { + position, + epoch: state.epoch, + }, + }) + } + // The filmstrip seeks across the video with its own decoder. It waits until the + // first frame is shown, no drag is in progress and the playback workers have nothing + // left to prepare, so thumbnails never compete with playback or a scrub. + fn next_thumbnail_job(&self) -> Option { + let mut state = self.lock(); + while !state.shutdown + && (state.displayed.is_none() + || state.scrubbing + || self.claimable_position(&state).is_some()) + { state = self .workers_wakeup .wait(state) .unwrap_or_else(std::sync::PoisonError::into_inner); } + if state.shutdown || state.next_thumbnail >= self.thumbnails { + return None; + } + let slot = state.next_thumbnail; + state.next_thumbnail += 1; + let frame = ((2 * slot + 1) as u128 * u128::from(self.duration) + / (2 * self.thumbnails) as u128) as usize; + Some(Job { + frame, + kind: JobKind::Thumbnail(slot), + }) } - fn complete(&self, job: &Job, tree: usvgr::Tree) -> bool { + #[cfg(test)] + pub(crate) fn is_scrubbing(&self) -> bool { + self.lock().scrubbing + } + + pub(crate) fn is_current_seek(&self, position: u64, epoch: u64) -> bool { + let state = self.lock(); + state.playhead == position && state.epoch == epoch + } + + fn complete(&self, position: u64, epoch: u64, tree: Option) -> bool { let mut state = self.lock(); - let is_stale = state.epoch != job.epoch + state.in_flight -= 1; + let is_stale = state.epoch != epoch || state .displayed - .is_some_and(|displayed| job.position <= displayed); + .is_some_and(|displayed| position <= displayed); - if !is_stale { - state.ready.insert(job.position, tree); + let tree = tree.filter(|_| !is_stale); + let inserted = tree.is_some(); + if let Some(tree) = tree { + state.ready.insert(position, tree); } - - !is_stale + drop(state); + // A full buffer lets the filmstrip worker start, a freed slot the playback workers. + self.workers_wakeup.notify_all(); + inserted } - /// Runs on a worker thread until `shutdown` is called. `on_frame_ready` is invoked - /// after every frame that was added to the queue (used to wake the event loop). pub(crate) fn run_worker<'a, 'media: 'a, TVideo: Video + Sync>( &self, video: &'a TVideo, ctx: &FFramesContext<'a, 'media>, usvg_options: &usvgr::Options<'_>, font_db: &usvgr::fontdb::Database, - video_decoders: VideoDecodersWorker, - on_frame_ready: &(dyn Fn() + Sync), + kind: WorkerKind, + on_frame_ready: &(dyn Fn(FrameReady) + Sync), ) { - // Text caches are `Rc` based and therefore owned by the worker. - let text_cache = TextCache::new(100); - let mut converter_cache = usvgr::Cache::new_with_text_cache(100); + // Text caches are `Rc` based and therefore owned by the worker. The entries are + // small. Fewer than the text nodes of one frame makes every frame lay out all + // its text again: 8 entries cost teej-podcast eight times the worker CPU. + const TEXT_CACHE_ENTRIES: usize = 100; + let text_cache = TextCache::new(TEXT_CACHE_ENTRIES); + let mut converter_cache = usvgr::Cache::new_with_text_cache(TEXT_CACHE_ENTRIES); + let video_decoders = match kind { + WorkerKind::Playback => self.playback_decoders.clone(), + WorkerKind::Thumbnails => VideoDecodersWorker::for_preview(self.software_decode), + }; - while let Some(job) = self.next_job() { + while let Some(job) = match kind { + WorkerKind::Playback => self.next_job(), + WorkerKind::Thumbnails => self.next_thumbnail_job(), + } { let frame = fframes::Frame::__internal_make_for_renderer( job.frame, job.frame, @@ -184,17 +315,204 @@ impl Scheduler { Ok(svgr.into_svg_tree(usvg_options, &mut converter_cache, font_db)?) }); - match tree { - Ok(tree) => { - if self.complete(&job, tree) { - on_frame_ready(); + let concurrent_jobs = match kind { + WorkerKind::Playback => self.lock().in_flight, + WorkerKind::Thumbnails => 1, + }; + video_decoders.finish_preview_frame(concurrent_jobs); + match (tree, job.kind) { + (Ok(tree), JobKind::Thumbnail(slot)) => { + if let Some(image) = thumbnail(&tree, TVideo::BACKGROUND_COLOR) { + on_frame_ready(FrameReady::Thumbnail(slot, image)); + } + } + (Ok(tree), JobKind::Playback { position, epoch }) => { + if self.complete(position, epoch, Some(tree)) { + on_frame_ready(FrameReady::Playback); + } + } + (Err(err), kind) => { + eprintln!( + "fframes player: failed to render frame {}: {err}", + job.frame + ); + if let JobKind::Playback { position, epoch } = kind { + self.complete(position, epoch, None); + on_frame_ready(FrameReady::Failed { position, epoch }); } } - Err(err) => eprintln!( - "fframes player: failed to render frame {}: {err}", - job.frame - ), } } } } + +/// Removes the frames at or before `playhead`. Returns the newest of them and the ones +/// it skipped, so the caller can drop those after releasing the lock. +fn take_due( + ready: &mut BTreeMap, + playhead: u64, +) -> (Option<(u64, T)>, BTreeMap) { + let future = ready.split_off(&playhead.saturating_add(1)); + let mut due = std::mem::replace(ready, future); + let latest = due.pop_last(); + (latest, due) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn tree() -> usvgr::Tree { + usvgr::Tree::from_str( + r#""#, + &usvgr::Options::default(), + &usvgr::fontdb::Database::new(), + ) + .unwrap() + } + + #[test] + fn taking_out_of_order_frames_preserves_future_frames_and_rejects_stale_jobs() { + let scheduler = Scheduler::new(100, 4, false, 2, false); + for index in 0..=3 { + assert_eq!(scheduler.next_job().unwrap().frame, index); + } + for index in [2, 1] { + assert!(scheduler.complete(index, 0, Some(tree()))); + } + assert_eq!(scheduler.take(1).unwrap().0, 1); + assert!(scheduler.take(1).is_none()); + assert!(!scheduler.complete(0, 0, Some(tree()))); + assert_eq!(scheduler.take(2).unwrap().0, 2); + scheduler.seek(50, false); + assert!(!scheduler.complete(3, 0, Some(tree()))); + assert_eq!(scheduler.next_job().unwrap().frame, 50); + assert!(scheduler.complete(50, 1, Some(tree()))); + assert_eq!(scheduler.take(50).unwrap().0, 50); + scheduler.shutdown(); + assert!(scheduler.next_job().is_none()); + } + + #[test] + fn prepared_frames_are_bounded_and_forward_seeks_reuse_trees() { + let scheduler = Scheduler::new(100, 4, false, 0, false); + for p in 0..4 { + assert_eq!(scheduler.next_job().unwrap().frame, p); + assert!(scheduler.complete(p as u64, 0, Some(tree()))); + } + assert!(scheduler.claim_playback(&mut scheduler.lock()).is_none()); + assert_eq!(scheduler.take(0).unwrap().0, 0); + assert_eq!(scheduler.next_job().unwrap().frame, 4); + scheduler.complete(4, 0, Some(tree())); + assert!(scheduler.seek_buffered(3, false)); + assert_eq!(scheduler.take(3).unwrap().0, 3); + assert!(!scheduler.seek_buffered(4, true)); + assert!(!scheduler.seek_buffered(0, false)); + assert_eq!(scheduler.take(4).unwrap().0, 4); + } + + #[test] + fn live_frames_take_priority_over_queued_trees() { + let scheduler = Scheduler::new(100, 4, false, 0, false); + for p in 0..4 { + assert_eq!(scheduler.next_job().unwrap().frame as u64, p); + scheduler.complete(p, 0, Some(tree())); + } + assert!(scheduler.claim_playback(&mut scheduler.lock()).is_none()); + assert_eq!(scheduler.take(30).unwrap().0, 3); + assert_eq!(scheduler.next_job().unwrap().frame, 30); + scheduler.complete(30, 0, Some(tree())); + assert_eq!(scheduler.take(30).unwrap().0, 30); + } + + #[test] + fn workers_bound_in_flight_frames_and_release_failed_jobs() { + let scheduler = Scheduler::new(100, 4, false, 0, false); + for p in 0..4 { + assert_eq!(scheduler.next_job().unwrap().frame, p); + } + assert!(scheduler.claim_playback(&mut scheduler.lock()).is_none()); + scheduler.complete(2, 0, None); + assert_eq!(scheduler.next_job().unwrap().frame, 4); + scheduler.take(30); + assert_eq!(scheduler.next_job().unwrap().frame, 30); + scheduler.seek(60, false); + assert!(!scheduler.complete(1, 0, Some(tree()))); + assert_eq!(scheduler.next_job().unwrap().frame, 60); + } + + #[test] + fn zero_prefetch_only_prepares_the_live_frame() { + let scheduler = Scheduler::new(3, 0, false, 0, false); + for p in 0..3 { + scheduler.take(p); + assert_eq!(scheduler.next_job().unwrap().frame as u64, p); + assert!(scheduler.claim_playback(&mut scheduler.lock()).is_none()); + assert!(scheduler.complete(p, 0, Some(tree()))); + assert_eq!(scheduler.take(p).unwrap().0, p); + } + assert!(scheduler.claim_playback(&mut scheduler.lock()).is_none()); + } + + #[test] + fn looping_buffer_uses_monotonic_positions_across_the_end() { + let scheduler = Scheduler::new(3, 2, true, 0, false); + scheduler.seek(2, true); + for p in 2..4 { + let job = scheduler.next_job().unwrap(); + assert_eq!(job.frame, p % 3); + assert!(scheduler.complete(p as u64, 1, Some(tree()))); + } + assert_eq!(scheduler.take(2).unwrap().0, 2); + assert!(scheduler.seek_buffered(3, true)); + assert_eq!(scheduler.take(3).unwrap().0, 3); + assert_eq!(scheduler.next_job().unwrap().frame, 1); + } + + #[test] + fn scrubbing_prepares_only_the_frame_under_the_pointer() { + let scheduler = Scheduler::new(100, 4, false, 0, false); + scheduler.set_scrubbing(true); + scheduler.seek(10, false); + assert_eq!(scheduler.next_job().unwrap().frame, 10); + assert!(scheduler.claim_playback(&mut scheduler.lock()).is_none()); + assert!(scheduler.complete(10, 1, Some(tree()))); + assert_eq!(scheduler.take(10).unwrap().0, 10); + assert!(scheduler.claim_playback(&mut scheduler.lock()).is_none()); + scheduler.set_scrubbing(false); + assert_eq!(scheduler.next_job().unwrap().frame, 11); + } + + #[test] + fn thumbnails_wait_for_a_full_playback_buffer_and_stop_when_finished() { + let scheduler = Scheduler::new(100, 3, false, 2, false); + assert_eq!(scheduler.next_job().unwrap().frame, 0); + scheduler.complete(0, 0, Some(tree())); + scheduler.take(0).unwrap(); + let (sender, thumbnails) = std::sync::mpsc::channel(); + let scheduler = &scheduler; + std::thread::scope(|scope| { + // The channel closes when the filmstrip worker runs out of jobs. + scope.spawn(move || { + while let Some(job) = scheduler.next_thumbnail_job() { + sender.send(job.frame).unwrap(); + } + }); + // Playback jobs come first while the buffer has room. + for p in 1..=3 { + assert_eq!(scheduler.next_job().unwrap().frame, p); + if p < 3 { + assert!(thumbnails.try_recv().is_err()); + } + assert!(scheduler.complete(p as u64, 0, Some(tree()))); + } + assert_eq!(thumbnails.recv().unwrap(), 25); + assert_eq!(thumbnails.recv().unwrap(), 75); + assert!(thumbnails.recv().is_err()); + }); + assert!(scheduler.claim_playback(&mut scheduler.lock()).is_none()); + let stopped = Scheduler::new(100, 3, false, 2, false); + stopped.shutdown(); + assert!(stopped.next_thumbnail_job().is_none()); + } +} diff --git a/fframes-native-player/src/studio.rs b/fframes-native-player/src/studio.rs new file mode 100644 index 00000000..49046f65 --- /dev/null +++ b/fframes-native-player/src/studio.rs @@ -0,0 +1,1180 @@ +//! Scene navigation and the clip timeline for the native preview. All layout and pointer +//! coordinates here are logical pixels, regardless of the window's backing scale. + +use std::ops::Range; + +use fframes::{ + AudioData, AudioTimelineSamples, AudioTimelineUnit, MediaProvider, ResolvedRenderingTimeline, + TimeBase, TimelineIndex, media::PreloadedAudioData, +}; +use fframes_skia_renderer::skia_safe::{ + Canvas, Color, Contains, FilterMode, Font, IRect, Image, ImageInfo, Paint, Path, PathBuilder, + Point, RRect, Rect, RoundOut, SamplingOptions, surfaces, +}; + +use crate::controls::{Control, Controls, ControlsState, draw_tabular, format_time, tabular_width}; + +pub(crate) struct Studio { + pub index: TimelineIndex, + pub video_aspect: f32, + audio: Vec>, + thumbnails: Vec>, + thumbnail_revision: u64, +} + +struct Clip { + name: String, + frames: Range, + waveform: Option, +} + +impl Studio { + pub fn new<'m>( + time: TimeBase, + timeline: &ResolvedRenderingTimeline<'_, AudioTimelineSamples>, + media: Option<&'m dyn MediaProvider<'m>>, + thumbnail_count: usize, + video_aspect: f32, + ) -> Self { + let index = TimelineIndex::new( + time.fps, + timeline.duration_in_frames, + timeline.scenes.as_ref(), + ); + let mut clips = Vec::new(); + if let Some(audio) = &timeline.audio_map { + for track in audio.tracks() { + clips.push(Clip { + name: track.file.clone(), + waveform: match media.and_then(|m| m.resolve_audio(&track.file)) { + Some(AudioData::Preloaded(data)) => { + let seconds = track.range.end.to_seconds(&time) + - track.range.start.to_seconds(&time); + Some(waveform( + data, + f64::from(track.mix.offset), + seconds * f64::from(track.mix.rate), + )) + } + _ => None, + }, + frames: track.range.start.to_frames(&time)..track.range.end.to_frames(&time), + }); + } + } + Self { + index, + video_aspect, + audio: pack_clips(clips), + thumbnails: vec![None; thumbnail_count], + thumbnail_revision: 0, + } + } + + pub fn set_thumbnail(&mut self, slot: usize, image: Image) { + self.thumbnails[slot] = Some(image); + self.thumbnail_revision += 1; + } + + pub fn adjacent_scene(&self, frame: usize, next: bool) -> Option { + let starts = self + .index + .scenes + .iter() + .map(|scene| scene.frames.start) + .filter(|&start| start < self.index.duration_in_frames); + if next { + starts.filter(|&start| start > frame).min() + } else { + starts.filter(|&start| start < frame).max() + } + } + + pub fn audio_rows(&self) -> usize { + self.audio.len() + } +} + +/// Rasterize only the small overview image on the frame worker that owns the decoder. +pub(crate) fn thumbnail(tree: &fframes::usvgr::Tree, bg: fframes::Color) -> Option { + let video = tree.size(); + let scale = (192. / video.width()).min(108. / video.height()); + let size = ( + (video.width() * scale).ceil() as i32, + (video.height() * scale).ceil() as i32, + ); + let mut surface = surfaces::raster_n32_premul(size)?; + let canvas = surface.canvas(); + canvas.clear(Color::from_argb(bg.a, bg.r, bg.g, bg.b)); + canvas.scale((scale, scale)); + fframes_skia_renderer::render::render_tree( + tree, + canvas, + &mut fframes_skia_renderer::render::RenderCache::new(), + ); + Some(surface.image_snapshot()) +} + +// A source envelope in unit coordinates, computed once per clip. Include both channels +// and keep silence past EOF when a cue extends beyond the source. +fn waveform(data: &PreloadedAudioData<'_>, offset: f64, seconds: f64) -> Path { + const BINS: usize = 512; + let rate = f64::from(data.sample_rate); + let (left, right) = data.channels(); + let mut path = PathBuilder::new(); + path.move_to((0., 1.)); + let sample = |i: usize| ((offset + seconds * i as f64 / BINS as f64) * rate) as usize; + for i in 0..BINS { + let start = sample(i).min(left.len()); + let end = sample(i + 1).min(left.len()); + // Bound startup work even for a multi-hour file. + let step = end.saturating_sub(start).div_ceil(64).max(1); + let peak = (start..end) + .step_by(step) + .map(|j| left[j].abs().max(right.get(j).copied().unwrap_or(0.).abs())) + .fold(0., f32::max) + .min(1.); + path.line_to(((i as f32 + 0.5) / BINS as f32, 1. - peak)); + } + path.line_to((1., 1.)).close(); + path.detach() +} + +// Put simultaneous sounds on separate rows while letting disjoint clips share a row. +fn pack_clips(mut clips: Vec) -> Vec> { + clips.sort_by_key(|clip| clip.frames.start); + let mut lanes: Vec> = Vec::new(); + for clip in clips { + if let Some(lane) = lanes.iter_mut().find(|lane| { + lane.last() + .is_some_and(|last| last.frames.end <= clip.frames.start) + }) { + lane.push(clip); + } else { + lanes.push(vec![clip]); + } + } + lanes +} + +const SURFACE: Color = Color::from_rgb(23, 23, 23); +const CANVAS: Color = Color::from_rgb(14, 14, 14); +const LINE: Color = Color::from_rgb(58, 58, 58); +const TEXT: Color = Color::from_rgb(249, 249, 249); +const QUIET: Color = Color::from_rgb(174, 174, 174); +const ORANGE: Color = Color::from_rgb(251, 106, 34); +// Theme.scenes in the legacy web studio. Orange remains reserved for the playhead. +const SCENE_COLORS: [Color; 6] = [ + Color::from_rgb(2, 133, 255), + Color::from_rgb(4, 184, 76), + Color::from_rgb(146, 79, 247), + Color::from_rgb(255, 102, 173), + Color::from_rgb(255, 195, 0), + Color::from_rgb(250, 66, 62), +]; +const AUDIO: Color = Color::from_rgb(45, 45, 45); +const TRANSPORT: f32 = 50.; +const RULER: f32 = 22.; +const SCENE_HEIGHT: f32 = 24.; +const LANE: f32 = 31.; +const MARGIN: f32 = 16.; +pub(crate) const PORTRAIT_SIDEBAR_WIDTH: f32 = 360.; + +#[derive(Default)] +pub(crate) struct StudioScroll { + pub audio: usize, +} + +pub(crate) struct StudioLayout { + pub stage: Rect, + timeline: Rect, + restore: Rect, + transport: Rect, + pub portrait: bool, + audio_slots: usize, + film_height: f32, + timeline_cache: Option, +} + +struct TimelineCache { + image: Image, + bounds: IRect, + frames: Range, + thumbnail_revision: u64, + audio_scroll: usize, + scale: (f32, f32), +} + +impl StudioLayout { + pub fn new(width: f32, height: f32, audio_rows: usize, video_aspect: f32) -> Option { + if width < 320. || height < 240. { + return None; + } + let restore = Rect::from_xywh(width - 112., 10., 96., 30.); + if video_aspect < 1. { + let panel = (width - height * video_aspect) + .clamp((width * 0.38).min(320.), (width * 0.60).min(416.)); + let narrow = panel < 320.; + let bottom = if narrow { height - TRANSPORT } else { height }; + let transport = if narrow { + Rect::from_xywh(0., bottom, width, TRANSPORT) + } else { + Rect::from_xywh(0., 0., panel, TRANSPORT) + }; + return Some(Self { + stage: Rect::from_ltrb(panel, 0., width, bottom), + timeline: Rect::from_ltrb(0., 0., panel, bottom), + transport, + restore, + portrait: true, + audio_slots: audio_rows.min(if panel >= 260. { 3 } else { 1 }), + film_height: 0., + timeline_cache: None, + }); + } + let audio_slots = audio_rows.min(if height >= 600. { + 3 + } else { + usize::from(height >= 340.) + }); + let film_height = if height >= 480. { + 58. + } else if height >= 340. { + 36. + } else { + 0. + }; + let timeline_y = height + - TRANSPORT + - RULER + - SCENE_HEIGHT + - film_height + - 16. + - LANE * audio_slots as f32; + Some(Self { + stage: Rect::from_ltrb(0., 0., width, timeline_y), + timeline: Rect::from_ltrb(0., timeline_y, width, height), + restore, + transport: Rect::from_xywh(0., timeline_y, width, TRANSPORT), + portrait: false, + audio_slots, + film_height, + timeline_cache: None, + }) + } + + fn tracks(&self) -> Rect { + if self.portrait { + let top = if self.transport.top == 0. { + self.transport.bottom + } else { + 0. + }; + return Rect::from_ltrb( + 6., + top + 18., + self.timeline.right - 6., + self.timeline.bottom - 10., + ); + } + Rect::from_ltrb( + 48., + self.timeline.top + TRANSPORT + 2., + self.timeline.right - MARGIN, + self.timeline.bottom - 5., + ) + } + + // Time runs down the panel. The same columns define the pixels and the hit targets. + fn portrait_columns(&self) -> (Rect, Rect, Rect) { + let track = self.tracks(); + let film_x = (track.width() * 0.14).clamp(30., 46.) + track.left; + let film = Rect::from_xywh( + film_x, + track.top, + (track.width() * 0.16).clamp(22., 52.), + track.height(), + ); + let audio_x = track.right - 18. * self.audio_slots as f32; + let scenes = Rect::from_ltrb(film.right + 6., track.top, audio_x - 3., track.bottom); + let audio = Rect::from_ltrb(audio_x, track.top, track.right, track.bottom); + (film, scenes, audio) + } + + fn draw_portrait_tracks( + &self, + ui: &Ink<'_>, + state: &ControlsState, + model: &Studio, + scroll: &StudioScroll, + ) { + let track = self.tracks(); + let (film, scenes, mut audio) = self.portrait_columns(); + if model.index.scenes.is_empty() { + audio.left = scenes.left; + } + let lane_width = audio.width() / self.audio_slots.max(1) as f32; + let duration = state.duration_in_frames; + let at = |f| self.frame_position(f, duration); + let canvas = ui.canvas; + ui.fill( + Rect::from_xywh(self.timeline.right - 1., 0., 1., self.timeline.bottom), + LINE, + ); + for frame in (0..=duration).step_by(tick_step(duration, state.fps, track.height(), 64.)) { + let y = at(frame); + let label = if duration < state.fps { + format!("{frame}f") + } else { + format_time(frame, state.fps) + }; + ui.clipped_text( + &label, + Rect::from_ltrb(track.left, y - 8., film.left - 3., y + 10.), + y + 4., + 11., + QUIET, + false, + ); + ui.fill(Rect::from_xywh(film.left - 4., y, 4., 1.), LINE); + } + if !model.thumbnails.is_empty() { + let h = film.height() / model.thumbnails.len() as f32; + canvas.save(); + canvas.clip_rrect(RRect::new_rect_xy(film, 4., 4.), None, true); + ui.fill(film, CANVAS); + for (i, image) in model.thumbnails.iter().enumerate() { + if let Some(image) = image { + ui.thumbnail( + image, + Rect::from_xywh(film.left, film.top + i as f32 * h, film.width(), h), + ); + } + } + canvas.restore(); + } + for scene in &model.index.scenes { + let top = at(scene.frames.start); + let bottom = at(scene.frames.end); + if bottom <= top { + continue; + } + let active = scene.frames.contains(&state.frame); + let color = SCENE_COLORS[scene.index % SCENE_COLORS.len()]; + let band = Rect::from_ltrb(scenes.left, top, scenes.right, (bottom - 1.).max(top)); + ui.fill(band, color.with_a(if active { 34 } else { 15 })); + ui.fill( + Rect::from_ltrb(band.left, top, band.left + 3., band.bottom), + color, + ); + if band.height() >= 13. { + let label = Rect::from_ltrb(band.left + 8., top, band.right - 2., band.bottom); + ui.clipped_text( + &scene.name, + label, + top + 12., + 11., + if active { TEXT } else { QUIET }, + active, + ); + } + } + let first = scroll + .audio + .min(model.audio_rows().saturating_sub(self.audio_slots)); + for (i, lane) in model + .audio + .iter() + .skip(first) + .take(self.audio_slots) + .enumerate() + { + let x = audio.left + lane_width * i as f32; + ui.text( + &format!("A{}", first + i + 1), + x, + track.top - 4., + 10., + QUIET, + false, + ); + for clip in lane { + let r = Rect::from_ltrb( + x, + at(clip.frames.start), + x + lane_width - 3., + at(clip.frames.end), + ); + ui.round(r, AUDIO, 3.); + if let Some(path) = &clip.waveform { + canvas.save(); + canvas.translate((r.right - 1., r.top)); + canvas.rotate(90., None); + canvas.scale((r.height(), r.width() - 2.)); + canvas.draw_path(path, &Ink::paint(Color::from_rgb(86, 86, 86))); + canvas.restore(); + } + if r.width() >= 55. && r.height() >= 17. { + let label = Rect::from_ltrb(r.left + 5., r.top, r.right - 3., r.bottom); + ui.clipped_text(&clip.name, label, r.top + 13., 11., TEXT, false); + } + } + } + } + + fn buttons(&self) -> [(Control, Rect); 6] { + let width = self.transport.width(); + let narrow = width < 460.; + let size = if narrow { 26. } else { 32. }; + let play_size = if narrow { 30. } else { 36. }; + let gap = if narrow { 2. } else { 5. }; + let focus_width = if narrow { 43. } else { 58. }; + let margin = if narrow { 8. } else { MARGIN }; + let mid = width / 2.; + let y = self.transport.center_y(); + let button = |x, w, h| Rect::from_xywh(x, y - h / 2., w, h); + let play = button(mid - play_size / 2., play_size, play_size); + let focus = button(width - margin - focus_width, focus_width, size); + let full = button(focus.left - gap - size, size, size); + [ + (Control::Back, button(play.left - gap - size, size, size)), + (Control::PlayPause, play), + (Control::Forward, button(play.right + gap, size, size)), + (Control::Loop, button(full.left - gap - size, size, size)), + (Control::Fullscreen, full), + (Control::Focus, focus), + ] + } + + pub fn time_axis(&self, point: Point) -> f32 { + if self.portrait { point.y } else { point.x } + } + + fn time_bounds(&self) -> (f32, f32) { + let track = self.tracks(); + if self.portrait { + (track.top, track.height()) + } else { + (track.left, track.width()) + } + } + + fn frame_position(&self, frame: usize, duration: usize) -> f32 { + let (start, length) = self.time_bounds(); + start + length * (frame.min(duration) as f32 / duration.max(1) as f32) + } + + pub fn frame_at(&self, position: f32, duration: usize) -> usize { + let (start, length) = self.time_bounds(); + let fraction = ((position - start) / length).clamp(0., 1.); + ((fraction * duration as f32).floor() as usize).min(duration.saturating_sub(1)) + } + + pub fn hit(&self, point: Point, model: &Studio) -> Option { + if let Some((control, _)) = self.buttons().into_iter().find(|(_, r)| r.contains(point)) { + return Some(control); + } + let tracks = self.tracks(); + if tracks.contains(point) { + let in_scenes = if self.portrait { + self.portrait_columns().1.contains(point) + } else { + (tracks.top + RULER..tracks.top + RULER + SCENE_HEIGHT).contains(&point.y) + }; + if in_scenes { + let frame = self.frame_at(self.time_axis(point), model.index.duration_in_frames); + if let Some(scene) = model.index.scenes_at(frame).last() { + return Some(Control::Scene(scene.frames.start)); + } + } + return Some(Control::Timeline); + } + (!self.stage.contains(point)).then_some(Control::Bar) + } + + pub fn scroll(&self, model: &Studio, scroll: &mut StudioScroll, point: Point, steps: isize) { + if self.tracks().contains(point) { + scroll.audio = scroll + .audio + .saturating_add_signed(steps) + .min(model.audio_rows().saturating_sub(self.audio_slots)); + } + } + + pub fn focus_hit(&self, point: Point) -> bool { + self.restore.contains(point) + } + + pub fn draw_restore(&self, canvas: &Canvas, controls: &Controls, hovered: bool) { + let ui = Ink::new(canvas, controls); + ui.round(self.restore, if hovered { LINE } else { SURFACE }, 7.); + ui.text( + "Studio · T", + self.restore.left + 13., + 30., + 13., + TEXT, + true, + ); + } + + pub fn draw( + &mut self, + canvas: &Canvas, + controls: &Controls, + state: &ControlsState, + model: &Studio, + scroll: &StudioScroll, + ) { + let ui = Ink::new(canvas, controls); + self.draw_cached_timeline(&ui, state, model, scroll); + self.draw_transport(&ui, controls, state); + self.draw_playhead(&ui, state); + } + + fn draw_transport(&self, ui: &Ink<'_>, controls: &Controls, state: &ControlsState) { + let canvas = ui.canvas; + ui.fill(self.transport, SURFACE); + ui.fill( + Rect::from_xywh(0., self.transport.top, self.transport.width(), 1.), + LINE, + ); + let buttons = self.buttons(); + let center = self.transport.center_y(); + let compact = self.transport.width() < 460.; + let left = if compact { 8. } else { MARGIN }; + let total = format_time(state.duration_in_frames, state.fps); + let time = format_time(state.frame, state.fps); + let size: f32 = if compact { 14. } else { 17. }; + let frame_size = (size * 0.74).min(if compact { 11. } else { 12. }); + let mut font = ui.medium.with_size(size).unwrap(); + let mut frame_font = ui.font.with_size(frame_size).unwrap(); + let mark_size = controls.frame_mark.root().intrinsic_size(); + let mark_aspect = mark_size.width / mark_size.height; + let mark_height_ratio = 1.5; + let inline_gap = if compact { 6. } else { 10. }; + let mut room = buttons[0].1.left - left - 6.; + let mut readout_width = tabular_width(&font, &format!("{total} / {total}")); + if !self.portrait { + room -= inline_gap + 2.; + let mark_width = frame_font.metrics().1.cap_height * mark_height_ratio * mark_aspect; + let last_frame = state.duration_in_frames.saturating_sub(1); + readout_width += mark_width + tabular_width(&frame_font, &last_frame.to_string()); + } + if readout_width > room { + let scale = room / readout_width; + font = font.with_size(size * scale).unwrap(); + frame_font = frame_font.with_size(frame_size * scale).unwrap(); + } + let frame_height = frame_font.metrics().1.cap_height; + let clock_height = font.metrics().1.cap_height; + let second_row = if self.portrait { 4. + frame_height } else { 0. }; + let baseline = center + (clock_height - second_row) / 2.; + let after_time = draw_tabular( + canvas, + &time, + Point::new(left, baseline), + &font, + &Ink::paint(TEXT), + ); + let after_clock = draw_tabular( + canvas, + &format!(" / {total}"), + Point::new(after_time, baseline), + &font, + &Ink::paint(QUIET), + ); + let frame_left = if self.portrait { + left + } else { + after_clock + inline_gap + }; + let frame_baseline = baseline + second_row; + let mark_height = frame_height * mark_height_ratio; + let scale = mark_height / mark_size.height; + canvas.save(); + canvas.translate((frame_left, frame_baseline - mark_height * 0.85)); + canvas.scale((scale, scale)); + controls.frame_mark.render(canvas); + canvas.restore(); + draw_tabular( + canvas, + &state.frame.to_string(), + Point::new(frame_left + mark_height * mark_aspect + 2., frame_baseline), + &frame_font, + &Ink::paint(QUIET), + ); + + for (control, rect) in buttons { + if control == Control::Focus { + if state.hovered == Some(control) { + ui.round(rect, LINE, 6.); + } + let font = ui.font.with_size(13.).unwrap(); + let x = rect.center_x() - font.measure_str("Focus", None).0 / 2.; + canvas.draw_str( + "Focus", + (x, center + font.metrics().1.cap_height / 2.), + &font, + &Ink::paint(TEXT), + ); + } else { + controls.draw_button(canvas, control, rect, state, 1.); + } + } + ui.fill( + Rect::from_xywh(0., self.transport.bottom - 1., self.transport.width(), 1.), + LINE, + ); + } + + fn draw_cached_timeline( + &mut self, + ui: &Ink<'_>, + state: &ControlsState, + model: &Studio, + scroll: &StudioScroll, + ) { + let canvas = ui.canvas; + let matrix = canvas.local_to_device_as_3x3(); + let scale = (matrix.scale_x(), matrix.scale_y()); + let valid = self.timeline_cache.as_ref().is_some_and(|cache| { + cache.frames.contains(&state.frame) + && cache.thumbnail_revision == model.thumbnail_revision + && cache.audio_scroll == scroll.audio + && cache.scale == scale + }); + if !valid { + // Align the panel to device pixels so its labels stay sharp at fractional DPI. + let bounds: IRect = matrix.map_rect(self.timeline).0.round_out(); + let info = ImageInfo::new_n32_premul(bounds.size(), None); + self.timeline_cache = canvas.new_surface(&info, None).map(|mut surface| { + let target = surface.canvas(); + target.clear(Color::TRANSPARENT); + target.translate((-bounds.left as f32, -bounds.top as f32)); + target.concat(&matrix); + self.draw_tracks( + &Ink { + canvas: target, + ..ui.clone() + }, + state, + model, + scroll, + ); + + // Scene labels change color at either end of an overlapping scene. + let mut frames = 0..usize::MAX; + for scene in &model.index.scenes { + for boundary in [scene.frames.start, scene.frames.end] { + if boundary <= state.frame { + frames.start = frames.start.max(boundary); + } else { + frames.end = frames.end.min(boundary); + } + } + } + TimelineCache { + image: surface.image_snapshot(), + bounds, + frames, + thumbnail_revision: model.thumbnail_revision, + audio_scroll: scroll.audio, + scale, + } + }); + } + if let Some(cache) = &self.timeline_cache { + canvas.save(); + canvas.reset_matrix(); + canvas.draw_image( + &cache.image, + (cache.bounds.left as f32, cache.bounds.top as f32), + None, + ); + canvas.restore(); + } else { + self.draw_tracks(ui, state, model, scroll); + } + } + + fn draw_tracks( + &self, + ui: &Ink<'_>, + state: &ControlsState, + model: &Studio, + scroll: &StudioScroll, + ) { + let canvas = ui.canvas; + ui.fill(self.timeline, SURFACE); + let track = self.tracks(); + if self.portrait { + return self.draw_portrait_tracks(ui, state, model, scroll); + } + let scene_top = track.top + RULER; + let duration = state.duration_in_frames; + let fps = state.fps.max(1); + let step = tick_step(duration, fps, track.width(), 100.); + for frame in (0..=duration).step_by(step) { + let x = self.frame_position(frame, duration); + let time = if duration < fps { + format!("{frame}f") + } else { + format_time(frame, state.fps) + }; + ui.text( + &time, + x.min(track.right - 37.), + track.top + 13., + 11., + QUIET, + false, + ); + ui.fill(Rect::from_ltrb(x, track.top + 16., x + 1., scene_top), LINE); + } + + let strip_top = scene_top + SCENE_HEIGHT; + let mut previous_end = 0; + for scene in &model.index.scenes { + let left = self.frame_position(scene.frames.start, duration); + let right = self.frame_position(scene.frames.end, duration); + if right <= left { + continue; + } + // Keep the earlier scene's end notch legible during a crossfade. + let rail = self + .frame_position(previous_end.max(scene.frames.start), duration) + .min(right); + previous_end = scene.frames.end; + let color = SCENE_COLORS[scene.index % SCENE_COLORS.len()]; + ui.fill( + Rect::from_ltrb(left, scene_top, right, track.bottom), + color.with_a(13), + ); + ui.fill( + Rect::from_ltrb(rail, scene_top, right, scene_top + 3.), + color, + ); + let notch = (right - rail).min(9.); + let mut marker = PathBuilder::new(); + marker.move_to((right - notch, scene_top)); + marker.line_to((right, scene_top)); + marker.line_to((right, scene_top + notch)); + marker.close(); + canvas.draw_path(&marker.detach(), &Ink::paint(color)); + if right - rail > 12. { + let ink = if scene.frames.contains(&state.frame) { + TEXT + } else { + QUIET + }; + ui.clipped_text( + &scene.name, + Rect::from_ltrb(rail + 3., scene_top, right - 2., strip_top), + scene_top + 16., + 11., + ink, + false, + ); + } + } + if self.film_height > 0. && !model.thumbnails.is_empty() { + let strip = Rect::from_ltrb( + track.left, + strip_top, + track.right, + strip_top + self.film_height, + ); + canvas.save(); + canvas.clip_rrect(RRect::new_rect_xy(strip, 5., 5.), None, true); + ui.fill(strip, CANVAS); + let w = strip.width() / model.thumbnails.len() as f32; + for (slot, image) in model.thumbnails.iter().enumerate() { + let Some(image) = image else { + continue; + }; + let rect = + Rect::from_xywh(track.left + slot as f32 * w, strip.top, w, self.film_height); + ui.thumbnail(image, rect); + } + canvas.restore(); + } + let first_lane = scroll + .audio + .min(model.audio_rows().saturating_sub(self.audio_slots)); + for (row, lane) in model + .audio + .iter() + .skip(first_lane) + .take(self.audio_slots) + .enumerate() + { + let top = strip_top + self.film_height + 6. + LANE * row as f32; + let label = if model.audio_rows() > self.audio_slots { + format!("A{}/{}", row + first_lane + 1, model.audio_rows()) + } else { + format!("A{}", row + first_lane + 1) + }; + ui.text(&label, MARGIN, top + 14., 12., QUIET, false); + for clip in lane { + let r = Rect::from_ltrb( + self.frame_position(clip.frames.start, duration), + top, + self.frame_position(clip.frames.end, duration), + top + LANE - 3., + ); + ui.round(r, AUDIO, 4.); + if let Some(path) = &clip.waveform { + canvas.save(); + canvas.translate((r.left, r.top + 2.)); + canvas.scale((r.width(), r.height() - 3.)); + canvas.draw_path(path, &Ink::paint(Color::from_rgb(86, 86, 86))); + canvas.restore(); + } + if r.width() > 20. { + let label = Rect::from_ltrb(r.left + 5., r.top, r.right - 2., r.bottom); + ui.clipped_text(&clip.name, label, r.top + 13., 11., TEXT, true); + } + } + } + } + + fn draw_playhead(&self, ui: &Ink<'_>, state: &ControlsState) { + let track = self.tracks(); + if self.portrait { + let film = self.portrait_columns().0; + let y = self.frame_position(state.frame, state.duration_in_frames); + ui.fill( + Rect::from_ltrb(film.left - 2., y - 1., track.right, y + 1.), + ORANGE, + ); + ui.round(Rect::from_xywh(film.left - 4., y - 4., 7., 8.), ORANGE, 2.); + return; + } + let x = self.frame_position(state.frame, state.duration_in_frames); + ui.fill( + Rect::from_ltrb(x - 1., track.top, x + 1., track.bottom), + ORANGE, + ); + ui.round(Rect::from_xywh(x - 4., track.top - 3., 8., 7.), ORANGE, 2.); + } +} + +fn tick_step(duration: usize, fps: usize, span: f32, spacing: f32) -> usize { + let fps = fps.max(1); + let unit = if duration < fps { 1 } else { fps }; + let ticks = (span / spacing).floor().max(1.) as usize; + let target = duration.div_ceil(unit).div_ceil(ticks).max(1); + let power = 10_usize.pow(target.ilog10()); + [1_usize, 2, 5, 10] + .into_iter() + .map(|m| power.saturating_mul(m)) + .find(|&step| step >= target) + .unwrap() + .saturating_mul(unit) +} + +#[derive(Clone)] +struct Ink<'a> { + canvas: &'a Canvas, + font: Font, + medium: Font, +} + +impl<'a> Ink<'a> { + fn new(canvas: &'a Canvas, controls: &Controls) -> Self { + Self { + canvas, + font: controls.font(false, 1.), + medium: controls.font(true, 1.), + } + } + + fn paint(color: Color) -> Paint { + let mut paint = Paint::default(); + paint.set_anti_alias(true); + paint.set_color(color); + paint + } + + fn thumbnail(&self, image: &Image, rect: Rect) { + let canvas = self.canvas; + let scale = + (rect.width() / image.width() as f32).max(rect.height() / image.height() as f32); + let mut dst = Rect::from_wh(image.width() as f32 * scale, image.height() as f32 * scale); + dst.offset(( + rect.center_x() - dst.center_x(), + rect.center_y() - dst.center_y(), + )); + canvas.save(); + canvas.clip_rect(rect, None, false); + canvas.draw_image_rect_with_sampling_options( + image, + None, + dst, + SamplingOptions::from(FilterMode::Linear), + &Paint::default(), + ); + canvas.restore(); + } + + fn fill(&self, rect: Rect, color: Color) { + self.canvas.draw_rect(rect, &Self::paint(color)); + } + + fn round(&self, rect: Rect, color: Color, radius: f32) { + self.canvas + .draw_round_rect(rect, radius, radius, &Self::paint(color)); + } + + fn text(&self, text: &str, x: f32, y: f32, size: f32, color: Color, medium: bool) { + let font = if medium { &self.medium } else { &self.font }; + self.canvas.draw_str( + text, + (x, y), + &font.with_size(size).unwrap(), + &Self::paint(color), + ); + } + + fn clipped_text( + &self, + text: &str, + rect: Rect, + baseline: f32, + size: f32, + color: Color, + medium: bool, + ) { + self.canvas.save(); + self.canvas.clip_rect(rect, None, false); + self.text(text, rect.left, baseline, size, color, medium); + self.canvas.restore(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use fframes::TimelineScene; + + fn studio() -> Studio { + Studio { + video_aspect: 16. / 9., + index: TimelineIndex { + fps: 30, + duration_in_frames: 100, + scenes: [0..60, 50..100, 110..115] + .into_iter() + .enumerate() + .map(|(index, frames)| TimelineScene { + index, + frames, + name: format!("Scene {index}"), + full_name: String::new(), + }) + .collect(), + }, + audio: vec![], + thumbnails: vec![], + thumbnail_revision: 0, + } + } + + #[test] + fn navigation_uses_resolved_starts_and_skips_unreachable_scenes() { + let studio = studio(); + assert_eq!(studio.adjacent_scene(49, true), Some(50)); + assert_eq!(studio.adjacent_scene(50, false), Some(0)); + assert_eq!(studio.adjacent_scene(51, false), Some(50)); + assert_eq!(studio.adjacent_scene(50, true), None); + assert_eq!(studio.adjacent_scene(0, false), None); + } + + #[test] + fn cached_timeline_matches_fresh_draws_after_scene_scroll_thumbnail_and_scale_changes() { + let controls = Controls::new(); + let mut model = studio(); + model.thumbnails = vec![None]; + model.audio = (0..4) + .map(|i| { + vec![Clip { + name: format!("Audio {i}"), + frames: i * 10..100, + waveform: None, + }] + }) + .collect(); + let mut scroll = StudioScroll::default(); + let mut state = ControlsState { + visible: true, + frame: 0, + duration_in_frames: 100, + fps: 30, + playing: true, + looping: true, + fullscreen: false, + hovered: None, + }; + let draw = |layout: &mut StudioLayout, + state: &ControlsState, + model: &Studio, + scroll: &StudioScroll, + scale: f32, + cached: bool| { + let size = (800. * scale) as i32; + let info = ImageInfo::new_n32_premul((size, size), None); + let mut surface = surfaces::raster(&info, None, None).unwrap(); + let canvas = surface.canvas(); + canvas.clear(Color::BLACK); + canvas.scale((scale, scale)); + if cached { + layout.draw(canvas, &controls, state, model, scroll); + } else { + let ui = Ink::new(canvas, &controls); + layout.draw_tracks(&ui, state, model, scroll); + layout.draw_transport(&ui, &controls, state); + layout.draw_playhead(&ui, state); + } + let mut pixels = vec![0; (size * size * 4) as usize]; + assert!(surface.read_pixels(&info, &mut pixels, size as usize * 4, (0, 0))); + pixels + }; + for aspect in [16. / 9., 9. / 16.] { + let mut layout = StudioLayout::new(800., 800., 4, aspect).unwrap(); + for scale in [1., 1.25, 2.] { + let mut first_id = None; + for frame in [0, 1, 49, 50, 59, 60, 99, 0] { + state.frame = frame; + assert!( + draw(&mut layout, &state, &model, &scroll, scale, true) + == draw(&mut layout, &state, &model, &scroll, scale, false), + "aspect={aspect}, frame={frame}, scale={scale}" + ); + let id = layout.timeline_cache.as_ref().unwrap().image.unique_id(); + if frame == 0 { + first_id = Some(id); + } else if frame == 1 { + assert_eq!(first_id, Some(id)); + } + } + } + scroll.audio = 1; + let mut thumbnail = surfaces::raster_n32_premul((20, 10)).unwrap(); + thumbnail.canvas().clear(Color::RED); + for updated_thumbnail in [false, true] { + if updated_thumbnail { + model.set_thumbnail(0, thumbnail.image_snapshot()); + } + assert!( + draw(&mut layout, &state, &model, &scroll, 2., true) + == draw(&mut layout, &state, &model, &scroll, 2., false), + "aspect={aspect}, updated_thumbnail={updated_thumbnail}" + ); + } + scroll.audio = 0; + model.thumbnails[0] = None; + } + } + + #[test] + fn timeline_seek_and_hit_regions_agree_across_sizes() { + let model = studio(); + for width in [320., 440., 460., 560., 1280.] { + let layout = StudioLayout::new(width, 700., 2, model.video_aspect).unwrap(); + let track = layout.tracks(); + assert_eq!(layout.hit(track.center(), &model), Some(Control::Timeline)); + assert_eq!(layout.hit(layout.stage.center(), &model), None); + assert_eq!(layout.stage.top, 0.); + let buttons = layout.buttons(); + assert_eq!(buttons[1].1.center_x(), width / 2.); + assert!( + buttons + .windows(2) + .all(|pair| pair[0].1.right <= pair[1].1.left) + ); + assert!(layout.focus_hit(layout.restore.center())); + assert_eq!(layout.hit(layout.restore.center(), &model), None); + assert_eq!(layout.frame_at(track.left - 1., 100), 0); + assert_eq!(layout.frame_at(track.right, 100), 99); + assert_eq!(layout.frame_at(track.center_x(), 100), 50); + assert_eq!(layout.frame_at(track.right, 1), 0); + assert_eq!(layout.frame_at(track.right, 0), 0); + let scene = Point::new(track.center_x() + 1., track.top + RULER + SCENE_HEIGHT / 2.); + assert_eq!(layout.hit(scene, &model), Some(Control::Scene(50))); + assert_eq!(layout.stage.left, 0.); + assert!(track.bottom < 700.); + for (control, rect) in layout.buttons() { + assert_eq!(layout.hit(rect.center(), &model), Some(control)); + } + } + let compact = StudioLayout::new(320., 240., 2, model.video_aspect).unwrap(); + assert_eq!(compact.timeline.bottom, 240.); + assert_eq!(compact.audio_slots, 0); + assert!(StudioLayout::new(300., 200., 0, model.video_aspect).is_none()); + } + + #[test] + fn portrait_uses_the_left_timeline_for_scenes_and_vertical_seeking() { + let model = studio(); + for (width, height) in [(765., 720.), (1280., 720.), (405., 720.), (320., 240.)] { + let ui = StudioLayout::new(width, height, 2, 9. / 16.).unwrap(); + let track = ui.tracks(); + let (film, scenes, audio) = ui.portrait_columns(); + assert!(ui.portrait); + assert_eq!(ui.stage.top, 0.); + assert_eq!(ui.stage.left, ui.timeline.right); + assert!( + track.right < ui.stage.left + && film.right < scenes.left + && scenes.right < audio.left + ); + assert_eq!(ui.hit(ui.stage.center(), &model), None); + assert_eq!(ui.frame_at(ui.time_axis(film.center()), 100), 50); + assert_eq!(ui.frame_at(track.top - 10., 100), 0); + assert_eq!(ui.frame_at(track.bottom + 10., 100), 99); + assert_eq!(ui.hit(film.center(), &model), Some(Control::Timeline)); + assert_eq!(ui.hit(scenes.center(), &model), Some(Control::Scene(50))); + assert_eq!(ui.buttons()[1].1.center_x(), ui.transport.center_x()); + for (control, button) in ui.buttons() { + assert_eq!(ui.hit(button.center(), &model), Some(control)); + } + if width == 765. { + assert_eq!(ui.stage, Rect::from_ltrb(360., 0., 765., 720.)); + } + } + assert!(!StudioLayout::new(405., 720., 2, 16. / 9.).unwrap().portrait); + assert!(!StudioLayout::new(405., 720., 2, 1.).unwrap().portrait); + } + + #[test] + fn waveform_uses_both_channels_and_the_selected_source_interval() { + let data = PreloadedAudioData { + sample_rate: 512, + samples: vec![0.; 1024].into(), + right: Some([vec![0.; 512], vec![0.75; 512]].concat().into()), + }; + assert_eq!(waveform(&data, 0., 1.).bounds().top, 1.); + assert_eq!(waveform(&data, 1., 1.).bounds().top, 0.25); + assert_eq!(waveform(&data, 2., 1.).bounds().top, 1.); + } + + #[test] + fn overlapping_audio_stacks_and_reuses_the_first_free_lane() { + let clips = [15..18, 0..32, 3..5, 5..7].map(|frames| Clip { + name: String::new(), + waveform: None, + frames, + }); + let lanes = pack_clips(Vec::from(clips)); + assert_eq!(lanes.len(), 2); + assert_eq!(lanes[0][0].frames, 0..32); + assert_eq!( + lanes[1] + .iter() + .map(|clip| clip.frames.clone()) + .collect::>(), + [3..5, 5..7, 15..18] + ); + } +} diff --git a/fframes/Cargo.toml b/fframes/Cargo.toml index d78ed2be..1924687d 100644 --- a/fframes/Cargo.toml +++ b/fframes/Cargo.toml @@ -91,6 +91,7 @@ mp3lame = ["fframes-media/mp3lame"] opus = ["fframes-media/opus"] vpx = ["fframes-media/vpx"] # enable various hardware accelerations +preview-videotoolbox = ["fframes-media/preview-videotoolbox"] videotoolbox = ["fframes-media/videotoolbox"] audiotoolbox = ["fframes-media/audiotoolbox"] vaapi = ["fframes-media/vaapi"] diff --git a/fframes/build.rs b/fframes/build.rs index 1e7cbb5d..3607dccc 100644 --- a/fframes/build.rs +++ b/fframes/build.rs @@ -1,13 +1,9 @@ -use std::env::VarError; - fn main() { let target_os = std::env::var("CARGO_CFG_TARGET_OS").unwrap_or_default(); if (target_os == "macos" || target_os == "ios") - && matches!( - std::env::var("CARGO_FEATURE_VIDEOTOOLBOX"), - Err(VarError::NotPresent) - ) + && std::env::var_os("CARGO_FEATURE_VIDEOTOOLBOX").is_none() + && std::env::var_os("CARGO_FEATURE_PREVIEW_VIDEOTOOLBOX").is_none() { println!( "cargo:warning=\x1B[1m⚠️ WARNING: Apple target arch without videotoolbox enabled ⚠️\x1B[0m" diff --git a/fframes/src/media_provider.rs b/fframes/src/media_provider.rs index d9f8ad94..0216a440 100644 --- a/fframes/src/media_provider.rs +++ b/fframes/src/media_provider.rs @@ -52,6 +52,14 @@ pub trait MediaProvider<'a>: Send + Sync + Debug { Vec::new() } + /// Whether the provider lists video files that need a decoder during playback. + /// A provider that can return `Some` from `resolve_video` must override this method + /// or `get_all_video_data`. Otherwise the native preview uses its no-video defaults: + /// more workers, and decoders that are not shared between them. + fn has_video_sources(&self) -> bool { + !self.get_all_video_data().is_empty() + } + #[cfg(feature = "lottie")] fn get_all_lottie_data(&self) -> Vec<(&crate::Lottie, &str)> { Vec::new() @@ -241,6 +249,10 @@ impl<'a> MediaProvider<'a> for DynamicMediaProvider<'a> { .collect() } + fn has_video_sources(&self) -> bool { + !self.videos.is_empty() + } + fn get_all_video_data(&self) -> Vec<(&fframes_media::VideoMedia, &str)> { self.videos .iter() @@ -293,6 +305,10 @@ impl<'a, const N: usize> MediaProvider<'a> for CombinedMediaProvider<'a, N> { .find_map(|provider| provider.resolve_video(name)) } + fn has_video_sources(&self) -> bool { + self.0.iter().any(|provider| provider.has_video_sources()) + } + #[cfg(feature = "lottie")] fn resolve_lottie(&'a self, name: &str) -> Option<&'a crate::Lottie> { self.0 diff --git a/fframes/src/renderer/cli.rs b/fframes/src/renderer/cli.rs index 29b64a5e..bb2cdf6e 100644 --- a/fframes/src/renderer/cli.rs +++ b/fframes/src/renderer/cli.rs @@ -131,6 +131,9 @@ pub struct PreviewArgs { /// Do not play the audio. #[arg(long)] pub mute: bool, + /// Decode source video on the CPU even if hardware decoding is available. + #[arg(long)] + pub software_decode: bool, /// Skia backend of the window: auto, metal, vulkan or cpu. #[arg(long, default_value = "auto")] pub backend: String, @@ -494,12 +497,15 @@ impl<'r, 'a, 'media: 'a, V: Video + Send + Sync, B: FFramesRenderBackend, A: Arg /// Parses the command line (unless `args` was given) and runs the command. pub fn run(mut self) -> ExitCode { - let renderer = self - .args - .get_or_insert_with(Cli::::parse) + let args = self.args.get_or_insert_with(Cli::::parse); + if matches!(args.command, Some(Command::Preview(_))) { + // The player opens its own window backend. + return self.execute(); + } + let renderer = args .renderer .take() - .or(self.renderers.default_renderer()); + .or_else(|| self.renderers.default_renderer()); let runner = Runner { video: self.video, options: self.options, @@ -550,6 +556,10 @@ impl<'r, 'a, 'media: 'a, V: Video + Send + Sync, B: FFramesRenderBackend, A: Arg args.output = args.output.or(Some(default_output)); render(json, cli.scale, video, options, args, backend) } + Command::Preview(args) => { + drop(backend); + preview(video, &options, player, args) + } command => { let mut frame_renderer = backend.frame_renderer(); let mut fallback; @@ -575,29 +585,7 @@ impl<'r, 'a, 'media: 'a, V: Video + Send + Sync, B: FFramesRenderBackend, A: Arg inspect(json, &mut previewer, frame_renderer, args) } Command::Audio(args) => audio(json, &previewer, args), - Command::Preview(args) => { - let Some(play) = player else { - return Err("this binary has no real-time player: add the \ - `fframes_native_player` crate and \ - `.preview(fframes_native_player::cli_preview)`. Without a \ - window use `strip`, `frame` or `render --draft`." - .to_owned()); - }; - let request = super::PreviewRequest { - start_frame: previewer - .timeline() - .resolve_frame(&args.at) - .map_err(err)?, - autoplay: !args.paused, - looping: !args.no_loop, - audio: !args.mute, - backend: args.backend, - }; - // Fonts and caches of the previewer are not needed by the player. - drop(previewer); - play(video, &options, &request).map(|()| ExitCode::SUCCESS) - } - Command::Render(_) => unreachable!(), + Command::Preview(_) | Command::Render(_) => unreachable!(), }) } }; @@ -630,6 +618,50 @@ fn print(json: bool, value: &T, text: impl FnOnce() -> String) { } } +fn preview<'a, 'media: 'a, V: Video>( + video: &'a V, + options: &RenderOptions<'a, 'media>, + player: Option>, + args: PreviewArgs, +) -> Result { + let Some(play) = player else { + return Err("this binary has no real-time player: add the \ + `fframes_native_player` crate and \ + `.preview(fframes_native_player::cli_preview)`. Without a \ + window use `strip`, `frame` or `render --draft`." + .to_owned()); + }; + let start_frame = { + let time_base = crate::TimeBase { + fps: V::FPS, + sample_rate: options.audio_encoder_options.sample_rate, + }; + let timeline = crate::FFramesRendererRuntime::resolve_video_timeline( + time_base, + video, + &video.define_scenes(), + options.media, + ) + .map_err(err)?; + crate::TimelineIndex::new( + V::FPS, + timeline.duration_in_frames, + timeline.scenes.as_ref(), + ) + .resolve_frame(&args.at) + .map_err(err)? + }; + let request = super::PreviewRequest { + start_frame, + autoplay: !args.paused, + looping: !args.no_loop, + audio: !args.mute, + software_decode: args.software_decode, + backend: args.backend, + }; + play(video, options, &request).map(|()| ExitCode::SUCCESS) +} + fn scenes_label(scenes: &[String]) -> String { if scenes.is_empty() { String::new() diff --git a/fframes/src/renderer/preview.rs b/fframes/src/renderer/preview.rs index 37fc45c9..c5a91f75 100644 --- a/fframes/src/renderer/preview.rs +++ b/fframes/src/renderer/preview.rs @@ -211,6 +211,7 @@ pub struct PreviewRequest { pub autoplay: bool, pub looping: bool, pub audio: bool, + pub software_decode: bool, /// `auto`, `metal`, `vulkan` or `cpu`. pub backend: String, } @@ -975,6 +976,38 @@ mod batch_tests { } } + #[cfg(feature = "cli")] + #[test] + fn native_preview_resolves_the_start_without_an_export_backend() { + use crate::cli::{self, clap::Parser}; + + let video = Changing { panic_at: None }; + let args = cli::Cli::::parse_from([ + "video", + "--renderer", + "gpu", + "preview", + "1s", + "--backend", + "cpu", + "--paused", + "--mute", + ]); + let mut opened = false; + let result = cli::new(&video, RenderOptions::default()) + .args(args) + .preview(|_, _, request| { + assert_eq!(request.start_frame, 10); + assert_eq!(request.backend, "cpu"); + assert!(!request.autoplay && !request.audio); + opened = true; + Ok(()) + }) + .run(); + assert_eq!(result, std::process::ExitCode::SUCCESS); + assert!(opened); + } + /// Out of order, with a repeated frame. const FRAMES: [usize; 9] = [23, 5, 0, 12, 12, 19, 20, 3, 8]; diff --git a/fframes/src/renderer/renderer.rs b/fframes/src/renderer/renderer.rs index 1219bf30..87d580c3 100644 --- a/fframes/src/renderer/renderer.rs +++ b/fframes/src/renderer/renderer.rs @@ -105,11 +105,76 @@ impl<'a, 'media: 'a> FFramesRendererRuntime<'a> { scenes: &Scenes<'a>, media: Option<&'media dyn MediaProvider<'media>>, ) -> FFramesRendererResult { - let timeline = crate::resolve_timeline( - &video.duration(), + Self::with_audio_sample_rate(time_base, video, scenes, media, || time_base.sample_rate) + } + + /// Choose the playback device's sample rate only for projects with audio tracks. + /// Silent projects keep the supplied time base and never need to open a device. + pub fn with_audio_sample_rate( + mut time_base: TimeBase, + video: &'a TVideo, + scenes: &Scenes<'a>, + media: Option<&'media dyn MediaProvider<'media>>, + audio_sample_rate: impl FnOnce() -> usize, + ) -> FFramesRendererResult { + let scenes = ScenesWithAudio::new(scenes); + let audio = video.audio(); + let has_tracks = + |map: &crate::AudioMap<'_>| map.0.as_ref().is_some_and(|tracks| !tracks.is_empty()); + if has_tracks(&audio) + || scenes + .0 + .iter() + .flatten() + .any(|scene| has_tracks(&scene.audio_map)) + { + time_base.sample_rate = audio_sample_rate(); + } + let timeline = Self::resolve_audio_timeline(time_base, video, &scenes, &audio, media)?; + + let mut font_source = RendererFontSource { + fontdb: fontdb::Database::new(), + }; + + if let Some(media) = media { + media.populate_font_source(&mut font_source); + } + + Ok(Self { + time_base, + timeline, + font_source, + }) + } + + #[cfg(feature = "cli")] + pub(super) fn resolve_video_timeline( + time_base: TimeBase, + video: &'a TVideo, + scenes: &Scenes<'a>, + media: Option<&'media dyn MediaProvider<'media>>, + ) -> FFramesRendererResult> { + Self::resolve_audio_timeline( + time_base, + video, &ScenesWithAudio::new(scenes), - &time_base, &video.audio(), + media, + ) + } + + fn resolve_audio_timeline( + time_base: TimeBase, + video: &'a TVideo, + scenes: &ScenesWithAudio<'a>, + audio: &crate::AudioMap<'_>, + media: Option<&'media dyn MediaProvider<'media>>, + ) -> FFramesRendererResult> { + Ok(crate::resolve_timeline( + &video.duration(), + scenes, + &time_base, + audio, |name| { let media = media.ok_or_else(|| { crate::error::FFramesError::RequiredAudioNotFound(name.to_owned()) @@ -137,21 +202,7 @@ impl<'a, 'media: 'a> FFramesRendererRuntime<'a> { crate::error::FFramesError::CanNotProcessAudioDuration(name.to_owned()) }) }, - )?; - - let mut font_source = RendererFontSource { - fontdb: fontdb::Database::new(), - }; - - if let Some(media) = media { - media.populate_font_source(&mut font_source); - } - - Ok(Self { - time_base, - timeline, - font_source, - }) + )?) } } diff --git a/fframes/src/renderer/tests/preview_tests.rs b/fframes/src/renderer/tests/preview_tests.rs index f2a767fd..f4314e74 100644 --- a/fframes/src/renderer/tests/preview_tests.rs +++ b/fframes/src/renderer/tests/preview_tests.rs @@ -201,3 +201,45 @@ fn frames_crop_to_a_clamped_rectangle() { let corner = frame.crop(190, 90, 50, 50); assert_eq!((corner.width, corner.height), (10, 10)); } + +#[test] +fn audio_device_rate_is_only_requested_for_playable_tracks() { + use crate::{AudioTimestamp::Second, FFramesRendererRuntime, Scenes, TimeBase}; + #[derive(Debug)] + struct Sound; + impl Scene for Sound { + fn audio(&self) -> AudioMap<'_> { + AudioMap::from([("clip.wav", Second(0.)..Second(1.))]) + } + fn duration(&self) -> crate::Duration<'_> { + crate::Duration::Seconds(1.) + } + fn render_frame<'a>(&'a self, _: Frame, _: &FFramesContext) -> Svgr<'a> { + unreachable!() + } + } + + let time = TimeBase { + fps: 10, + sample_rate: 44100, + }; + let silent = FFramesRendererRuntime::with_audio_sample_rate( + time, + &TwoScenes, + &TwoScenes.define_scenes(), + None, + || panic!("silent scenes opened the audio device"), + ) + .unwrap(); + assert_eq!(silent.time_base.sample_rate, 44100); + + let scenes = Scenes::from(vec![&Sound as &dyn Scene]); + let voiced = + FFramesRendererRuntime::with_audio_sample_rate(time, &TwoScenes, &scenes, None, || 48000) + .unwrap(); + assert_eq!(voiced.time_base.sample_rate, 48000); + assert_eq!( + voiced.timeline.audio_map.unwrap().tracks()[0].range.end.0, + 48000 + ); +} diff --git a/fframes/src/video_data.rs b/fframes/src/video_data.rs index 1fab172f..b6112ee3 100644 --- a/fframes/src/video_data.rs +++ b/fframes/src/video_data.rs @@ -8,7 +8,11 @@ use crate::error::Result; use fframes_media::ImageMetadata; pub use fframes_media::{FrameConvertOptions, PlaybackSpeed, ResizeVideoFrame}; #[cfg(not(target_arch = "wasm32"))] -use std::collections::HashMap; +use std::{ + collections::HashMap, + path::{Path, PathBuf}, + sync::Mutex, +}; #[cfg(target_arch = "wasm32")] use std::cell::Cell; @@ -43,20 +47,11 @@ pub fn set_video_frame_resolver(resolver: VideoFrameResolver) { } pub trait FFramesSyncedVideoFrame<'media> { - /// Gets the original image from the frame - /// This will perform color space conversion and return the frame at `width()` by - /// `height()`. If you need to render the video frame in the smaller size, use `into_resized_image`. - /// - /// This method take self to reuse frame data allocations, but you should not - /// call this method more than once per the render function + /// Converts the frame to RGBA at its display dimensions. + /// Native renderers share the pixels between consecutive requests at the same size. #[allow(clippy::wrong_self_convention)] fn into_image(&self) -> ImageData<'media>; - /// Performs color space conversion and resizes the image to the specified size. - /// Recommended to use this method if you render image in the smaller size than the original - /// video e.g. as a pattern of some other shape or as a filler of standalone image. - /// - /// This method take self to reuse frame data allocations, but you should not - /// call this method more than once per the render function + /// Converts the frame to RGBA at the requested size. #[allow(clippy::wrong_self_convention)] fn into_resized_image(&self, options: &FrameConvertOptions) -> Option>; /// Get the height of the frame with non-square pixels stretched to square ones @@ -151,23 +146,23 @@ impl<'media> FFramesSyncedVideoFrame<'media> for WasmEditorVideoFrameFallback<'m } } +#[cfg(not(target_arch = "wasm32"))] +fn decoded_image<'media>(pixels: Arc) -> ImageData<'media> { + let metadata = ImageMetadata { + width: pixels.width, + height: pixels.height, + }; + let id = pixels.id.clone(); + ImageData::new_from_raw_data(pixels, id, metadata) +} + #[cfg(not(target_arch = "wasm32"))] impl<'media> FFramesSyncedVideoFrame<'media> for crate::media::FFmpegFrameBuf { fn into_image(&self) -> ImageData<'media> { let image_data = unsafe { self.convert_last_decoded_frame_into_svg_image(None) } - // we unwrap here because we guarantee that the sws scaler would work correctly - // unless the wrong dimensions are provided .expect("Failed to convert frame to image"); - let filename = image_data.id.clone(); - ImageData::new_from_raw_data( - image_data, - filename, - ImageMetadata { - width: self.get_width(), - height: self.get_height(), - }, - ) + decoded_image(image_data) } fn into_resized_image(&self, options: &FrameConvertOptions) -> Option> { @@ -177,15 +172,7 @@ impl<'media> FFramesSyncedVideoFrame<'media> for crate::media::FFmpegFrameBuf { }) .ok()?; - let filename = image_data.id.clone(); - Some(ImageData::new_from_raw_data( - image_data, - filename, - ImageMetadata { - width: self.get_width(), - height: self.get_height(), - }, - )) + Some(decoded_image(image_data)) } fn width(&self) -> u32 { @@ -213,26 +200,120 @@ impl<'media> FFramesSyncedVideoFrame<'media> for crate::media::FFmpegFrameBuf { } } -#[derive(Clone)] -#[allow(dead_code)] -pub struct VideoDecodersWorker { - buffer_size: usize, - #[cfg(not(target_arch = "wasm32"))] - map: Arc>>>, -} +#[cfg(not(target_arch = "wasm32"))] +type DecoderSlot = Arc>>; -/// A decoder of a file at one playback speed. #[cfg(not(target_arch = "wasm32"))] -type SpeedDecoder = (PlaybackSpeed, fframes_media::FFmpegDecoder); +#[derive(Default)] +struct DecoderEntry { + /// The preview job that used the decoder last, see `DecoderPool::finish_frame`. + last_used: u64, + /// The order of the last `DecoderPool::slot` call, for `MAX_DECODERS_PER_VIDEO`. + last_access: u64, + slot: DecoderSlot, +} /// Each decoder keeps the file open and its frames in memory. A file played at more -/// speeds closes the decoder that was used least recently. +/// speeds closes the idle decoder that was used least recently. #[cfg(not(target_arch = "wasm32"))] const MAX_DECODERS_PER_VIDEO: usize = 4; +#[cfg(not(target_arch = "wasm32"))] +#[derive(Default)] +struct DecoderPool { + /// The decoders of each file, one per playback speed: offsets of different speeds + /// point to different parts of the file. + sources: HashMap>, + generation: u64, + accesses: u64, +} + +#[cfg(not(target_arch = "wasm32"))] +impl DecoderPool { + fn slot(&mut self, path: &Path, speed: PlaybackSpeed) -> DecoderSlot { + self.accesses += 1; + let decoders = if let Some(decoders) = self.sources.get_mut(path) { + decoders + } else { + self.sources.entry(path.to_owned()).or_default() + }; + let index = if let Some(index) = decoders.iter().position(|(s, _)| *s == speed) { + index + } else { + if decoders.len() >= MAX_DECODERS_PER_VIDEO + && let Some(idle) = decoders + .iter() + .enumerate() + .filter(|(_, (_, entry))| Arc::strong_count(&entry.slot) == 1) + .min_by_key(|(_, (_, entry))| entry.last_access) + .map(|(index, _)| index) + { + decoders.remove(idle); + } + decoders.push((speed, DecoderEntry::default())); + decoders.len() - 1 + }; + let entry = &mut decoders[index].1; + entry.last_used = self.generation; + entry.last_access = self.accesses; + Arc::clone(&entry.slot) + } + + /// The decoders of `path` at every speed. A bare file name finds an unambiguous source. + fn slots_of(&self, path: &Path) -> Vec { + let decoders = self.sources.get(path).or_else(|| { + let mut matches = self + .sources + .iter() + .filter(|(source, _)| source.file_name() == Some(path.as_os_str())); + let (_, decoders) = matches.next()?; + matches.next().is_none().then_some(decoders) + }); + decoders + .map(|decoders| { + decoders + .iter() + .map(|(_, entry)| Arc::clone(&entry.slot)) + .collect() + }) + .unwrap_or_default() + } + + fn finish_frame(&mut self, concurrent_jobs: usize) -> Vec { + // Count completed jobs, not timeline positions. Scrubs, loops and out-of-order + // workers share the same grace period. Keep all layers of an active composition. + self.generation += 1; + let generation = self.generation; + let grace = concurrent_jobs.max(1) as u64 + 1; + let mut retired = Vec::new(); + self.sources.retain(|_, decoders| { + retired.extend( + decoders + .extract_if(.., |(_, entry)| { + generation - entry.last_used > grace && Arc::strong_count(&entry.slot) == 1 + }) + .map(|(_, entry)| entry.slot), + ); + !decoders.is_empty() + }); + retired + } +} + +#[derive(Clone)] +#[allow(dead_code)] +pub struct VideoDecodersWorker { + buffer_size: usize, + #[cfg(not(target_arch = "wasm32"))] + pool: Arc>, + #[cfg(not(target_arch = "wasm32"))] + mode: fframes_media::DecodeMode, +} + impl std::fmt::Debug for VideoDecodersWorker { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("WorkerLocalDecoders").finish() + f.debug_struct("VideoDecodersWorker") + .finish_non_exhaustive() } } @@ -247,10 +328,37 @@ impl VideoDecodersWorker { VideoDecodersWorker { buffer_size, #[cfg(not(target_arch = "wasm32"))] - map: Arc::new(std::sync::RwLock::new(HashMap::new())), + pool: Arc::default(), + #[cfg(not(target_arch = "wasm32"))] + mode: fframes_media::DecodeMode::Configured, + } + } + + /// The native studio keeps only recently used source decoders between frames. + #[cfg(not(target_arch = "wasm32"))] + pub fn for_preview(software_decode: bool) -> Self { + Self { + mode: if software_decode { + fframes_media::DecodeMode::Software + } else { + fframes_media::DecodeMode::Preview + }, + ..Self::default() } } + /// Retire idle decoder sessions outside the pool lock, including after a seek + /// into a scene that contains no video files. + #[cfg(not(target_arch = "wasm32"))] + pub fn finish_preview_frame(&self, concurrent_jobs: usize) { + let retired = self + .pool + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .finish_frame(concurrent_jobs); + drop(retired); + } + #[cfg(target_arch = "wasm32")] pub(crate) fn get_synced_frame<'media>( &self, @@ -323,13 +431,17 @@ impl VideoDecodersWorker { pts: i64, filename: &str, ) -> Result>> { - let frame = self.map.read().unwrap().get(filename).and_then(|decoders| { - decoders - .iter() - .find_map(|(_, decoder)| decoder.get_decoded_image_in_buf(pts)) - }); - - Ok(frame) + let slots = self.pool.lock()?.slots_of(Path::new(filename)); + for slot in slots { + let image = slot + .lock()? + .as_ref() + .and_then(|decoder| decoder.get_decoded_image_in_buf(pts)); + if image.is_some() { + return Ok(image); + } + } + Ok(None) } #[cfg(not(target_arch = "wasm32"))] @@ -340,94 +452,87 @@ impl VideoDecodersWorker { ctx: &FFramesContext<'_, 'media>, options: &SyncVideoFrameInput, ) -> Result + 'media>>> { - use crate::error::FFramesError; - use fframes_media::FFramesMediaError; - - let resource_name = path - .file_name() - .ok_or_else(|| FFramesError::MediaError(FFramesMediaError::MediaDirectoryProvided))? - .to_string_lossy(); - + // A slot serializes the cursor of this source at this speed. Opening a file, + // decoding a frame or tearing down a hardware session never holds the pool lock. let speed = PlaybackSpeed::new(options.speed); - - // It is important to lock it for the whole duration of he function to - // avoid any parallel encoder creation. - let mut map_guard = self.map.write()?; - // Every speed gets its own decoder, offsets of different speeds point to - // different parts of the file. - let decoders = map_guard.entry(resource_name.into_owned()).or_default(); - let decoder = decoder_for_speed(decoders, speed, || unsafe { - fframes_media::FFmpegDecoder::new(path, ctx.time_base.fps, self.buffer_size) - .map(|decoder| decoder.with_playback_speed(speed)) - })?; + let slot = self.pool.lock()?.slot(path, speed); + let mut guard = slot.lock()?; unsafe { + let decoder = match guard.as_mut() { + Some(decoder) => decoder, + None => guard.insert( + fframes_media::FFmpegDecoder::new_with_mode( + path, + ctx.time_base.fps, + self.buffer_size, + self.mode, + )? + .with_playback_speed(speed), + ), + }; let has_frame = if options.looping { decoder.decode_looping(offset)? } else { decoder.decode_up_to(offset)? }; - Ok(has_frame.then(|| decoder.get_raw_frame())) + Ok(has_frame.then(|| decoder.retained_frame()).transpose()?) } } } -/// The decoder of `speed`, made with `create` if there is none. The most recently used -/// decoder is first, and at most `MAX_DECODERS_PER_VIDEO` are kept. -#[cfg(not(target_arch = "wasm32"))] -fn decoder_for_speed( - decoders: &mut Vec<(PlaybackSpeed, D)>, - speed: PlaybackSpeed, - create: impl FnOnce() -> std::result::Result, -) -> std::result::Result<&mut D, E> { - if let Some(index) = decoders.iter().position(|(s, _)| *s == speed) { - decoders[..=index].rotate_right(1); - } else { - decoders.insert(0, (speed, create()?)); - decoders.truncate(MAX_DECODERS_PER_VIDEO); - } - Ok(&mut decoders[0].1) -} - #[cfg(all(test, not(target_arch = "wasm32")))] -mod decoder_cache_tests { +mod decoder_pool_tests { use super::*; - fn use_speed(decoders: &mut Vec<(PlaybackSpeed, f32)>, speed: f32, created: &mut usize) { - decoder_for_speed(decoders, PlaybackSpeed::new(speed), || { - *created += 1; - Ok::<_, ()>(speed) - }) - .unwrap(); - } - #[test] - fn speeds_that_round_to_the_same_value_share_a_decoder() { - let (mut decoders, mut created) = (Vec::new(), 0); - for i in 0..100 { - use_speed(&mut decoders, 1.0 + i as f32 * 0.000_001, &mut created); + fn clips_age_out_without_evicting_active_layers_or_in_flight_work() { + let mut pool = DecoderPool::default(); + let first = Path::new("first/clip.mp4"); + let second = Path::new("second/clip.mp4"); + let normal = PlaybackSpeed::NORMAL; + let held = pool.slot(first, normal); + assert!(!Arc::ptr_eq(&held, &pool.slot(second, normal))); + for _ in 0..24 { + // Overlapping sources in a composite must all stay warm. + for i in 0..12 { + pool.slot(Path::new(&format!("layer-{i}.mp4")), normal); + } + drop(pool.finish_frame(5)); } - assert_eq!((decoders.len(), created), (1, 1)); + assert!(!pool.sources.contains_key(second)); + assert!(Arc::ptr_eq(&held, &pool.slot(first, normal))); + assert_eq!(pool.sources.len(), 13); + drop(held); + for _ in 0..9 { + drop(pool.finish_frame(5)); + } + assert!(pool.sources.is_empty()); } #[test] - fn least_recently_used_decoders_are_closed() { - let (mut decoders, mut created) = (Vec::new(), 0); - for i in 0..100 { - use_speed(&mut decoders, 1.0 + i as f32 * 0.1, &mut created); + fn a_file_keeps_one_decoder_per_speed_and_closes_the_idle_one_used_least_recently() { + let mut pool = DecoderPool::default(); + let clip = Path::new("clip.mp4"); + let normal = pool.slot(clip, PlaybackSpeed::new(1.0)); + // Speeds that round to the same thousandth share a decoder. + assert!(Arc::ptr_eq( + &normal, + &pool.slot(clip, PlaybackSpeed::new(1.000_1)) + )); + let held = pool.slot(clip, PlaybackSpeed::new(2.0)); + for speed in [0.5, 1.5] { + pool.slot(clip, PlaybackSpeed::new(speed)); } - assert_eq!((decoders.len(), created), (MAX_DECODERS_PER_VIDEO, 100)); - - // A used decoder moves to the front and is kept. - use_speed(&mut decoders, 1.0 + 97.0 * 0.1, &mut created); - use_speed(&mut decoders, 0.25, &mut created); - use_speed(&mut decoders, 0.5, &mut created); - use_speed(&mut decoders, 1.0 + 97.0 * 0.1, &mut created); - assert_eq!(created, 102); - let speeds: Vec<_> = decoders.iter().map(|(s, _)| s.thousandths()).collect(); - assert_eq!(speeds, [10_700, 500, 250, 10_900]); + assert_eq!(pool.sources[clip].len(), MAX_DECODERS_PER_VIDEO); + drop(normal); + // A fifth speed closes the idle decoder used least recently, not the one in use. + pool.slot(clip, PlaybackSpeed::new(3.0)); + let speeds: Vec<_> = pool.sources[clip] + .iter() + .map(|(speed, _)| speed.thousandths()) + .collect(); + assert_eq!(speeds, [2000, 500, 1500, 3000]); + assert_eq!(pool.slots_of(Path::new("clip.mp4")).len(), 4); + drop(held); } } - -// These implementations are already thread-safe due to RwLock -unsafe impl Send for VideoDecodersWorker {} -unsafe impl Sync for VideoDecodersWorker {}