From 39b0222f48fddfb49708f6033347d5a6a161c9ae Mon Sep 17 00:00:00 2001 From: dmtrKovalenko Date: Tue, 27 Oct 2026 16:33:19 -0700 Subject: [PATCH 01/21] feat(player): Add a timeline to the native preview Show scenes, filmstrip thumbnails, and audio waveforms in a scrubbable timeline. Place the timeline below landscape videos and beside portrait videos. Keep the frame counter beside the clock in landscape Studio. --- README.md | 5 + fframes-native-player/src/app.rs | 202 ++++- fframes-native-player/src/controls.rs | 125 +-- fframes-native-player/src/frame-f.svg | 1 + fframes-native-player/src/lib.rs | 46 +- fframes-native-player/src/options.rs | 5 +- fframes-native-player/src/presenter/mod.rs | 68 +- fframes-native-player/src/scheduler.rs | 63 +- fframes-native-player/src/studio.rs | 978 +++++++++++++++++++++ 9 files changed, 1392 insertions(+), 101 deletions(-) create mode 100644 fframes-native-player/src/frame-f.svg create mode 100644 fframes-native-player/src/studio.rs diff --git a/README.md b/README.md index 1ede72cf..10e339c8 100644 --- a/README.md +++ b/README.md @@ -339,6 +339,11 @@ cargo fframes new my-video cd my-video && cargo run --release -- preview ``` +The native preview places the timeline below landscape and square videos and to the left of portrait +videos. Drag the filmstrip to scrub, or click a colored scene marker to pause at its first frame. +Use `[` and `]` to jump between scene starts, `,` and `.` to step one frame, and `Space` to play or pause. +The dock shows the elapsed time, total duration, and frame number. Press `T` to switch to Focus mode. + `cargo fframes new` takes `--template single-scene|multi-scene`, `--format landscape|portrait|square|uhd`, `--fps`, `--title` and `--backend`. On macOS and Linux (arm64, x86_64) the first build downloads prebuilt Skia and ffmpeg libraries and takes under a minute on a fast machine; other targets and feature combinations diff --git a/fframes-native-player/src/app.rs b/fframes-native-player/src/app.rs index 94ddc630..2ba6d265 100644 --- a/fframes-native-player/src/app.rs +++ b/fframes-native-player/src/app.rs @@ -5,7 +5,7 @@ use std::time::{Duration, Instant}; use fframes::usvgr; use winit::application::ApplicationHandler; use winit::dpi::{LogicalSize, PhysicalPosition}; -use winit::event::{ElementState, KeyEvent, MouseButton, WindowEvent}; +use winit::event::{ElementState, KeyEvent, MouseButton, MouseScrollDelta, WindowEvent}; use winit::event_loop::{ActiveEventLoop, ControlFlow}; use winit::keyboard::{Key, NamedKey}; use winit::window::{CursorIcon, Fullscreen, Window, WindowId}; @@ -15,13 +15,17 @@ use crate::error::PlayerError; use crate::options::PlayerBackend; use crate::presenter::Presenter; use crate::scheduler::Scheduler; +use crate::studio::{Studio, StudioLayout, StudioScroll}; +use fframes_skia_renderer::skia_safe::Point; /// The control bar hides after this long without pointer movement while playing. const CONTROLS_IDLE_TIMEOUT: Duration = Duration::from_millis(2500); /// Sent by the frame workers whenever a frame becomes available. -#[derive(Debug, Clone, Copy)] -pub(crate) struct FrameReady; +pub(crate) enum FrameReady { + Playback, + Thumbnail(usize, fframes_skia_renderer::skia_safe::Image), +} /// Playback clock in player positions (see `scheduler`), driven by wall time. struct Clock { @@ -61,8 +65,10 @@ impl Clock { } pub(crate) struct AppConfig<'a> { + pub studio: Studio, pub title: &'a str, pub window_size: LogicalSize, + pub fit_portrait_to_screen: bool, pub backend: PlayerBackend, pub vsync: bool, pub fps: usize, @@ -95,7 +101,12 @@ pub(crate) struct App<'s> { cursor_inside: bool, last_pointer_activity: Instant, hovered: Option, - seeking_with_mouse: bool, + drag: Option, + scene_press_axis: f32, + /// The newest destination under the pointer while a native frame is being prepared. + scrub_target: Option, + studio_open: bool, + scroll: StudioScroll, stats_since: Instant, /// New frames drawn into the window since `stats_since`. Frames taken from the @@ -137,7 +148,11 @@ impl<'s> App<'s> { cursor_inside: false, last_pointer_activity: now, hovered: None, - seeking_with_mouse: false, + drag: None, + scene_press_axis: 0., + scrub_target: None, + studio_open: true, + scroll: StudioScroll::default(), stats_since: now, frames_shown: 0, presented: None, @@ -166,6 +181,7 @@ impl<'s> App<'s> { /// Restarts playback at `position`, all the prepared frames are discarded. fn seek_to(&mut self, position: u64, playing: bool) { + self.scrub_target = None; self.clock.set(position, playing); self.scheduler.seek(position, self.looping); @@ -175,13 +191,17 @@ impl<'s> App<'s> { } } + fn playhead(&self) -> usize { + self.scrub_target.unwrap_or_else(|| self.current_frame()) + } + fn seek_to_frame(&mut self, frame: i64) { let frame = frame.clamp(0, self.last_frame() as i64) as u64; self.seek_to(frame, self.clock.is_playing()); } fn toggle_play(&mut self) { - let frame = self.current_frame(); + let frame = self.playhead(); if self.clock.is_playing() { self.seek_to(frame as u64, false); } else if !self.looping && frame >= self.last_frame() { @@ -203,8 +223,15 @@ impl<'s> App<'s> { } if let Some(frame) = self.scheduler.take(position) { + let next_frame = self.scheduler.frame_of(frame.0, self.looping); self.current = Some(frame); self.needs_redraw = true; + if let Some(target) = self.scrub_target.take() + && target != next_frame + { + self.seek_to(target as u64, false); + self.scrub_target = Some(target); + } } let controls_visible = self.controls_should_be_visible(); @@ -269,7 +296,7 @@ impl<'s> App<'s> { return; } - let frame = self.current_frame() as i64; + let frame = self.playhead() as i64; let second = self.config.fps as i64; match event.logical_key.as_ref() { Key::Named(NamedKey::Space) => self.toggle_play(), @@ -295,6 +322,13 @@ impl<'s> App<'s> { self.controls_enabled = !self.controls_enabled; self.needs_redraw = true; } + Key::Character("t") => self.toggle_studio(), + Key::Character("[" | "]") => { + let next = event.logical_key == Key::Character("]".into()); + if let Some(target) = self.config.studio.adjacent_scene(frame as usize, next) { + self.seek_to_frame(target as i64); + } + } Key::Character("f") => self.toggle_fullscreen(), Key::Character(digit) if digit.len() == 1 && digit.as_bytes()[0].is_ascii_digit() => { let tenth = i64::from(digit.as_bytes()[0] - b'0'); @@ -310,6 +344,24 @@ impl<'s> App<'s> { self.seek_to(frame, false); } + fn studio_layout(&self) -> Option<&StudioLayout> { + self.presenter.as_ref()?.studio_layout.as_ref() + } + + fn pointer(&self) -> Point { + let scale = self.window.as_ref().map_or(1., |w| w.scale_factor()); + let pos = self.cursor.to_logical::(scale); + Point::new(pos.x, pos.y) + } + + fn toggle_studio(&mut self) { + self.studio_open = !self.studio_open; + self.drag = None; + self.hovered = None; + self.last_pointer_activity = Instant::now(); + self.needs_redraw = true; + } + fn toggle_fullscreen(&mut self) { let Some(window) = &self.window else { return }; let fullscreen = match window.fullscreen() { @@ -329,27 +381,54 @@ impl<'s> App<'s> { fn controls_should_be_visible(&self) -> bool { self.controls_enabled && (!self.clock.is_playing() - || self.seeking_with_mouse + || self.drag.is_some() || (self.cursor_inside && self.hovered.is_some()) || self.last_pointer_activity.elapsed() < CONTROLS_IDLE_TIMEOUT) } fn control_under_cursor(&self) -> Option { - if !self.controls_visible { - return None; + let point = self.pointer(); + if self.studio_open { + return self.studio_layout()?.hit(point, &self.config.studio); } - - self.presenter.as_ref().and_then(|presenter| { - presenter + if self.controls_visible + && let Some(control) = self + .presenter + .as_ref()? .controls_layout() .hit(self.cursor.x as f32, self.cursor.y as f32) - }) + { + return Some(control); + } + let layout = self.studio_layout()?; + (self.controls_visible && layout.focus_hit(point)).then_some(Control::Focus) } fn seek_to_cursor(&mut self) { let Some(presenter) = &self.presenter else { return; }; + if self.drag == Some(Control::Timeline) { + if let Some(ui) = self.studio_layout() { + let frame = + ui.frame_at(ui.time_axis(self.pointer()), self.config.duration_in_frames); + // Finish the in-flight frame before requesting the newest pointer + // position. Starting a fresh generation on every move can starve it. + if self.scrub_target.is_none() { + if !self.clock.is_playing() + && self.current.is_some() + && frame == self.current_frame() + && self.clock.position() == frame as u64 + { + return; + } + self.seek_to(frame as u64, false); + } + self.scrub_target = Some(frame); + self.needs_redraw = true; + } + return; + } let progress = f64::from( presenter .controls_layout() @@ -364,7 +443,14 @@ impl<'s> App<'s> { self.cursor_inside = true; self.last_pointer_activity = Instant::now(); - if self.seeking_with_mouse { + if matches!(self.drag, Some(Control::Scene(_))) + && self.studio_layout().is_some_and(|ui| { + (ui.time_axis(self.pointer()) - self.scene_press_axis).abs() >= 3. + }) + { + self.drag = Some(Control::Timeline); + } + if matches!(self.drag, Some(Control::Seek | Control::Timeline)) { self.seek_to_cursor(); } @@ -376,6 +462,12 @@ impl<'s> App<'s> { if let Some(window) = &self.window { window.set_cursor(match hovered { None | Some(Control::Bar) => CursorIcon::Default, + Some(Control::Timeline) + if self.studio_layout().is_some_and(|ui| ui.portrait) => + { + CursorIcon::NsResize + } + Some(Control::Timeline) => CursorIcon::EwResize, Some(_) => CursorIcon::Pointer, }); } @@ -394,16 +486,23 @@ impl<'s> App<'s> { } fn on_pointer_pressed(&mut self) { - let frame = self.current_frame() as i64; + let frame = self.playhead() as i64; let second = self.config.fps as i64; match self.control_under_cursor() { Some(Control::Back) => self.seek_to_frame(frame - second), Some(Control::Forward) => self.seek_to_frame(frame + second), - Some(Control::Seek) => { - self.seeking_with_mouse = true; + Some(control @ (Control::Seek | Control::Timeline)) => { + self.drag = Some(control); self.seek_to_cursor(); } + Some(control @ Control::Scene(_)) => { + if let Some(ui) = self.studio_layout() { + self.scene_press_axis = ui.time_axis(self.pointer()); + } + self.drag = Some(control); + } + Some(Control::Focus) => self.toggle_studio(), Some(Control::Loop) => { self.looping = !self.looping; let playing = self.clock.is_playing(); @@ -417,10 +516,18 @@ impl<'s> App<'s> { self.needs_redraw = true; } + fn on_pointer_released(&mut self) { + if let Some(control @ Control::Scene(start)) = self.drag.take() + && self.control_under_cursor() == Some(control) + { + self.step(start as i64); + } + } + fn redraw(&mut self) -> Result<(), PlayerError> { let controls = ControlsState { visible: self.controls_visible, - frame: self.current_frame(), + frame: self.playhead(), duration_in_frames: self.config.duration_in_frames, fps: self.config.fps, playing: self.clock.is_playing(), @@ -430,7 +537,13 @@ impl<'s> App<'s> { }; if let Some(presenter) = self.presenter.as_mut() { - presenter.present(self.current.as_ref().map(|(_, tree)| tree), &controls)?; + presenter.present( + self.current.as_ref().map(|(_, tree)| tree), + &controls, + &self.config.studio, + &self.scroll, + self.studio_open, + )?; let position = self.current.as_ref().map(|(position, _)| *position); if position.is_some() && position != self.presented { @@ -454,6 +567,24 @@ impl ApplicationHandler for App<'_> { return; } + if self.config.fit_portrait_to_screen + && let Some(monitor) = event_loop + .primary_monitor() + .or_else(|| event_loop.available_monitors().next()) + { + let screen = monitor.size().to_logical::(monitor.scale_factor()); + // Leave room for the window decorations and desktop controls. + let height = f64::from(self.config.window_size.height) + .min(screen.height - 72.) + .max(1.); + let width = height * f64::from(self.config.studio.video_aspect) + + f64::from(crate::studio::PORTRAIT_SIDEBAR_WIDTH); + self.config.window_size = LogicalSize::new( + width.min(screen.width - 24.).max(1.).round() as u32, + height.round() as u32, + ); + } + let attributes = Window::default_attributes() .with_title(self.config.title) .with_inner_size(self.config.window_size); @@ -470,6 +601,7 @@ impl ApplicationHandler for App<'_> { self.config.vsync, self.config.background, duration_label, + &self.config.studio, ) { Ok(presenter) => self.presenter = Some(presenter), Err(err) => return self.fail(event_loop, err), @@ -479,7 +611,11 @@ impl ApplicationHandler for App<'_> { self.needs_redraw = true; } - fn user_event(&mut self, _event_loop: &ActiveEventLoop, _event: FrameReady) { + fn user_event(&mut self, _event_loop: &ActiveEventLoop, event: FrameReady) { + if let FrameReady::Thumbnail(slot, image) = event { + self.config.studio.thumbnails[slot] = Some(image); + self.needs_redraw = true; + } self.tick(); } @@ -499,6 +635,28 @@ impl ApplicationHandler for App<'_> { } WindowEvent::KeyboardInput { event, .. } => self.handle_key(event_loop, event), WindowEvent::CursorMoved { position, .. } => self.on_pointer_moved(position), + WindowEvent::MouseWheel { delta, .. } if self.studio_open && self.cursor_inside => { + let point = self.pointer(); + let dy = match delta { + MouseScrollDelta::LineDelta(_, dy) => f64::from(dy), + MouseScrollDelta::PixelDelta(delta) => delta.y, + }; + if let Some(ui) = self + .presenter + .as_ref() + .and_then(|p| p.studio_layout.as_ref()) + { + ui.scroll( + &self.config.studio, + &mut self.scroll, + point, + isize::from(dy < 0.) - isize::from(dy > 0.), + ); + self.hovered = self.control_under_cursor(); + self.needs_redraw = true; + } + } + WindowEvent::Focused(false) => self.drag = None, WindowEvent::CursorLeft { .. } => { self.cursor_inside = false; if self.hovered.take().is_some() { @@ -511,7 +669,7 @@ impl ApplicationHandler for App<'_> { .. } => match state { ElementState::Pressed => self.on_pointer_pressed(), - ElementState::Released => self.seeking_with_mouse = false, + ElementState::Released => self.on_pointer_released(), }, _ => {} } diff --git a/fframes-native-player/src/controls.rs b/fframes-native-player/src/controls.rs index bd026f95..e7a03c1b 100644 --- a/fframes-native-player/src/controls.rs +++ b/fframes-native-player/src/controls.rs @@ -16,6 +16,9 @@ pub(crate) enum Control { PlayPause, Forward, Seek, + Timeline, + Scene(usize), + Focus, Loop, Fullscreen, /// The bar background, swallows clicks so they do not toggle playback. @@ -86,6 +89,7 @@ pub(crate) struct Controls { regular: Typeface, medium: Typeface, icons: IconPaths, + pub(crate) frame_mark: Path, } struct IconPaths { @@ -120,6 +124,8 @@ impl ControlsLayout { height: f32, scale: f32, ) -> Self { + // Keep both time labels and icon hitboxes apart in a narrow window. + let scale = scale.min(width / 400.); let px = |logical: f32| logical * scale; let font = controls.font(true, scale); let time_width = tabular_width(&font, duration_label); @@ -221,9 +227,19 @@ impl Controls { } }; + let svg = fframes::roxmltree::Document::parse(include_str!("frame-f.svg")) + .expect("valid frame mark SVG"); + let frame_mark = svg + .root_element() + .first_element_child() + .and_then(|path| path.attribute("d")) + .and_then(Path::from_svg) + .expect("valid frame mark path"); + Self { regular: typeface(font_style::Weight::NORMAL), medium: typeface(font_style::Weight::MEDIUM), + frame_mark, icons: IconPaths { play: icon(icons::PLAY), pause: icon(icons::PAUSE), @@ -236,7 +252,7 @@ impl Controls { } } - fn font(&self, medium: bool, scale: f32) -> Font { + pub(crate) fn font(&self, medium: bool, scale: f32) -> Font { let typeface = if medium { &self.medium } else { &self.regular }; let mut font = Font::from_typeface(typeface, FONT_SIZE * scale); font.set_subpixel(true); @@ -272,53 +288,15 @@ impl Controls { let hovered = |control| state.hovered == Some(control); - self.ghost_button( - canvas, - layout.back, - &self.icons.back, - hovered(Control::Back), - false, - scale, - ); - self.primary_button( - canvas, - layout.play, - if state.playing { - &self.icons.pause - } else { - &self.icons.play - }, - hovered(Control::PlayPause), - scale, - ); - self.ghost_button( - canvas, - layout.forward, - &self.icons.forward, - hovered(Control::Forward), - false, - scale, - ); - self.ghost_button( - canvas, - layout.looping, - &self.icons.looping, - hovered(Control::Loop), - state.looping, - scale, - ); - self.ghost_button( - canvas, - layout.fullscreen, - if state.fullscreen { - &self.icons.collapse - } else { - &self.icons.expand - }, - hovered(Control::Fullscreen), - false, - scale, - ); + for (control, rect) in [ + (Control::Back, layout.back), + (Control::PlayPause, layout.play), + (Control::Forward, layout.forward), + (Control::Loop, layout.looping), + (Control::Fullscreen, layout.fullscreen), + ] { + self.draw_button(canvas, control, rect, state, scale); + } let mut text = paint.clone(); text.set_color(TEXT); @@ -369,6 +347,46 @@ impl Controls { } } + pub(crate) fn draw_button( + &self, + canvas: &Canvas, + control: Control, + rect: Rect, + state: &ControlsState, + scale: f32, + ) { + let hovered = state.hovered == Some(control); + let icon = match control { + Control::PlayPause => { + let icon = if state.playing { + &self.icons.pause + } else { + &self.icons.play + }; + return self.primary_button(canvas, rect, icon, hovered, scale); + } + Control::Back => &self.icons.back, + Control::Forward => &self.icons.forward, + Control::Loop => &self.icons.looping, + Control::Fullscreen => { + if state.fullscreen { + &self.icons.collapse + } else { + &self.icons.expand + } + } + _ => return, + }; + self.ghost_button( + canvas, + rect, + icon, + hovered, + control == Control::Loop && state.looping, + scale, + ); + } + fn ghost_button( &self, canvas: &Canvas, @@ -427,7 +445,7 @@ fn digit_width(font: &Font) -> f32 { .fold(0., f32::max) } -fn tabular_width(font: &Font, text: &str) -> f32 { +pub(crate) fn tabular_width(font: &Font, text: &str) -> f32 { let digit = digit_width(font); text.chars() .map(|char| { @@ -440,7 +458,13 @@ fn tabular_width(font: &Font, text: &str) -> f32 { .sum() } -fn draw_tabular(canvas: &Canvas, text: &str, origin: Point, font: &Font, paint: &Paint) { +pub(crate) fn draw_tabular( + canvas: &Canvas, + text: &str, + origin: Point, + font: &Font, + paint: &Paint, +) -> f32 { let digit = digit_width(font); let mut x = origin.x; for char in text.chars() { @@ -454,6 +478,7 @@ fn draw_tabular(canvas: &Canvas, text: &str, origin: Point, font: &Font, paint: x += width; } } + x } /// `mm:ss`, or `h:mm:ss` from an hour on, like the web editor. diff --git a/fframes-native-player/src/frame-f.svg b/fframes-native-player/src/frame-f.svg new file mode 100644 index 00000000..f3b7bcea --- /dev/null +++ b/fframes-native-player/src/frame-f.svg @@ -0,0 +1 @@ + diff --git a/fframes-native-player/src/lib.rs b/fframes-native-player/src/lib.rs index 9de36745..558e848c 100644 --- a/fframes-native-player/src/lib.rs +++ b/fframes-native-player/src/lib.rs @@ -20,8 +20,11 @@ //! - `h`/`l` or `←`/`→`: seek one second back/forward //! - `j`/`k` or `↓`/`↑` (also `,`/`.`): step one frame back/forward, pauses //! - `g`/`G` or `Home`/`End`: first/last frame, `0`-`9`: jump to 0-90% -//! - the control bar (shown while paused or when the pointer moves): back, play/pause, -//! forward, a seek slider, looping and full screen, `b` hides it +//! - Studio keeps playback controls above the timeline. Focus mode shows a floating +//! dock with a seek slider while paused or on pointer activity; `b` hides the dock. +//! - click a scene to jump to its start, or drag across the timeline to scrub +//! - `[` / `]`: previous/next scene start, `t`: switch between the studio and video-only views +//! - scroll the audio lanes to browse overlapping clips //! - `r`: toggle looping, `f`: full screen, `q`/`Esc`: quit //! //! [`play`] must be called from the main thread and only once per process (a winit @@ -36,6 +39,7 @@ mod icons; mod options; mod presenter; mod scheduler; +mod studio; pub use error::*; pub use options::*; @@ -125,25 +129,44 @@ pub fn play<'a, 'media: 'a, TVideo: Video + Sync>( // The window owns the Skia context, so the trees of `render_svgr` render on the CPU. let svgr_images = fframes::SvgrImages::new(Some(Box::new(fframes::cpu::CpuSvgrRasterizer))); + let window_size = initial_window_size::(options); + let aspect = TVideo::WIDTH as f32 / TVideo::HEIGHT.max(1) as f32; + let tiles = if aspect < 1. { + window_size.height as f32 / 60. + } else { + window_size.width as f32 / (60. * aspect) + }; + let thumbnail_count = (tiles.ceil() as usize) + .clamp(1, 32) + .min(timeline.duration_in_frames); let scheduler = Scheduler::new( timeline.duration_in_frames, options.prefetch_frames, options.looping, + thumbnail_count, ); let proxy = Mutex::new(event_loop.create_proxy()); - let notify_frame_ready = || { + let notify_frame_ready = |event| { let _ = proxy .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) - .send_event(FrameReady); + .send_event(event); }; let mut app = App::new( AppConfig { title: options.title, - window_size: initial_window_size::(options), + window_size, + fit_portrait_to_screen: aspect < 1. && options.window_size.is_none(), backend: options.backend, vsync: options.vsync, + studio: studio::Studio::new( + time_base, + &timeline, + options.media, + thumbnail_count, + aspect, + ), fps: TVideo::FPS, duration_in_frames: timeline.duration_in_frames, background: TVideo::BACKGROUND_COLOR, @@ -266,6 +289,15 @@ fn initial_window_size(options: &PlayerOptions) -> LogicalSize { pub default_font: &'a str, /// Window title, defaults to "fframes". pub title: &'a str, - /// Initial window size in logical pixels. Defaults to the video size, scaled down to - /// fit 1280x720. + /// Initial window size in logical pixels. Landscape defaults fit within 1280×720. + /// Portrait defaults fit up to 960 pixels tall with a timeline on the left, subject + /// to the size of the display. pub window_size: Option<(u32, u32)>, pub backend: PlayerBackend, /// Wait for the display refresh to show a frame. With `false` a frame is shown as soon diff --git a/fframes-native-player/src/presenter/mod.rs b/fframes-native-player/src/presenter/mod.rs index ece7f6c1..0cdf72f6 100644 --- a/fframes-native-player/src/presenter/mod.rs +++ b/fframes-native-player/src/presenter/mod.rs @@ -18,6 +18,7 @@ use winit::window::Window; use crate::controls::{Controls, ControlsLayout, ControlsState}; use crate::error::PlayerResult; use crate::options::PlayerBackend; +use crate::studio::{Studio, StudioLayout, StudioScroll}; /// The images of the window that Skia draws into. trait WindowTarget { @@ -49,6 +50,9 @@ pub(crate) struct Presenter { layout: ControlsLayout, /// The longest time label, sizes the time slots of the control bar. duration_label: String, + pub studio_layout: Option, + audio_rows: usize, + video_aspect: f32, } impl Presenter { @@ -58,6 +62,7 @@ impl Presenter { vsync: bool, background: fframes::Color, duration_label: String, + model: &Studio, ) -> PlayerResult { let target = create_target(&window, backend, vsync)?; let controls = Controls::new(); @@ -78,6 +83,9 @@ impl Presenter { controls, layout, duration_label, + studio_layout: None, + audio_rows: model.audio_rows(), + video_aspect: model.video_aspect, }; presenter.resize()?; @@ -102,6 +110,13 @@ impl Presenter { } self.target.resize(size.width, size.height)?; + let logical = size.to_logical::(self.window.scale_factor()); + self.studio_layout = StudioLayout::new( + logical.width, + logical.height, + self.audio_rows, + self.video_aspect, + ); self.layout = ControlsLayout::new( &self.controls, &self.duration_label, @@ -118,26 +133,53 @@ impl Presenter { &mut self, tree: Option<&usvgr::Tree>, controls: &ControlsState, + model: &Studio, + scroll: &StudioScroll, + studio_open: bool, ) -> PlayerResult<()> { + let pixel_scale = self.window.scale_factor() as f32; let Self { render_cache, target, background, controls: bar, layout, + studio_layout, .. } = self; target.present(&mut |canvas| { let size = canvas.base_layer_size(); let (width, height) = (size.width as f32, size.height as f32); - canvas.clear(Color::BLACK); + let view = studio_layout.as_ref().filter(|_| studio_open); + let viewport = view.map_or_else( + || Rect::from_wh(width, height), + |ui| { + let stage = ui.stage; + Rect::from_ltrb( + stage.left * pixel_scale, + stage.top * pixel_scale, + stage.right * pixel_scale, + stage.bottom * pixel_scale, + ) + }, + ); + canvas.clear(if view.is_some() { + Color::from_rgb(14, 14, 14) + } else { + Color::BLACK + }); if let Some(tree) = tree { let video = tree.size(); - let scale = (width / video.width()).min(height / video.height()); - let offset_x = (width - video.width() * scale) / 2.; - let offset_y = (height - video.height() * scale) / 2.; + let scale = + (viewport.width() / video.width()).min(viewport.height() / video.height()); + let offset_x = viewport.center_x() - video.width() * scale / 2.; + let offset_y = if view.is_some_and(|ui| ui.portrait) { + viewport.top + } else { + viewport.center_y() - video.height() * scale / 2. + }; canvas.save(); canvas.translate((offset_x, offset_y)); @@ -152,7 +194,23 @@ impl Presenter { canvas.restore(); } - bar.draw(canvas, layout, controls); + if let Some(ui) = studio_layout.as_ref() { + canvas.save(); + canvas.scale((pixel_scale, pixel_scale)); + if studio_open { + ui.draw(canvas, bar, controls, model, scroll); + } else if controls.visible { + ui.draw_restore( + canvas, + bar, + controls.hovered == Some(crate::controls::Control::Focus), + ); + } + canvas.restore(); + } + if !studio_open { + bar.draw(canvas, layout, controls); + } }) } } diff --git a/fframes-native-player/src/scheduler.rs b/fframes-native-player/src/scheduler.rs index 63aa0bb1..3a9f9f44 100644 --- a/fframes-native-player/src/scheduler.rs +++ b/fframes-native-player/src/scheduler.rs @@ -7,6 +7,7 @@ use std::collections::BTreeMap; use std::sync::{Condvar, Mutex, MutexGuard}; +use crate::{app::FrameReady, studio::thumbnail}; use fframes::{FFramesContext, TextCache, Video, VideoDecodersWorker, usvgr}; pub(crate) struct Scheduler { @@ -14,6 +15,7 @@ pub(crate) struct Scheduler { workers_wakeup: Condvar, duration: u64, lookahead: u64, + thumbnails: usize, } struct State { @@ -28,16 +30,26 @@ struct State { looping: bool, ready: BTreeMap, shutdown: bool, + next_thumbnail: usize, } struct Job { - position: u64, - epoch: u64, frame: usize, + kind: JobKind, +} + +enum JobKind { + Playback { position: u64, epoch: u64 }, + Thumbnail(usize), } impl Scheduler { - pub(crate) fn new(duration_in_frames: usize, lookahead: usize, looping: bool) -> Self { + pub(crate) fn new( + duration_in_frames: usize, + lookahead: usize, + looping: bool, + thumbnails: usize, + ) -> Self { Self { state: Mutex::new(State { epoch: 0, @@ -47,10 +59,12 @@ impl Scheduler { looping, ready: BTreeMap::new(), shutdown: false, + next_thumbnail: 0, }), workers_wakeup: Condvar::new(), duration: duration_in_frames.max(1) as u64, lookahead: lookahead.max(1) as u64, + thumbnails, } } @@ -126,9 +140,23 @@ impl Scheduler { state.next += 1; return Some(Job { - position, - epoch: state.epoch, frame: self.frame_of(position, state.looping), + kind: JobKind::Playback { + position, + epoch: state.epoch, + }, + }); + } + + // Sample the overview once, only after all due playback jobs have been assigned. + if state.next_thumbnail < self.thumbnails { + let slot = state.next_thumbnail; + state.next_thumbnail += 1; + let frame = ((2 * slot + 1) as u128 * u128::from(self.duration) + / (2 * self.thumbnails) as u128) as usize; + return Some(Job { + frame, + kind: JobKind::Thumbnail(slot), }); } @@ -139,15 +167,15 @@ impl Scheduler { } } - fn complete(&self, job: &Job, tree: usvgr::Tree) -> bool { + fn complete(&self, position: u64, epoch: u64, tree: usvgr::Tree) -> bool { let mut state = self.lock(); - let is_stale = state.epoch != job.epoch + let is_stale = state.epoch != epoch || state .displayed - .is_some_and(|displayed| job.position <= displayed); + .is_some_and(|displayed| position <= displayed); if !is_stale { - state.ready.insert(job.position, tree); + state.ready.insert(position, tree); } !is_stale @@ -162,7 +190,7 @@ impl Scheduler { usvg_options: &usvgr::Options<'_>, font_db: &usvgr::fontdb::Database, video_decoders: VideoDecodersWorker, - on_frame_ready: &(dyn Fn() + Sync), + on_frame_ready: &(dyn Fn(FrameReady) + Sync), ) { // Text caches are `Rc` based and therefore owned by the worker. let text_cache = TextCache::new(100); @@ -184,13 +212,18 @@ impl Scheduler { Ok(svgr.into_svg_tree(usvg_options, &mut converter_cache, font_db)?) }); - match tree { - Ok(tree) => { - if self.complete(&job, tree) { - on_frame_ready(); + match (tree, job.kind) { + (Ok(tree), JobKind::Thumbnail(slot)) => { + if let Some(image) = thumbnail(&tree, TVideo::BACKGROUND_COLOR) { + on_frame_ready(FrameReady::Thumbnail(slot, image)); + } + } + (Ok(tree), JobKind::Playback { position, epoch }) => { + if self.complete(position, epoch, tree) { + on_frame_ready(FrameReady::Playback); } } - Err(err) => eprintln!( + (Err(err), _) => eprintln!( "fframes player: failed to render frame {}: {err}", job.frame ), diff --git a/fframes-native-player/src/studio.rs b/fframes-native-player/src/studio.rs new file mode 100644 index 00000000..d09b1dbd --- /dev/null +++ b/fframes-native-player/src/studio.rs @@ -0,0 +1,978 @@ +//! Scene navigation and the clip timeline for the native preview. All layout and pointer +//! coordinates here are logical pixels, regardless of the window's backing scale. + +use std::ops::Range; + +use fframes::{ + AudioData, AudioTimelineSamples, AudioTimelineUnit, MediaProvider, ResolvedRenderingTimeline, + TimeBase, TimelineIndex, media::PreloadedAudioData, +}; +use fframes_skia_renderer::skia_safe::{ + Canvas, Color, Contains, FilterMode, Font, Image, Paint, Path, PathBuilder, Point, RRect, Rect, + SamplingOptions, surfaces, +}; + +use crate::controls::{Control, Controls, ControlsState, draw_tabular, format_time, tabular_width}; + +pub(crate) struct Studio { + pub index: TimelineIndex, + pub video_aspect: f32, + audio: Vec>, + pub thumbnails: Vec>, +} + +struct Clip { + name: String, + frames: Range, + waveform: Option, +} + +impl Studio { + pub fn new<'m>( + time: TimeBase, + timeline: &ResolvedRenderingTimeline<'_, AudioTimelineSamples>, + media: Option<&'m dyn MediaProvider<'m>>, + thumbnail_count: usize, + video_aspect: f32, + ) -> Self { + let index = TimelineIndex::new( + time.fps, + timeline.duration_in_frames, + timeline.scenes.as_ref(), + ); + let mut clips = Vec::new(); + if let Some(audio) = &timeline.audio_map { + for track in audio.tracks() { + clips.push(Clip { + name: track.file.clone(), + waveform: match media.and_then(|m| m.resolve_audio(&track.file)) { + Some(AudioData::Preloaded(data)) => { + let seconds = track.range.end.to_seconds(&time) + - track.range.start.to_seconds(&time); + Some(waveform( + data, + f64::from(track.mix.offset), + seconds * f64::from(track.mix.rate), + )) + } + _ => None, + }, + frames: track.range.start.to_frames(&time)..track.range.end.to_frames(&time), + }); + } + } + Self { + index, + video_aspect, + audio: pack_clips(clips), + thumbnails: vec![None; thumbnail_count], + } + } + + pub fn adjacent_scene(&self, frame: usize, next: bool) -> Option { + let starts = self + .index + .scenes + .iter() + .map(|scene| scene.frames.start) + .filter(|&start| start < self.index.duration_in_frames); + if next { + starts.filter(|&start| start > frame).min() + } else { + starts.filter(|&start| start < frame).max() + } + } + + pub fn audio_rows(&self) -> usize { + self.audio.len() + } +} + +/// Rasterize only the small overview image on the frame worker that owns the decoder. +pub(crate) fn thumbnail(tree: &fframes::usvgr::Tree, bg: fframes::Color) -> Option { + let video = tree.size(); + let scale = (192. / video.width()).min(108. / video.height()); + let size = ( + (video.width() * scale).ceil() as i32, + (video.height() * scale).ceil() as i32, + ); + let mut surface = surfaces::raster_n32_premul(size)?; + let canvas = surface.canvas(); + canvas.clear(Color::from_argb(bg.a, bg.r, bg.g, bg.b)); + canvas.scale((scale, scale)); + fframes_skia_renderer::render::render_tree( + tree, + canvas, + &mut fframes_skia_renderer::render::RenderCache::new(), + ); + Some(surface.image_snapshot()) +} + +// A source envelope in unit coordinates, computed once per clip. Include both channels +// and keep silence past EOF when a cue extends beyond the source. +fn waveform(data: &PreloadedAudioData<'_>, offset: f64, seconds: f64) -> Path { + const BINS: usize = 512; + let rate = f64::from(data.sample_rate); + let (left, right) = data.channels(); + let mut path = PathBuilder::new(); + path.move_to((0., 1.)); + let sample = |i: usize| ((offset + seconds * i as f64 / BINS as f64) * rate) as usize; + for i in 0..BINS { + let start = sample(i).min(left.len()); + let end = sample(i + 1).min(left.len()); + // Bound startup work even for a multi-hour file. + let step = end.saturating_sub(start).div_ceil(64).max(1); + let peak = (start..end) + .step_by(step) + .map(|j| left[j].abs().max(right.get(j).copied().unwrap_or(0.).abs())) + .fold(0., f32::max) + .min(1.); + path.line_to(((i as f32 + 0.5) / BINS as f32, 1. - peak)); + } + path.line_to((1., 1.)).close(); + path.detach() +} + +// Put simultaneous sounds on separate rows while letting disjoint clips share a row. +fn pack_clips(mut clips: Vec) -> Vec> { + clips.sort_by_key(|clip| clip.frames.start); + let mut lanes: Vec> = Vec::new(); + for clip in clips { + if let Some(lane) = lanes.iter_mut().find(|lane| { + lane.last() + .is_some_and(|last| last.frames.end <= clip.frames.start) + }) { + lane.push(clip); + } else { + lanes.push(vec![clip]); + } + } + lanes +} + +const SURFACE: Color = Color::from_rgb(23, 23, 23); +const CANVAS: Color = Color::from_rgb(14, 14, 14); +const LINE: Color = Color::from_rgb(58, 58, 58); +const TEXT: Color = Color::from_rgb(249, 249, 249); +const QUIET: Color = Color::from_rgb(174, 174, 174); +const ORANGE: Color = Color::from_rgb(251, 106, 34); +// Theme.scenes in the legacy web studio. Orange remains reserved for the playhead. +const SCENE_COLORS: [Color; 6] = [ + Color::from_rgb(2, 133, 255), + Color::from_rgb(4, 184, 76), + Color::from_rgb(146, 79, 247), + Color::from_rgb(255, 102, 173), + Color::from_rgb(255, 195, 0), + Color::from_rgb(250, 66, 62), +]; +const AUDIO: Color = Color::from_rgb(45, 45, 45); +const TRANSPORT: f32 = 50.; +const RULER: f32 = 22.; +const SCENE_HEIGHT: f32 = 24.; +const LANE: f32 = 31.; +const MARGIN: f32 = 16.; +pub(crate) const PORTRAIT_SIDEBAR_WIDTH: f32 = 360.; + +#[derive(Default)] +pub(crate) struct StudioScroll { + pub audio: usize, +} + +pub(crate) struct StudioLayout { + pub stage: Rect, + timeline: Rect, + restore: Rect, + transport: Rect, + pub portrait: bool, + audio_slots: usize, + film_height: f32, +} + +impl StudioLayout { + pub fn new(width: f32, height: f32, audio_rows: usize, video_aspect: f32) -> Option { + if width < 320. || height < 240. { + return None; + } + let restore = Rect::from_xywh(width - 112., 10., 96., 30.); + if video_aspect < 1. { + let panel = (width - height * video_aspect) + .clamp((width * 0.38).min(320.), (width * 0.60).min(416.)); + let narrow = panel < 320.; + let bottom = if narrow { height - TRANSPORT } else { height }; + let transport = if narrow { + Rect::from_xywh(0., bottom, width, TRANSPORT) + } else { + Rect::from_xywh(0., 0., panel, TRANSPORT) + }; + return Some(Self { + stage: Rect::from_ltrb(panel, 0., width, bottom), + timeline: Rect::from_ltrb(0., 0., panel, bottom), + transport, + restore, + portrait: true, + audio_slots: audio_rows.min(if panel >= 260. { 3 } else { 1 }), + film_height: 0., + }); + } + let audio_slots = audio_rows.min(if height >= 600. { + 3 + } else { + usize::from(height >= 340.) + }); + let film_height = if height >= 480. { + 58. + } else if height >= 340. { + 36. + } else { + 0. + }; + let timeline_y = height + - TRANSPORT + - RULER + - SCENE_HEIGHT + - film_height + - 16. + - LANE * audio_slots as f32; + Some(Self { + stage: Rect::from_ltrb(0., 0., width, timeline_y), + timeline: Rect::from_ltrb(0., timeline_y, width, height), + restore, + transport: Rect::from_xywh(0., timeline_y, width, TRANSPORT), + portrait: false, + audio_slots, + film_height, + }) + } + + fn tracks(&self) -> Rect { + if self.portrait { + let top = if self.transport.top == 0. { + self.transport.bottom + } else { + 0. + }; + return Rect::from_ltrb( + 6., + top + 18., + self.timeline.right - 6., + self.timeline.bottom - 10., + ); + } + Rect::from_ltrb( + 48., + self.timeline.top + TRANSPORT + 2., + self.timeline.right - MARGIN, + self.timeline.bottom - 5., + ) + } + + // Time runs down the panel. The same columns define the pixels and the hit targets. + fn portrait_columns(&self) -> (Rect, Rect, Rect) { + let track = self.tracks(); + let film_x = (track.width() * 0.14).clamp(30., 46.) + track.left; + let film = Rect::from_xywh( + film_x, + track.top, + (track.width() * 0.16).clamp(22., 52.), + track.height(), + ); + let audio_x = track.right - 18. * self.audio_slots as f32; + let scenes = Rect::from_ltrb(film.right + 6., track.top, audio_x - 3., track.bottom); + let audio = Rect::from_ltrb(audio_x, track.top, track.right, track.bottom); + (film, scenes, audio) + } + + fn draw_portrait_tracks( + &self, + ui: &Ink<'_>, + state: &ControlsState, + model: &Studio, + scroll: &StudioScroll, + ) { + let track = self.tracks(); + let (film, scenes, mut audio) = self.portrait_columns(); + if model.index.scenes.is_empty() { + audio.left = scenes.left; + } + let lane_width = audio.width() / self.audio_slots.max(1) as f32; + let duration = state.duration_in_frames; + let at = |f| self.frame_position(f, duration); + let canvas = ui.canvas; + ui.fill( + Rect::from_xywh(self.timeline.right - 1., 0., 1., self.timeline.bottom), + LINE, + ); + for frame in (0..=duration).step_by(tick_step(duration, state.fps, track.height(), 64.)) { + let y = at(frame); + let label = if duration < state.fps { + format!("{frame}f") + } else { + format_time(frame, state.fps) + }; + ui.clipped_text( + &label, + Rect::from_ltrb(track.left, y - 8., film.left - 3., y + 10.), + y + 4., + 11., + QUIET, + false, + ); + ui.fill(Rect::from_xywh(film.left - 4., y, 4., 1.), LINE); + } + if !model.thumbnails.is_empty() { + let h = film.height() / model.thumbnails.len() as f32; + canvas.save(); + canvas.clip_rrect(RRect::new_rect_xy(film, 4., 4.), None, true); + ui.fill(film, CANVAS); + for (i, image) in model.thumbnails.iter().enumerate() { + if let Some(image) = image { + ui.thumbnail( + image, + Rect::from_xywh(film.left, film.top + i as f32 * h, film.width(), h), + ); + } + } + canvas.restore(); + } + for scene in &model.index.scenes { + let top = at(scene.frames.start); + let bottom = at(scene.frames.end); + if bottom <= top { + continue; + } + let active = scene.frames.contains(&state.frame); + let color = SCENE_COLORS[scene.index % SCENE_COLORS.len()]; + let band = Rect::from_ltrb(scenes.left, top, scenes.right, (bottom - 1.).max(top)); + ui.fill(band, color.with_a(if active { 34 } else { 15 })); + ui.fill( + Rect::from_ltrb(band.left, top, band.left + 3., band.bottom), + color, + ); + if band.height() >= 13. { + let label = Rect::from_ltrb(band.left + 8., top, band.right - 2., band.bottom); + ui.clipped_text( + &scene.name, + label, + top + 12., + 11., + if active { TEXT } else { QUIET }, + active, + ); + } + } + let first = scroll + .audio + .min(model.audio_rows().saturating_sub(self.audio_slots)); + for (i, lane) in model + .audio + .iter() + .skip(first) + .take(self.audio_slots) + .enumerate() + { + let x = audio.left + lane_width * i as f32; + ui.text( + &format!("A{}", first + i + 1), + x, + track.top - 4., + 10., + QUIET, + false, + ); + for clip in lane { + let r = Rect::from_ltrb( + x, + at(clip.frames.start), + x + lane_width - 3., + at(clip.frames.end), + ); + ui.round(r, AUDIO, 3.); + if let Some(path) = &clip.waveform { + canvas.save(); + canvas.translate((r.right - 1., r.top)); + canvas.rotate(90., None); + canvas.scale((r.height(), r.width() - 2.)); + canvas.draw_path(path, &Ink::paint(Color::from_rgb(86, 86, 86))); + canvas.restore(); + } + if r.width() >= 55. && r.height() >= 17. { + let label = Rect::from_ltrb(r.left + 5., r.top, r.right - 3., r.bottom); + ui.clipped_text(&clip.name, label, r.top + 13., 11., TEXT, false); + } + } + } + let y = at(state.frame); + ui.fill( + Rect::from_ltrb(film.left - 2., y - 1., track.right, y + 1.), + ORANGE, + ); + ui.round(Rect::from_xywh(film.left - 4., y - 4., 7., 8.), ORANGE, 2.); + } + + fn buttons(&self) -> [(Control, Rect); 6] { + let width = self.transport.width(); + let narrow = width < 460.; + let size = if narrow { 26. } else { 32. }; + let play_size = if narrow { 30. } else { 36. }; + let gap = if narrow { 2. } else { 5. }; + let focus_width = if narrow { 43. } else { 58. }; + let margin = if narrow { 8. } else { MARGIN }; + let mid = width / 2.; + let y = self.transport.center_y(); + let button = |x, w, h| Rect::from_xywh(x, y - h / 2., w, h); + let play = button(mid - play_size / 2., play_size, play_size); + let focus = button(width - margin - focus_width, focus_width, size); + let full = button(focus.left - gap - size, size, size); + [ + (Control::Back, button(play.left - gap - size, size, size)), + (Control::PlayPause, play), + (Control::Forward, button(play.right + gap, size, size)), + (Control::Loop, button(full.left - gap - size, size, size)), + (Control::Fullscreen, full), + (Control::Focus, focus), + ] + } + + pub fn time_axis(&self, point: Point) -> f32 { + if self.portrait { point.y } else { point.x } + } + + fn time_bounds(&self) -> (f32, f32) { + let track = self.tracks(); + if self.portrait { + (track.top, track.height()) + } else { + (track.left, track.width()) + } + } + + fn frame_position(&self, frame: usize, duration: usize) -> f32 { + let (start, length) = self.time_bounds(); + start + length * (frame.min(duration) as f32 / duration.max(1) as f32) + } + + pub fn frame_at(&self, position: f32, duration: usize) -> usize { + let (start, length) = self.time_bounds(); + let fraction = ((position - start) / length).clamp(0., 1.); + ((fraction * duration as f32).floor() as usize).min(duration.saturating_sub(1)) + } + + pub fn hit(&self, point: Point, model: &Studio) -> Option { + if let Some((control, _)) = self.buttons().into_iter().find(|(_, r)| r.contains(point)) { + return Some(control); + } + let tracks = self.tracks(); + if tracks.contains(point) { + let in_scenes = if self.portrait { + self.portrait_columns().1.contains(point) + } else { + (tracks.top + RULER..tracks.top + RULER + SCENE_HEIGHT).contains(&point.y) + }; + if in_scenes { + let frame = self.frame_at(self.time_axis(point), model.index.duration_in_frames); + if let Some(scene) = model.index.scenes_at(frame).last() { + return Some(Control::Scene(scene.frames.start)); + } + } + return Some(Control::Timeline); + } + (!self.stage.contains(point)).then_some(Control::Bar) + } + + pub fn scroll(&self, model: &Studio, scroll: &mut StudioScroll, point: Point, steps: isize) { + if self.tracks().contains(point) { + scroll.audio = scroll + .audio + .saturating_add_signed(steps) + .min(model.audio_rows().saturating_sub(self.audio_slots)); + } + } + + pub fn focus_hit(&self, point: Point) -> bool { + self.restore.contains(point) + } + + pub fn draw_restore(&self, canvas: &Canvas, controls: &Controls, hovered: bool) { + let ui = Ink { + canvas, + font: controls.font(false, 1.), + medium: controls.font(true, 1.), + }; + ui.round(self.restore, if hovered { LINE } else { SURFACE }, 7.); + ui.text( + "Studio · T", + self.restore.left + 13., + 30., + 13., + TEXT, + true, + ); + } + + pub fn draw( + &self, + canvas: &Canvas, + controls: &Controls, + state: &ControlsState, + model: &Studio, + scroll: &StudioScroll, + ) { + let ui = Ink { + canvas, + font: controls.font(false, 1.), + medium: controls.font(true, 1.), + }; + ui.fill(self.timeline, SURFACE); + ui.fill(self.transport, SURFACE); + ui.fill( + Rect::from_xywh(0., self.transport.top, self.transport.width(), 1.), + LINE, + ); + let buttons = self.buttons(); + let center = self.transport.center_y(); + let compact = self.transport.width() < 460.; + let left = if compact { 8. } else { MARGIN }; + let total = format_time(state.duration_in_frames, state.fps); + let time = format_time(state.frame, state.fps); + let size: f32 = if compact { 14. } else { 17. }; + let frame_size = (size * 0.74).min(if compact { 11. } else { 12. }); + let mut font = ui.medium.with_size(size).unwrap(); + let mut frame_font = ui.font.with_size(frame_size).unwrap(); + let bounds = controls.frame_mark.bounds(); + let inline_gap = if compact { 6. } else { 10. }; + let mut room = buttons[0].1.left - left - 6.; + let mut readout_width = tabular_width(&font, &format!("{total} / {total}")); + if !self.portrait { + room -= inline_gap + 2.; + let mark_width = + frame_font.metrics().1.cap_height * 1.2 * bounds.width() / bounds.height(); + let last_frame = state.duration_in_frames.saturating_sub(1); + readout_width += mark_width + tabular_width(&frame_font, &last_frame.to_string()); + } + if readout_width > room { + let scale = room / readout_width; + font = font.with_size(size * scale).unwrap(); + frame_font = frame_font.with_size(frame_size * scale).unwrap(); + } + let frame_height = frame_font.metrics().1.cap_height; + let clock_height = font.metrics().1.cap_height; + let second_row = if self.portrait { 4. + frame_height } else { 0. }; + let baseline = center + (clock_height - second_row) / 2.; + let after_time = draw_tabular( + canvas, + &time, + Point::new(left, baseline), + &font, + &Ink::paint(TEXT), + ); + let after_clock = draw_tabular( + canvas, + &format!(" / {total}"), + Point::new(after_time, baseline), + &font, + &Ink::paint(QUIET), + ); + let frame_left = if self.portrait { + left + } else { + after_clock + inline_gap + }; + let frame_baseline = baseline + second_row; + let mark_height = frame_height * 1.2; + let scale = mark_height / bounds.height(); + canvas.save(); + canvas.translate((frame_left, frame_baseline + 1. - mark_height)); + canvas.scale((scale, scale)); + canvas.translate((-bounds.left, -bounds.top)); + canvas.draw_path(&controls.frame_mark, &Ink::paint(ORANGE)); + canvas.restore(); + draw_tabular( + canvas, + &state.frame.to_string(), + Point::new(frame_left + bounds.width() * scale + 2., frame_baseline), + &frame_font, + &Ink::paint(QUIET), + ); + + for (control, rect) in buttons { + if control == Control::Focus { + if state.hovered == Some(control) { + ui.round(rect, LINE, 6.); + } + let font = ui.font.with_size(13.).unwrap(); + let x = rect.center_x() - font.measure_str("Focus", None).0 / 2.; + canvas.draw_str( + "Focus", + (x, center + font.metrics().1.cap_height / 2.), + &font, + &Ink::paint(TEXT), + ); + } else { + controls.draw_button(canvas, control, rect, state, 1.); + } + } + ui.fill( + Rect::from_xywh(0., self.transport.bottom - 1., self.transport.width(), 1.), + LINE, + ); + let track = self.tracks(); + if self.portrait { + return self.draw_portrait_tracks(&ui, state, model, scroll); + } + let scene_top = track.top + RULER; + let duration = state.duration_in_frames; + let fps = state.fps.max(1); + let step = tick_step(duration, fps, track.width(), 100.); + for frame in (0..=duration).step_by(step) { + let x = self.frame_position(frame, duration); + let time = if duration < fps { + format!("{frame}f") + } else { + format_time(frame, state.fps) + }; + ui.text( + &time, + x.min(track.right - 37.), + track.top + 13., + 11., + QUIET, + false, + ); + ui.fill(Rect::from_ltrb(x, track.top + 16., x + 1., scene_top), LINE); + } + + let strip_top = scene_top + SCENE_HEIGHT; + let mut previous_end = 0; + for scene in &model.index.scenes { + let left = self.frame_position(scene.frames.start, duration); + let right = self.frame_position(scene.frames.end, duration); + if right <= left { + continue; + } + // Keep the earlier scene's end notch legible during a crossfade. + let rail = self + .frame_position(previous_end.max(scene.frames.start), duration) + .min(right); + previous_end = scene.frames.end; + let color = SCENE_COLORS[scene.index % SCENE_COLORS.len()]; + ui.fill( + Rect::from_ltrb(left, scene_top, right, track.bottom), + color.with_a(13), + ); + ui.fill( + Rect::from_ltrb(rail, scene_top, right, scene_top + 3.), + color, + ); + let notch = (right - rail).min(9.); + let mut marker = PathBuilder::new(); + marker.move_to((right - notch, scene_top)); + marker.line_to((right, scene_top)); + marker.line_to((right, scene_top + notch)); + marker.close(); + canvas.draw_path(&marker.detach(), &Ink::paint(color)); + if right - rail > 12. { + let ink = if scene.frames.contains(&state.frame) { + TEXT + } else { + QUIET + }; + ui.clipped_text( + &scene.name, + Rect::from_ltrb(rail + 3., scene_top, right - 2., strip_top), + scene_top + 16., + 11., + ink, + false, + ); + } + } + if self.film_height > 0. && !model.thumbnails.is_empty() { + let strip = Rect::from_ltrb( + track.left, + strip_top, + track.right, + strip_top + self.film_height, + ); + canvas.save(); + canvas.clip_rrect(RRect::new_rect_xy(strip, 5., 5.), None, true); + ui.fill(strip, CANVAS); + let w = strip.width() / model.thumbnails.len() as f32; + for (slot, image) in model.thumbnails.iter().enumerate() { + let Some(image) = image else { + continue; + }; + let rect = + Rect::from_xywh(track.left + slot as f32 * w, strip.top, w, self.film_height); + ui.thumbnail(image, rect); + } + canvas.restore(); + } + let first_lane = scroll + .audio + .min(model.audio_rows().saturating_sub(self.audio_slots)); + for (row, lane) in model + .audio + .iter() + .skip(first_lane) + .take(self.audio_slots) + .enumerate() + { + let top = strip_top + self.film_height + 6. + LANE * row as f32; + let label = if model.audio_rows() > self.audio_slots { + format!("A{}/{}", row + first_lane + 1, model.audio_rows()) + } else { + format!("A{}", row + first_lane + 1) + }; + ui.text(&label, MARGIN, top + 14., 12., QUIET, false); + for clip in lane { + let r = Rect::from_ltrb( + self.frame_position(clip.frames.start, duration), + top, + self.frame_position(clip.frames.end, duration), + top + LANE - 3., + ); + ui.round(r, AUDIO, 4.); + if let Some(path) = &clip.waveform { + canvas.save(); + canvas.translate((r.left, r.top + 2.)); + canvas.scale((r.width(), r.height() - 3.)); + canvas.draw_path(path, &Ink::paint(Color::from_rgb(86, 86, 86))); + canvas.restore(); + } + if r.width() > 20. { + let label = Rect::from_ltrb(r.left + 5., r.top, r.right - 2., r.bottom); + ui.clipped_text(&clip.name, label, r.top + 13., 11., TEXT, true); + } + } + } + let x = self.frame_position(state.frame, duration); + ui.fill( + Rect::from_ltrb(x - 1., track.top, x + 1., track.bottom), + ORANGE, + ); + ui.round(Rect::from_xywh(x - 4., track.top - 3., 8., 7.), ORANGE, 2.); + } +} + +fn tick_step(duration: usize, fps: usize, span: f32, spacing: f32) -> usize { + let fps = fps.max(1); + let unit = if duration < fps { 1 } else { fps }; + let ticks = (span / spacing).floor().max(1.) as usize; + let target = duration.div_ceil(unit).div_ceil(ticks).max(1); + let power = 10_usize.pow(target.ilog10()); + [1_usize, 2, 5, 10] + .into_iter() + .map(|m| power.saturating_mul(m)) + .find(|&step| step >= target) + .unwrap() + .saturating_mul(unit) +} + +struct Ink<'a> { + canvas: &'a Canvas, + font: Font, + medium: Font, +} + +impl Ink<'_> { + fn paint(color: Color) -> Paint { + let mut paint = Paint::default(); + paint.set_anti_alias(true); + paint.set_color(color); + paint + } + + fn thumbnail(&self, image: &Image, rect: Rect) { + let canvas = self.canvas; + let scale = + (rect.width() / image.width() as f32).max(rect.height() / image.height() as f32); + let mut dst = Rect::from_wh(image.width() as f32 * scale, image.height() as f32 * scale); + dst.offset(( + rect.center_x() - dst.center_x(), + rect.center_y() - dst.center_y(), + )); + canvas.save(); + canvas.clip_rect(rect, None, false); + canvas.draw_image_rect_with_sampling_options( + image, + None, + dst, + SamplingOptions::from(FilterMode::Linear), + &Paint::default(), + ); + canvas.restore(); + } + + fn fill(&self, rect: Rect, color: Color) { + self.canvas.draw_rect(rect, &Self::paint(color)); + } + + fn round(&self, rect: Rect, color: Color, radius: f32) { + self.canvas + .draw_round_rect(rect, radius, radius, &Self::paint(color)); + } + + fn text(&self, text: &str, x: f32, y: f32, size: f32, color: Color, medium: bool) { + let font = if medium { &self.medium } else { &self.font }; + self.canvas.draw_str( + text, + (x, y), + &font.with_size(size).unwrap(), + &Self::paint(color), + ); + } + + fn clipped_text( + &self, + text: &str, + rect: Rect, + baseline: f32, + size: f32, + color: Color, + medium: bool, + ) { + self.canvas.save(); + self.canvas.clip_rect(rect, None, false); + self.text(text, rect.left, baseline, size, color, medium); + self.canvas.restore(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use fframes::TimelineScene; + + fn studio() -> Studio { + Studio { + video_aspect: 16. / 9., + index: TimelineIndex { + fps: 30, + duration_in_frames: 100, + scenes: [0..60, 50..100, 110..115] + .into_iter() + .enumerate() + .map(|(index, frames)| TimelineScene { + index, + frames, + name: format!("Scene {index}"), + full_name: String::new(), + }) + .collect(), + }, + audio: vec![], + thumbnails: vec![], + } + } + + #[test] + fn navigation_uses_resolved_starts_and_skips_unreachable_scenes() { + let studio = studio(); + assert_eq!(studio.adjacent_scene(49, true), Some(50)); + assert_eq!(studio.adjacent_scene(50, false), Some(0)); + assert_eq!(studio.adjacent_scene(51, false), Some(50)); + assert_eq!(studio.adjacent_scene(50, true), None); + assert_eq!(studio.adjacent_scene(0, false), None); + } + + #[test] + fn timeline_seek_and_hit_regions_agree_across_sizes() { + let model = studio(); + for width in [320., 440., 460., 560., 1280.] { + let layout = StudioLayout::new(width, 700., 2, model.video_aspect).unwrap(); + let track = layout.tracks(); + assert_eq!(layout.hit(track.center(), &model), Some(Control::Timeline)); + assert_eq!(layout.hit(layout.stage.center(), &model), None); + assert_eq!(layout.stage.top, 0.); + let buttons = layout.buttons(); + assert_eq!(buttons[1].1.center_x(), width / 2.); + assert!( + buttons + .windows(2) + .all(|pair| pair[0].1.right <= pair[1].1.left) + ); + assert!(layout.focus_hit(layout.restore.center())); + assert_eq!(layout.hit(layout.restore.center(), &model), None); + assert_eq!(layout.frame_at(track.left - 1., 100), 0); + assert_eq!(layout.frame_at(track.right, 100), 99); + assert_eq!(layout.frame_at(track.center_x(), 100), 50); + assert_eq!(layout.frame_at(track.right, 1), 0); + assert_eq!(layout.frame_at(track.right, 0), 0); + let scene = Point::new(track.center_x() + 1., track.top + RULER + SCENE_HEIGHT / 2.); + assert_eq!(layout.hit(scene, &model), Some(Control::Scene(50))); + assert_eq!(layout.stage.left, 0.); + assert!(track.bottom < 700.); + for (control, rect) in layout.buttons() { + assert_eq!(layout.hit(rect.center(), &model), Some(control)); + } + } + let compact = StudioLayout::new(320., 240., 2, model.video_aspect).unwrap(); + assert_eq!(compact.timeline.bottom, 240.); + assert_eq!(compact.audio_slots, 0); + assert!(StudioLayout::new(300., 200., 0, model.video_aspect).is_none()); + } + + #[test] + fn portrait_uses_the_left_timeline_for_scenes_and_vertical_seeking() { + let model = studio(); + for (width, height) in [(765., 720.), (1280., 720.), (405., 720.), (320., 240.)] { + let ui = StudioLayout::new(width, height, 2, 9. / 16.).unwrap(); + let track = ui.tracks(); + let (film, scenes, audio) = ui.portrait_columns(); + assert!(ui.portrait); + assert_eq!(ui.stage.top, 0.); + assert_eq!(ui.stage.left, ui.timeline.right); + assert!( + track.right < ui.stage.left + && film.right < scenes.left + && scenes.right < audio.left + ); + assert_eq!(ui.hit(ui.stage.center(), &model), None); + assert_eq!(ui.frame_at(ui.time_axis(film.center()), 100), 50); + assert_eq!(ui.frame_at(track.top - 10., 100), 0); + assert_eq!(ui.frame_at(track.bottom + 10., 100), 99); + assert_eq!(ui.hit(film.center(), &model), Some(Control::Timeline)); + assert_eq!(ui.hit(scenes.center(), &model), Some(Control::Scene(50))); + assert_eq!(ui.buttons()[1].1.center_x(), ui.transport.center_x()); + for (control, button) in ui.buttons() { + assert_eq!(ui.hit(button.center(), &model), Some(control)); + } + if width == 765. { + assert_eq!(ui.stage, Rect::from_ltrb(360., 0., 765., 720.)); + } + } + assert!(!StudioLayout::new(405., 720., 2, 16. / 9.).unwrap().portrait); + assert!(!StudioLayout::new(405., 720., 2, 1.).unwrap().portrait); + } + + #[test] + fn waveform_uses_both_channels_and_the_selected_source_interval() { + let data = PreloadedAudioData { + sample_rate: 512, + samples: vec![0.; 1024].into(), + right: Some([vec![0.; 512], vec![0.75; 512]].concat().into()), + }; + assert_eq!(waveform(&data, 0., 1.).bounds().top, 1.); + assert_eq!(waveform(&data, 1., 1.).bounds().top, 0.25); + assert_eq!(waveform(&data, 2., 1.).bounds().top, 1.); + } + + #[test] + fn overlapping_audio_stacks_and_reuses_the_first_free_lane() { + let clips = [15..18, 0..32, 3..5, 5..7].map(|frames| Clip { + name: String::new(), + waveform: None, + frames, + }); + let lanes = pack_clips(Vec::from(clips)); + assert_eq!(lanes.len(), 2); + assert_eq!(lanes[0][0].frames, 0..32); + assert_eq!( + lanes[1] + .iter() + .map(|clip| clip.frames.clone()) + .collect::>(), + [3..5, 5..7, 15..18] + ); + } +} From aa209bff26b90aca9b3a732d2c6aad91796f17bd Mon Sep 17 00:00:00 2001 From: dmtrKovalenko Date: Tue, 27 Oct 2026 17:06:39 -0700 Subject: [PATCH 02/21] fix(player): Use the compact fframes logo for frame numbers --- fframes-native-player/src/controls.rs | 13 ++++--------- fframes-native-player/src/frame-f.svg | 6 +++++- fframes-native-player/src/studio.rs | 18 +++++++++--------- 3 files changed, 18 insertions(+), 19 deletions(-) diff --git a/fframes-native-player/src/controls.rs b/fframes-native-player/src/controls.rs index e7a03c1b..d0af0c54 100644 --- a/fframes-native-player/src/controls.rs +++ b/fframes-native-player/src/controls.rs @@ -89,7 +89,7 @@ pub(crate) struct Controls { regular: Typeface, medium: Typeface, icons: IconPaths, - pub(crate) frame_mark: Path, + pub(crate) frame_mark: skia_safe::svg::Dom, } struct IconPaths { @@ -227,14 +227,9 @@ impl Controls { } }; - let svg = fframes::roxmltree::Document::parse(include_str!("frame-f.svg")) - .expect("valid frame mark SVG"); - let frame_mark = svg - .root_element() - .first_element_child() - .and_then(|path| path.attribute("d")) - .and_then(Path::from_svg) - .expect("valid frame mark path"); + let frame_mark = + skia_safe::svg::Dom::from_str(include_str!("frame-f.svg"), FontMgr::empty()) + .expect("valid fframes mark SVG"); Self { regular: typeface(font_style::Weight::NORMAL), diff --git a/fframes-native-player/src/frame-f.svg b/fframes-native-player/src/frame-f.svg index f3b7bcea..c2375eae 100644 --- a/fframes-native-player/src/frame-f.svg +++ b/fframes-native-player/src/frame-f.svg @@ -1 +1,5 @@ - + + + + + diff --git a/fframes-native-player/src/studio.rs b/fframes-native-player/src/studio.rs index d09b1dbd..768e24fc 100644 --- a/fframes-native-player/src/studio.rs +++ b/fframes-native-player/src/studio.rs @@ -538,14 +538,15 @@ impl StudioLayout { let frame_size = (size * 0.74).min(if compact { 11. } else { 12. }); let mut font = ui.medium.with_size(size).unwrap(); let mut frame_font = ui.font.with_size(frame_size).unwrap(); - let bounds = controls.frame_mark.bounds(); + let mark_size = controls.frame_mark.root().intrinsic_size(); + let mark_aspect = mark_size.width / mark_size.height; + let mark_height_ratio = 1.5; let inline_gap = if compact { 6. } else { 10. }; let mut room = buttons[0].1.left - left - 6.; let mut readout_width = tabular_width(&font, &format!("{total} / {total}")); if !self.portrait { room -= inline_gap + 2.; - let mark_width = - frame_font.metrics().1.cap_height * 1.2 * bounds.width() / bounds.height(); + let mark_width = frame_font.metrics().1.cap_height * mark_height_ratio * mark_aspect; let last_frame = state.duration_in_frames.saturating_sub(1); readout_width += mark_width + tabular_width(&frame_font, &last_frame.to_string()); } @@ -578,18 +579,17 @@ impl StudioLayout { after_clock + inline_gap }; let frame_baseline = baseline + second_row; - let mark_height = frame_height * 1.2; - let scale = mark_height / bounds.height(); + let mark_height = frame_height * mark_height_ratio; + let scale = mark_height / mark_size.height; canvas.save(); - canvas.translate((frame_left, frame_baseline + 1. - mark_height)); + canvas.translate((frame_left, frame_baseline - mark_height * 0.85)); canvas.scale((scale, scale)); - canvas.translate((-bounds.left, -bounds.top)); - canvas.draw_path(&controls.frame_mark, &Ink::paint(ORANGE)); + controls.frame_mark.render(canvas); canvas.restore(); draw_tabular( canvas, &state.frame.to_string(), - Point::new(frame_left + bounds.width() * scale + 2., frame_baseline), + Point::new(frame_left + mark_height * mark_aspect + 2., frame_baseline), &frame_font, &Ink::paint(QUIET), ); From 36740fbb73e7b0f64c86bbd067465bbb35adcdee Mon Sep 17 00:00:00 2001 From: dmtrKovalenko Date: Wed, 28 Oct 2026 19:39:31 +0000 Subject: [PATCH 03/21] perf(media): Reuse converted video frames during playback --- fframes-media/src/video_decoder.rs | 262 +++++++++-------------- fframes-media/tests/video_decoder_eof.rs | 66 ++++-- fframes/src/video_data.rs | 47 ++-- 3 files changed, 171 insertions(+), 204 deletions(-) diff --git a/fframes-media/src/video_decoder.rs b/fframes-media/src/video_decoder.rs index fa7525b6..3cc81605 100644 --- a/fframes-media/src/video_decoder.rs +++ b/fframes-media/src/video_decoder.rs @@ -3,13 +3,13 @@ use crate::error::Result; use crate::video_types::{FrameConvertOptions, PlaybackSpeed}; use ffmpeg_sys_fframes::SwsFlags::{SWS_BICUBIC, SWS_BILINEAR}; use ffmpeg_sys_fframes::*; -use std::cell::UnsafeCell; +use std::borrow::Cow; use std::collections::VecDeque; use std::ffi::{CStr, CString, c_void}; use std::mem::size_of; use std::path::Path; use std::ptr; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use usvgr::PreloadedImageData; /// The video stream's duration in its own time base. Matroska and `WebM` leave the stream @@ -112,10 +112,11 @@ struct SwsScaler { sws_ctx: *mut SwsContext, options: Option, source_format: Option, + source_has_alpha: bool, source_color: Option<(AVColorSpace, AVColorRange)>, frame_data_len: usize, video_stream_info: VideoStreamInfo, - linesize: [i32; 7], + linesize: [i32; 4], } impl Drop for SwsScaler { @@ -240,8 +241,8 @@ impl SwsScaler { } } - fn calculate_linesize(width: i32) -> [i32; 7] { - let mut linesize = [0; 7]; + fn calculate_linesize(width: i32) -> [i32; 4] { + let mut linesize = [0; 4]; linesize[0] = width * PIX_FMT_SIZE as i32; linesize } @@ -253,7 +254,7 @@ impl SwsScaler { ); let frame_data_len = width as usize * height as usize * PIX_FMT_SIZE; - let linesize: [i32; 7] = Self::calculate_linesize(width); + let linesize: [i32; 4] = Self::calculate_linesize(width); SwsScaler { linesize, frame_data_len, @@ -262,6 +263,7 @@ impl SwsScaler { sws_ctx: std::ptr::null_mut(), options: None, source_format: None, + source_has_alpha: false, source_color: None, video_stream_info, } @@ -290,6 +292,9 @@ impl SwsScaler { sws_freeContext(self.sws_ctx); self.sws_ctx = ctx; self.source_format = Some(pix_fmt); + let desc = unsafe { av_pix_fmt_desc_get(pix_fmt) }; + self.source_has_alpha = + !desc.is_null() && unsafe { (*desc).flags } & AV_PIX_FMT_FLAG_ALPHA as u64 != 0; self.source_color = Some(source_color); self.width = new_width; self.height = new_height; @@ -326,29 +331,35 @@ impl SwsScaler { self.reinit_sws_context(source_pix_fmt, source_color, options)?; } - // A reused buffer may still have the length of another output size. This - // reallocates only when the output grows and does nothing when the size is - // unchanged. `sws_scale` can write past the image, see `SWS_DST_PADDING`. + // Reserve the SIMD tail without zeroing a full frame before swscale fills it. let capacity = self.frame_data_len + SWS_DST_PADDING; rgba_dst.reserve(capacity.saturating_sub(rgba_dst.len())); - rgba_dst.resize(self.frame_data_len, 0); + // sws_scale reads four plane pointers even for packed RGBA. + let mut planes = [ptr::null_mut(); 4]; + planes[0] = rgba_dst.as_mut_ptr(); let ret = sws_scale( self.sws_ctx, (*source_frame).data.as_ptr().cast::<*const u8>(), (*source_frame).linesize.as_ptr(), 0, self.video_stream_info.height, - &rgba_dst.as_mut_ptr(), + planes.as_ptr(), self.linesize.as_ptr(), ); - if ret < 0 { + if ret != self.height { return Err(FFramesMediaError::LibAVAudioDecodingError(( ret, - "Error while converting/scaling frame".to_string(), + format!( + "Expected {} output rows from sws_scale, got {ret}", + self.height + ), ))); } + // SAFETY: sws_scale initialized every RGBA row with width * 4 stride. + // The tail padding lies inside the allocation but outside the image. + unsafe { rgba_dst.set_len(self.frame_data_len) }; Ok(()) } @@ -379,9 +390,17 @@ pub struct FFmpegDecoder { unsafe impl Send for FFmpegDecoder {} unsafe impl Sync for FFmpegDecoder {} +#[derive(Debug)] struct ScaledFrameImage { pts: i64, - data: Vec, + options: Option, + image: Arc, +} + +#[derive(Debug)] +struct ConvertedFrames { + scaler: SwsScaler, + images: VecDeque, } #[derive(Debug)] @@ -389,66 +408,26 @@ pub struct FFmpegFrameBuf { resource_name: String, latest_av_frame: *mut AVFrame, video_stream_info: VideoStreamInfo, - // This is not a safe operation but it gives a dramatic performance improvement - // for the updating the underlying datavec. So we are relying on the constraint that - // the decoder can operate (thus write the datavec) only within the frame, but - // the frame can be sent and read outside the dnecoder - data_buf: Arc>>, - sws_scaler: UnsafeCell, + converted: Mutex, } impl Drop for FFmpegFrameBuf { fn drop(&mut self) { - unsafe { - if !self.latest_av_frame.is_null() { - av_frame_unref(self.latest_av_frame); - av_frame_free(&raw mut self.latest_av_frame); - } - - if let Some(queue) = self.data_buf.get().as_mut() { - queue.clear(); - } - } + unsafe { av_frame_free(&raw mut self.latest_av_frame) }; } } -/// Creates svgr preloaded image data with correctly blended color. -/// -/// `rgba_data` borrows a slot of the decoder's ring buffer (with a fabricated -/// `'static` lifetime) that is recycled by later decodes, so the returned -/// image always owns a copy of the pixels. -fn create_preloaded_image( - source_fmt: AVPixelFormat, - resource_name: String, - width: u32, - height: u32, - rgba_data: &'static [u8], -) -> Arc { - let has_alpha_channel = unsafe { - let desc = av_pix_fmt_desc_get(source_fmt); - - !desc.is_null() && (*desc).flags & AV_PIX_FMT_FLAG_ALPHA as u64 != 0 - }; - - if has_alpha_channel { - // Pre-multiplies alpha and returns an owned copy. - Arc::new(PreloadedImageData::new( - resource_name, - width, - height, - rgba_data, - )) - } else { - // Opaque pixels are already final; only the copy is needed. - Arc::new(PreloadedImageData { - id: resource_name, - data: std::borrow::Cow::Owned(rgba_data.to_vec()), - width, - height, - }) +fn premultiply_alpha(pixels: &mut [u8]) { + for pixel in pixels.as_chunks_mut::<4>().0 { + let alpha = u16::from(pixel[3]); + for channel in &mut pixel[..3] { + *channel = ((u16::from(*channel) * alpha + 127) / 255) as u8; + } } } +// The decoder owns writes to the AVFrame. Its unsafe readers require that it +// stays unchanged while they run. The conversion ring and scaler share a lock. unsafe impl Send for FFmpegFrameBuf {} unsafe impl Sync for FFmpegFrameBuf {} @@ -469,43 +448,14 @@ impl FFmpegFrameBuf { video_stream_info, latest_av_frame: av_frame, resource_name, - #[allow(clippy::arc_with_non_send_sync)] - data_buf: Arc::new(UnsafeCell::new(VecDeque::with_capacity(buf_size))), - sws_scaler: UnsafeCell::new(sws_ctx), + converted: Mutex::new(ConvertedFrames { + scaler: sws_ctx, + images: VecDeque::with_capacity(buf_size.max(1)), + }), }) } } - unsafe fn write_new_frame(&self) -> Option<&'static mut ScaledFrameImage> { - unsafe { - let queue = self.data_buf.get().as_mut()?; - let latest_pts = (*self.latest_av_frame).pts; - - // fast path for the cpu renderer which will always have capacity 1 - if queue.capacity() == 1 && queue.len() == 1 { - // the pts identifies the image (and its renderer cache entries), it has to - // follow the pixels - let image = queue.get_mut(0)?; - image.pts = latest_pts; - return Some(image); - } - - // `SwsScaler::convert` sizes the buffer once it knows the requested output size - if queue.len() == queue.capacity() { - let mut last_buffer = queue.pop_front()?; - last_buffer.pts = latest_pts; - queue.push_back(last_buffer); - } else { - queue.push_back(ScaledFrameImage { - pts: latest_pts, - data: Vec::new(), - }); - } - - queue.get_mut(queue.len() - 1) - } - } - pub fn get_stream_width(&self) -> u32 { self.video_stream_info.display_width as u32 } @@ -515,67 +465,55 @@ impl FFmpegFrameBuf { } pub fn get_width(&self) -> u32 { - unsafe { self.sws_scaler.get().as_ref() } - .expect("Critical mememroy error: SWS scale is not allocated") - .width as u32 + self.converted.lock().unwrap().scaler.width as u32 } pub fn get_height(&self) -> u32 { - unsafe { self.sws_scaler.get().as_ref() } - .expect("Critical mememroy error: SWS scale is not allocated") - .height as u32 + self.converted.lock().unwrap().scaler.height as u32 } - /// Returns fframe image constructed from the underlying ffmpeg's frame + /// Converts the current frame to an immutable, premultiplied RGBA image. /// # Safety - /// This is a libav based function which involves C ffi cals - /// In addition it is manually transmutes the pointer owned by the decoder to the fframes - /// images so it should not be used outside of the rendering worker. + /// The decoder must not replace the underlying `AVFrame` during conversion. pub unsafe fn convert_last_decoded_frame_into_svg_image( &self, options: Option<&FrameConvertOptions>, ) -> Result> { - unsafe { - let scaler = self.sws_scaler.get().as_mut().unwrap(); - // fast path if we have the image already decoded and converted last time with the - // same options (the scaler keeps them), e.g. the same video frame is requested at - // the same size from multiple scenes - match self.data_buf.get().as_ref().and_then(|q| q.back()) { - Some(image) - if image.pts == (*self.latest_av_frame).pts - && scaler.options == options.copied() => - { - return Ok(create_preloaded_image( - scaler.video_stream_info.pixel_format, - encode_video_resource(&self.resource_name, image.pts), - self.get_width(), - self.get_height(), - &image.data, - )); - } - _ => {} - } - - let image = self.write_new_frame().unwrap(); - if let Err(err) = - scaler.convert(options.copied(), self.latest_av_frame, &mut image.data) - { - // the buffer already has this frame's pts but not its pixels (a reused buffer - // still holds an older frame), so drop it instead of returning it for this pts - if let Some(queue) = self.data_buf.get().as_mut() { - queue.pop_back(); - } - return Err(err); - } + let pts = unsafe { (*self.latest_av_frame).pts }; + let options = options.copied(); + let mut converted = self.converted.lock().unwrap(); + let ConvertedFrames { scaler, images } = &mut *converted; + if let Some(last) = images.back() + && last.pts == pts + && last.options == options + { + return Ok(Arc::clone(&last.image)); + } - Ok(create_preloaded_image( - scaler.video_stream_info.pixel_format, - encode_video_resource(&self.resource_name, image.pts), - scaler.width as u32, - scaler.height as u32, - &image.data, - )) + // A queued frame may still own the evicted image. + let mut pixels = if images.len() == images.capacity() { + Arc::try_unwrap(images.pop_front().unwrap().image) + .map(|image| image.data.into_owned()) + .unwrap_or_default() + } else { + Vec::new() + }; + unsafe { scaler.convert(options, self.latest_av_frame, &mut pixels)? }; + if scaler.source_has_alpha { + premultiply_alpha(&mut pixels); } + let image = Arc::new(PreloadedImageData { + id: encode_video_resource(&self.resource_name, pts), + width: scaler.width as u32, + height: scaler.height as u32, + data: Cow::Owned(pixels), + }); + images.push_back(ScaledFrameImage { + pts, + options, + image: Arc::clone(&image), + }); + Ok(image) } /// Returns the timestamp of the frame in the native frame timebase @@ -622,16 +560,15 @@ impl FFmpegDecoder { } pub fn get_decoded_image_in_buf(&self, pts: i64) -> Option> { - let buf = unsafe { self.frame_buf.data_buf.get().as_ref() }?; - let image = buf.iter().find(|i| i.pts == pts)?; - - Some(create_preloaded_image( - self.frame_buf.video_stream_info.pixel_format, - encode_video_resource(&self.frame_buf.resource_name, image.pts), - self.frame_buf.get_width(), - self.frame_buf.get_height(), - &image.data, - )) + self.frame_buf + .converted + .lock() + .unwrap() + .images + .iter() + .rev() + .find(|slot| slot.pts == pts) + .map(|slot| Arc::clone(&slot.image)) } /// Plays the stream `speed` times faster: offset `n` shows the frame at @@ -1226,6 +1163,19 @@ mod color_tests { const WIDTH: i32 = 32; const HEIGHT: i32 = 16; + #[test] + fn in_place_alpha_matches_the_renderer_for_every_channel_and_alpha() { + let mut pixels = Vec::new(); + for alpha in 0..=255u8 { + for color in 0..=255u8 { + pixels.extend_from_slice(&[color, 255 - color, color, alpha]); + } + } + let expected = PreloadedImageData::blend_rgba_slice(&pixels); + premultiply_alpha(&mut pixels); + assert_eq!(pixels, expected); + } + struct TestFrame(*mut AVFrame); impl TestFrame { diff --git a/fframes-media/tests/video_decoder_eof.rs b/fframes-media/tests/video_decoder_eof.rs index ecca2a30..b6a919a1 100644 --- a/fframes-media/tests/video_decoder_eof.rs +++ b/fframes-media/tests/video_decoder_eof.rs @@ -3,6 +3,7 @@ use std::collections::HashMap; use std::path::PathBuf; use std::process::Command; +use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; use fframes_media::{FFmpegDecoder, FrameConvertOptions, PlaybackSpeed, ResizeVideoFrame}; @@ -173,6 +174,28 @@ fn videos_without_b_frames_stop_at_eof() { } } +#[test] +fn converted_images_survive_single_buffer_reuse_and_decoder_drop() { + let video = VideoFixture::new(3); + let mut decoder = unsafe { FFmpegDecoder::new(&video.path(), FPS, 1) }.unwrap(); + let mut images = Vec::new(); + for index in [0, 17, 2, 90, 299] { + assert!(unsafe { decoder.decode_up_to(index) }.unwrap()); + let image = unsafe { + decoder + .get_raw_frame() + .convert_last_decoded_frame_into_svg_image(None) + } + .unwrap(); + let expected = image.data.to_vec(); + images.push((image, expected)); + } + drop(decoder); + for (image, expected) in images { + assert_eq!(image.data.as_ref(), expected); + } +} + #[test] fn a_frame_requested_again_at_another_size_is_converted_to_that_size() { let video = VideoFixture::new(0); @@ -201,22 +224,33 @@ fn a_frame_requested_again_at_another_size_is_converted_to_that_size() { .convert_last_decoded_frame_into_svg_image(None) .unwrap(); assert_eq!((image.width, image.height), (32, 32)); - // The second thumbnail request is served from the cache - for _ in 0..2 { - let image = frame - .convert_last_decoded_frame_into_svg_image(Some(&thumbnail)) - .unwrap(); - assert_eq!( - (image.width, image.height), - (16, 16), - "buffer size {buffer_size}" - ); - // A buffer last sized for a larger output can be longer than the image - assert!( - image.data[..thumbnail_len] == expected.data[..thumbnail_len], - "the thumbnail differs from a decoder that only converts to 16x16 \ - (buffer size {buffer_size})" - ); + let thumb = frame + .convert_last_decoded_frame_into_svg_image(Some(&thumbnail)) + .unwrap(); + assert_eq!((thumb.width, thumb.height), (16, 16)); + assert_eq!(thumb.data.len(), thumbnail_len); + assert_eq!(thumb.data, expected.data); + let again = frame + .convert_last_decoded_frame_into_svg_image(Some(&thumbnail)) + .unwrap(); + assert!(Arc::ptr_eq(&thumb, &again)); + let pts = frame.get_pts(); + assert!(Arc::ptr_eq( + &thumb, + &decoder.get_decoded_image_in_buf(pts).unwrap() + )); + assert!(decoder.decode_up_to(6).unwrap()); + let _full_size = frame + .convert_last_decoded_frame_into_svg_image(None) + .unwrap(); + if buffer_size == 1 { + assert!(decoder.get_decoded_image_in_buf(pts).is_none()); + } else { + // The cached thumbnail retains its own dimensions after a full-size conversion. + assert!(Arc::ptr_eq( + &thumb, + &decoder.get_decoded_image_in_buf(pts).unwrap() + )); } } } diff --git a/fframes/src/video_data.rs b/fframes/src/video_data.rs index 1fab172f..3fdf5553 100644 --- a/fframes/src/video_data.rs +++ b/fframes/src/video_data.rs @@ -43,20 +43,11 @@ pub fn set_video_frame_resolver(resolver: VideoFrameResolver) { } pub trait FFramesSyncedVideoFrame<'media> { - /// Gets the original image from the frame - /// This will perform color space conversion and return the frame at `width()` by - /// `height()`. If you need to render the video frame in the smaller size, use `into_resized_image`. - /// - /// This method take self to reuse frame data allocations, but you should not - /// call this method more than once per the render function + /// Converts the frame to RGBA at its display dimensions. + /// Native renderers share the pixels between consecutive requests at the same size. #[allow(clippy::wrong_self_convention)] fn into_image(&self) -> ImageData<'media>; - /// Performs color space conversion and resizes the image to the specified size. - /// Recommended to use this method if you render image in the smaller size than the original - /// video e.g. as a pattern of some other shape or as a filler of standalone image. - /// - /// This method take self to reuse frame data allocations, but you should not - /// call this method more than once per the render function + /// Converts the frame to RGBA at the requested size. #[allow(clippy::wrong_self_convention)] fn into_resized_image(&self, options: &FrameConvertOptions) -> Option>; /// Get the height of the frame with non-square pixels stretched to square ones @@ -151,23 +142,23 @@ impl<'media> FFramesSyncedVideoFrame<'media> for WasmEditorVideoFrameFallback<'m } } +#[cfg(not(target_arch = "wasm32"))] +fn decoded_image<'media>(pixels: Arc) -> ImageData<'media> { + let metadata = ImageMetadata { + width: pixels.width, + height: pixels.height, + }; + let id = pixels.id.clone(); + ImageData::new_from_raw_data(pixels, id, metadata) +} + #[cfg(not(target_arch = "wasm32"))] impl<'media> FFramesSyncedVideoFrame<'media> for crate::media::FFmpegFrameBuf { fn into_image(&self) -> ImageData<'media> { let image_data = unsafe { self.convert_last_decoded_frame_into_svg_image(None) } - // we unwrap here because we guarantee that the sws scaler would work correctly - // unless the wrong dimensions are provided .expect("Failed to convert frame to image"); - let filename = image_data.id.clone(); - ImageData::new_from_raw_data( - image_data, - filename, - ImageMetadata { - width: self.get_width(), - height: self.get_height(), - }, - ) + decoded_image(image_data) } fn into_resized_image(&self, options: &FrameConvertOptions) -> Option> { @@ -177,15 +168,7 @@ impl<'media> FFramesSyncedVideoFrame<'media> for crate::media::FFmpegFrameBuf { }) .ok()?; - let filename = image_data.id.clone(); - Some(ImageData::new_from_raw_data( - image_data, - filename, - ImageMetadata { - width: self.get_width(), - height: self.get_height(), - }, - )) + Some(decoded_image(image_data)) } fn width(&self) -> u32 { From c9618212e04dc3ff6331ee2baf8c09f69932a3b8 Mon Sep 17 00:00:00 2001 From: dmtrKovalenko Date: Wed, 28 Oct 2026 19:39:38 +0000 Subject: [PATCH 04/21] perf(player): Keep playback decoders separate from the filmstrip --- README.md | 8 +- fframes-native-player/src/app.rs | 105 +++++++-- fframes-native-player/src/lib.rs | 29 +-- fframes-native-player/src/options.rs | 45 +++- fframes-native-player/src/presenter/mod.rs | 7 +- fframes-native-player/src/scheduler.rs | 136 +++++++++-- fframes-native-player/src/studio.rs | 250 +++++++++++++++++++-- fframes/src/media_provider.rs | 13 ++ 8 files changed, 503 insertions(+), 90 deletions(-) diff --git a/README.md b/README.md index 10e339c8..112a88ff 100644 --- a/README.md +++ b/README.md @@ -339,10 +339,10 @@ cargo fframes new my-video cd my-video && cargo run --release -- preview ``` -The native preview places the timeline below landscape and square videos and to the left of portrait -videos. Drag the filmstrip to scrub, or click a colored scene marker to pause at its first frame. -Use `[` and `]` to jump between scene starts, `,` and `.` to step one frame, and `Space` to play or pause. -The dock shows the elapsed time, total duration, and frame number. Press `T` to switch to Focus mode. +The preview timeline sits below landscape and square videos and to the left of portrait videos. +Drag the filmstrip to scrub, or click a scene marker to pause on its first frame. Use `[` and `]` +to jump between scenes and `,` and `.` to step one frame. Press `Space` to play or pause, or `T` +to hide the timeline and expand the video. `cargo fframes new` takes `--template single-scene|multi-scene`, `--format landscape|portrait|square|uhd`, `--fps`, `--title` and `--backend`. On macOS and Linux (arm64, x86_64) the first build downloads prebuilt Skia and ffmpeg diff --git a/fframes-native-player/src/app.rs b/fframes-native-player/src/app.rs index 2ba6d265..1a2ec683 100644 --- a/fframes-native-player/src/app.rs +++ b/fframes-native-player/src/app.rs @@ -21,9 +21,10 @@ use fframes_skia_renderer::skia_safe::Point; /// The control bar hides after this long without pointer movement while playing. const CONTROLS_IDLE_TIMEOUT: Duration = Duration::from_millis(2500); -/// Sent by the frame workers whenever a frame becomes available. +/// A frame worker finished a playback or filmstrip job. pub(crate) enum FrameReady { Playback, + Failed { position: u64, epoch: u64 }, Thumbnail(usize, fframes_skia_renderer::skia_safe::Image), } @@ -226,12 +227,7 @@ impl<'s> App<'s> { let next_frame = self.scheduler.frame_of(frame.0, self.looping); self.current = Some(frame); self.needs_redraw = true; - if let Some(target) = self.scrub_target.take() - && target != next_frame - { - self.seek_to(target as u64, false); - self.scrub_target = Some(target); - } + self.finish_scrub(next_frame); } let controls_visible = self.controls_should_be_visible(); @@ -250,6 +246,31 @@ impl<'s> App<'s> { self.update_title(); } + fn finish_scrub(&mut self, frame: usize) { + if let Some(target) = self.scrub_target.take() { + if target != frame { + self.seek_to(target as u64, false); + self.scrub_target = Some(target); + } + self.needs_redraw = true; + } + } + + fn frame_ready(&mut self, event: FrameReady) { + match event { + FrameReady::Thumbnail(slot, image) => { + self.config.studio.set_thumbnail(slot, image); + self.needs_redraw = true; + } + FrameReady::Failed { position, epoch } + if self.scheduler.is_current_seek(position, epoch) => + { + self.finish_scrub(self.scheduler.frame_of(position, self.looping)); + } + FrameReady::Playback | FrameReady::Failed { .. } => {} + } + } + fn update_title(&mut self) { let now = Instant::now(); let stats_elapsed = now.duration_since(self.stats_since).as_secs_f64(); @@ -388,8 +409,10 @@ impl<'s> App<'s> { fn control_under_cursor(&self) -> Option { let point = self.pointer(); - if self.studio_open { - return self.studio_layout()?.hit(point, &self.config.studio); + if self.studio_open + && let Some(layout) = self.studio_layout() + { + return layout.hit(point, &self.config.studio); } if self.controls_visible && let Some(control) = self @@ -612,10 +635,7 @@ impl ApplicationHandler for App<'_> { } fn user_event(&mut self, _event_loop: &ActiveEventLoop, event: FrameReady) { - if let FrameReady::Thumbnail(slot, image) = event { - self.config.studio.thumbnails[slot] = Some(image); - self.needs_redraw = true; - } + self.frame_ready(event); self.tick(); } @@ -685,3 +705,62 @@ impl ApplicationHandler for App<'_> { }); } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn failed_scrub_continues_at_the_latest_pointer_position() { + let timeline = fframes::ResolvedRenderingTimeline { + audio_map: None, + scenes: None, + duration_in_frames: 100, + }; + let time = fframes::TimeBase { + fps: 30, + sample_rate: 44100, + }; + let scheduler = Scheduler::new(100, 3, false, 0); + let mut app = App::new( + AppConfig { + studio: Studio::new(time, &timeline, None, 0, 1.), + title: "scrub", + window_size: LogicalSize::new(640, 480), + fit_portrait_to_screen: false, + backend: PlayerBackend::Cpu, + vsync: false, + fps: time.fps, + duration_in_frames: timeline.duration_in_frames, + background: fframes::Color::BLACK, + looping: false, + autoplay: false, + start_frame: 0, + }, + &scheduler, + #[cfg(feature = "audio")] + None, + ); + app.seek_to(10, false); + app.scrub_target = Some(20); + // A stale completion (even at the same position) and a failed lookahead + // frame must not advance the pending scrub. + for (position, epoch) in [(10, 1), (11, 2)] { + app.frame_ready(FrameReady::Failed { position, epoch }); + assert_eq!(app.clock.position(), 10); + assert_eq!(app.scrub_target, Some(20)); + } + app.frame_ready(FrameReady::Failed { + position: 10, + epoch: 2, + }); + assert_eq!(app.clock.position(), 20); + assert_eq!(app.scrub_target, Some(20)); + app.frame_ready(FrameReady::Failed { + position: 20, + epoch: 3, + }); + assert_eq!(app.scrub_target, None); + assert!(scheduler.is_current_seek(20, 3)); + } +} diff --git a/fframes-native-player/src/lib.rs b/fframes-native-player/src/lib.rs index 558e848c..1a7a0639 100644 --- a/fframes-native-player/src/lib.rs +++ b/fframes-native-player/src/lib.rs @@ -47,14 +47,12 @@ pub use options::*; use std::collections::HashMap; use std::sync::Mutex; -use fframes::{ - FFramesContext, FFramesRendererRuntime, TimeBase, Video, VideoDecodersWorker, VideoSize, usvgr, -}; +use fframes::{FFramesContext, FFramesRendererRuntime, TimeBase, Video, VideoSize, usvgr}; use winit::dpi::LogicalSize; use winit::event_loop::EventLoop; use crate::app::{App, AppConfig, FrameReady}; -use crate::scheduler::Scheduler; +use crate::scheduler::{Scheduler, WorkerKind}; /// Opens a window and plays `video` in real time until the window is closed. /// @@ -179,24 +177,17 @@ pub fn play<'a, 'media: 'a, TVideo: Video + Sync>( audio.as_ref(), ); + let run_worker = |kind| { + let db = font_source.as_db_ref(); + svgr_images.enter(&usvg_options, db, || { + scheduler.run_worker(video, &ctx, &usvg_options, db, kind, ¬ify_frame_ready); + }); + }; let result = std::thread::scope(|scope| { for _ in 0..options.render_threads() { - scope.spawn(|| { - let db = font_source.as_db_ref(); - svgr_images.enter(&usvg_options, db, || { - scheduler.run_worker( - video, - &ctx, - &usvg_options, - db, - // Decoders hand out their current frame buffer, which is recycled by - // the next decode, so they must not be shared between threads. - VideoDecodersWorker::new(options.prefetch_frames.max(1) * 2), - ¬ify_frame_ready, - ); - }); - }); + scope.spawn(|| run_worker(WorkerKind::Playback)); } + scope.spawn(|| run_worker(WorkerKind::Thumbnails)); #[cfg(feature = "audio")] if let (Some(audio), Some(audio_map)) = (audio.as_ref(), timeline.audio_map.as_ref()) { diff --git a/fframes-native-player/src/options.rs b/fframes-native-player/src/options.rs index c3a512ac..db8518fd 100644 --- a/fframes-native-player/src/options.rs +++ b/fframes-native-player/src/options.rs @@ -42,7 +42,8 @@ pub struct PlayerOptions<'a, 'media> { /// Play the video's audio map on the default output device. pub audio: bool, /// Threads building frame trees (`Video::render_frame`) in parallel. - /// Defaults to half of the available cores, at most 8. + /// Defaults to one worker for video sources so the decoder reads frames in order. + /// Other projects use half the available cores, up to eight workers. pub render_threads: Option, /// How many frames ahead of the playhead may be prepared. pub prefetch_frames: usize, @@ -73,8 +74,44 @@ impl Default for PlayerOptions<'_, '_> { impl PlayerOptions<'_, '_> { pub(crate) fn render_threads(&self) -> usize { - self.render_threads - .unwrap_or_else(|| (fframes::get_thread_count() / 2).clamp(1, 8)) - .max(1) + if let Some(count) = self.render_threads { + return count.max(1); + } + if self.media.is_some_and(MediaProvider::has_video_sources) { + return 1; + } + (fframes::get_thread_count() / 2).clamp(1, 8) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn video_sources_default_to_one_worker() { + let media = fframes::DynamicMediaProvider::default().with_video( + "clip.mp4", + fframes::media::VideoMedia { + path: "clip.mp4".into(), + metadata: None, + }, + ); + let media = fframes::CombinedMediaProvider::from([&() as &dyn MediaProvider, &media]); + let mut options = PlayerOptions { + media: Some(&media), + ..Default::default() + }; + assert_eq!(options.render_threads(), 1); + options.render_threads = Some(4); + assert_eq!(options.render_threads(), 4); + options.render_threads = Some(0); + assert_eq!(options.render_threads(), 1); + options.render_threads = None; + options.media = None; + assert_eq!( + options.render_threads(), + (fframes::get_thread_count() / 2).clamp(1, 8) + ); } } diff --git a/fframes-native-player/src/presenter/mod.rs b/fframes-native-player/src/presenter/mod.rs index 0cdf72f6..76b5d116 100644 --- a/fframes-native-player/src/presenter/mod.rs +++ b/fframes-native-player/src/presenter/mod.rs @@ -42,6 +42,7 @@ trait WindowTarget { // Fields drop in declaration order: the cached Skia objects before the target that owns // the context and the device. pub(crate) struct Presenter { + pub studio_layout: Option, render_cache: RenderCache, target: Box, window: Rc, @@ -50,7 +51,6 @@ pub(crate) struct Presenter { layout: ControlsLayout, /// The longest time label, sizes the time slots of the control bar. duration_label: String, - pub studio_layout: Option, audio_rows: usize, video_aspect: f32, } @@ -194,7 +194,8 @@ impl Presenter { canvas.restore(); } - if let Some(ui) = studio_layout.as_ref() { + let show_bar = view.is_none(); + if let Some(ui) = studio_layout.as_mut() { canvas.save(); canvas.scale((pixel_scale, pixel_scale)); if studio_open { @@ -208,7 +209,7 @@ impl Presenter { } canvas.restore(); } - if !studio_open { + if show_bar { bar.draw(canvas, layout, controls); } }) diff --git a/fframes-native-player/src/scheduler.rs b/fframes-native-player/src/scheduler.rs index 3a9f9f44..06f7ae25 100644 --- a/fframes-native-player/src/scheduler.rs +++ b/fframes-native-player/src/scheduler.rs @@ -43,6 +43,12 @@ enum JobKind { Thumbnail(usize), } +#[derive(Clone, Copy)] +pub(crate) enum WorkerKind { + Playback, + Thumbnails, +} + impl Scheduler { pub(crate) fn new( duration_in_frames: usize, @@ -101,17 +107,26 @@ impl Scheduler { /// frame rate instead of freezing. pub(crate) fn take(&self, playhead: u64) -> Option<(u64, usvgr::Tree)> { let mut state = self.lock(); + let advanced = state.playhead != playhead; + let first_frame = state.displayed.is_none(); state.playhead = playhead; - let newer = state.ready.split_off(&(playhead + 1)); - let due = std::mem::replace(&mut state.ready, newer); - let taken = due.into_iter().next_back(); + let mut taken = None; + while state + .ready + .first_key_value() + .is_some_and(|(&position, _)| position <= playhead) + { + taken = state.ready.pop_first(); + } if let Some((position, _)) = &taken { state.displayed = Some(*position); } drop(state); - self.workers_wakeup.notify_all(); + if advanced || (first_frame && taken.is_some()) { + self.workers_wakeup.notify_all(); + } taken } @@ -148,23 +163,38 @@ impl Scheduler { }); } - // Sample the overview once, only after all due playback jobs have been assigned. - if state.next_thumbnail < self.thumbnails { - let slot = state.next_thumbnail; - state.next_thumbnail += 1; - let frame = ((2 * slot + 1) as u128 * u128::from(self.duration) - / (2 * self.thumbnails) as u128) as usize; - return Some(Job { - frame, - kind: JobKind::Thumbnail(slot), - }); - } + state = self + .workers_wakeup + .wait(state) + .unwrap_or_else(std::sync::PoisonError::into_inner); + } + } + // The filmstrip seeks across the video with its own decoder. + fn next_thumbnail_job(&self) -> Option { + let mut state = self.lock(); + while !state.shutdown && state.displayed.is_none() { state = self .workers_wakeup .wait(state) .unwrap_or_else(std::sync::PoisonError::into_inner); } + if state.shutdown || state.next_thumbnail >= self.thumbnails { + return None; + } + let slot = state.next_thumbnail; + state.next_thumbnail += 1; + let frame = ((2 * slot + 1) as u128 * u128::from(self.duration) + / (2 * self.thumbnails) as u128) as usize; + Some(Job { + frame, + kind: JobKind::Thumbnail(slot), + }) + } + + pub(crate) fn is_current_seek(&self, position: u64, epoch: u64) -> bool { + let state = self.lock(); + state.playhead == position && state.epoch == epoch } fn complete(&self, position: u64, epoch: u64, tree: usvgr::Tree) -> bool { @@ -181,22 +211,25 @@ impl Scheduler { !is_stale } - /// Runs on a worker thread until `shutdown` is called. `on_frame_ready` is invoked - /// after every frame that was added to the queue (used to wake the event loop). pub(crate) fn run_worker<'a, 'media: 'a, TVideo: Video + Sync>( &self, video: &'a TVideo, ctx: &FFramesContext<'a, 'media>, usvg_options: &usvgr::Options<'_>, font_db: &usvgr::fontdb::Database, - video_decoders: VideoDecodersWorker, + kind: WorkerKind, on_frame_ready: &(dyn Fn(FrameReady) + Sync), ) { // Text caches are `Rc` based and therefore owned by the worker. let text_cache = TextCache::new(100); let mut converter_cache = usvgr::Cache::new_with_text_cache(100); + // Queued trees own their pixels. The decoder only needs its last conversion. + let video_decoders = VideoDecodersWorker::default(); - while let Some(job) = self.next_job() { + while let Some(job) = match kind { + WorkerKind::Playback => self.next_job(), + WorkerKind::Thumbnails => self.next_thumbnail_job(), + } { let frame = fframes::Frame::__internal_make_for_renderer( job.frame, job.frame, @@ -223,11 +256,68 @@ impl Scheduler { on_frame_ready(FrameReady::Playback); } } - (Err(err), _) => eprintln!( - "fframes player: failed to render frame {}: {err}", - job.frame - ), + (Err(err), kind) => { + eprintln!( + "fframes player: failed to render frame {}: {err}", + job.frame + ); + if let JobKind::Playback { position, epoch } = kind { + on_frame_ready(FrameReady::Failed { position, epoch }); + } + } } } } } + +#[cfg(test)] +mod tests { + use super::*; + + fn tree() -> usvgr::Tree { + usvgr::Tree::from_str( + r#""#, + &usvgr::Options::default(), + &usvgr::fontdb::Database::new(), + ) + .unwrap() + } + + #[test] + fn taking_out_of_order_frames_preserves_future_frames_and_rejects_stale_jobs() { + let scheduler = Scheduler::new(100, 3, false, 2); + for index in 0..3 { + assert_eq!(scheduler.next_job().unwrap().frame, index); + } + for index in [2, 0, 1] { + assert!(scheduler.complete(index, 0, tree())); + } + assert_eq!(scheduler.take(1).unwrap().0, 1); + assert!(scheduler.take(1).is_none()); + assert!(!scheduler.complete(0, 0, tree())); + assert_eq!(scheduler.take(2).unwrap().0, 2); + scheduler.seek(50, false); + assert!(!scheduler.complete(3, 0, tree())); + assert_eq!(scheduler.next_job().unwrap().frame, 50); + assert!(scheduler.complete(50, 1, tree())); + assert_eq!(scheduler.take(50).unwrap().0, 50); + scheduler.shutdown(); + assert!(scheduler.next_job().is_none()); + } + + #[test] + fn thumbnails_do_not_consume_playback_jobs_and_stop_when_finished() { + let scheduler = Scheduler::new(100, 3, false, 2); + assert_eq!(scheduler.next_job().unwrap().frame, 0); + scheduler.complete(0, 0, tree()); + scheduler.take(0).unwrap(); + assert_eq!(scheduler.next_thumbnail_job().unwrap().frame, 25); + assert_eq!(scheduler.next_job().unwrap().frame, 1); + assert_eq!(scheduler.next_thumbnail_job().unwrap().frame, 75); + assert!(scheduler.next_thumbnail_job().is_none()); + assert_eq!(scheduler.next_job().unwrap().frame, 2); + let stopped = Scheduler::new(100, 3, false, 2); + stopped.shutdown(); + assert!(stopped.next_thumbnail_job().is_none()); + } +} diff --git a/fframes-native-player/src/studio.rs b/fframes-native-player/src/studio.rs index 768e24fc..49046f65 100644 --- a/fframes-native-player/src/studio.rs +++ b/fframes-native-player/src/studio.rs @@ -8,8 +8,8 @@ use fframes::{ TimeBase, TimelineIndex, media::PreloadedAudioData, }; use fframes_skia_renderer::skia_safe::{ - Canvas, Color, Contains, FilterMode, Font, Image, Paint, Path, PathBuilder, Point, RRect, Rect, - SamplingOptions, surfaces, + Canvas, Color, Contains, FilterMode, Font, IRect, Image, ImageInfo, Paint, Path, PathBuilder, + Point, RRect, Rect, RoundOut, SamplingOptions, surfaces, }; use crate::controls::{Control, Controls, ControlsState, draw_tabular, format_time, tabular_width}; @@ -18,7 +18,8 @@ pub(crate) struct Studio { pub index: TimelineIndex, pub video_aspect: f32, audio: Vec>, - pub thumbnails: Vec>, + thumbnails: Vec>, + thumbnail_revision: u64, } struct Clip { @@ -66,9 +67,15 @@ impl Studio { video_aspect, audio: pack_clips(clips), thumbnails: vec![None; thumbnail_count], + thumbnail_revision: 0, } } + pub fn set_thumbnail(&mut self, slot: usize, image: Image) { + self.thumbnails[slot] = Some(image); + self.thumbnail_revision += 1; + } + pub fn adjacent_scene(&self, frame: usize, next: bool) -> Option { let starts = self .index @@ -186,6 +193,16 @@ pub(crate) struct StudioLayout { pub portrait: bool, audio_slots: usize, film_height: f32, + timeline_cache: Option, +} + +struct TimelineCache { + image: Image, + bounds: IRect, + frames: Range, + thumbnail_revision: u64, + audio_scroll: usize, + scale: (f32, f32), } impl StudioLayout { @@ -212,6 +229,7 @@ impl StudioLayout { portrait: true, audio_slots: audio_rows.min(if panel >= 260. { 3 } else { 1 }), film_height: 0., + timeline_cache: None, }); } let audio_slots = audio_rows.min(if height >= 600. { @@ -241,6 +259,7 @@ impl StudioLayout { portrait: false, audio_slots, film_height, + timeline_cache: None, }) } @@ -401,12 +420,6 @@ impl StudioLayout { } } } - let y = at(state.frame); - ui.fill( - Rect::from_ltrb(film.left - 2., y - 1., track.right, y + 1.), - ORANGE, - ); - ui.round(Rect::from_xywh(film.left - 4., y - 4., 7., 8.), ORANGE, 2.); } fn buttons(&self) -> [(Control, Rect); 6] { @@ -493,11 +506,7 @@ impl StudioLayout { } pub fn draw_restore(&self, canvas: &Canvas, controls: &Controls, hovered: bool) { - let ui = Ink { - canvas, - font: controls.font(false, 1.), - medium: controls.font(true, 1.), - }; + let ui = Ink::new(canvas, controls); ui.round(self.restore, if hovered { LINE } else { SURFACE }, 7.); ui.text( "Studio · T", @@ -510,19 +519,21 @@ impl StudioLayout { } pub fn draw( - &self, + &mut self, canvas: &Canvas, controls: &Controls, state: &ControlsState, model: &Studio, scroll: &StudioScroll, ) { - let ui = Ink { - canvas, - font: controls.font(false, 1.), - medium: controls.font(true, 1.), - }; - ui.fill(self.timeline, SURFACE); + let ui = Ink::new(canvas, controls); + self.draw_cached_timeline(&ui, state, model, scroll); + self.draw_transport(&ui, controls, state); + self.draw_playhead(&ui, state); + } + + fn draw_transport(&self, ui: &Ink<'_>, controls: &Controls, state: &ControlsState) { + let canvas = ui.canvas; ui.fill(self.transport, SURFACE); ui.fill( Rect::from_xywh(0., self.transport.top, self.transport.width(), 1.), @@ -615,9 +626,90 @@ impl StudioLayout { Rect::from_xywh(0., self.transport.bottom - 1., self.transport.width(), 1.), LINE, ); + } + + fn draw_cached_timeline( + &mut self, + ui: &Ink<'_>, + state: &ControlsState, + model: &Studio, + scroll: &StudioScroll, + ) { + let canvas = ui.canvas; + let matrix = canvas.local_to_device_as_3x3(); + let scale = (matrix.scale_x(), matrix.scale_y()); + let valid = self.timeline_cache.as_ref().is_some_and(|cache| { + cache.frames.contains(&state.frame) + && cache.thumbnail_revision == model.thumbnail_revision + && cache.audio_scroll == scroll.audio + && cache.scale == scale + }); + if !valid { + // Align the panel to device pixels so its labels stay sharp at fractional DPI. + let bounds: IRect = matrix.map_rect(self.timeline).0.round_out(); + let info = ImageInfo::new_n32_premul(bounds.size(), None); + self.timeline_cache = canvas.new_surface(&info, None).map(|mut surface| { + let target = surface.canvas(); + target.clear(Color::TRANSPARENT); + target.translate((-bounds.left as f32, -bounds.top as f32)); + target.concat(&matrix); + self.draw_tracks( + &Ink { + canvas: target, + ..ui.clone() + }, + state, + model, + scroll, + ); + + // Scene labels change color at either end of an overlapping scene. + let mut frames = 0..usize::MAX; + for scene in &model.index.scenes { + for boundary in [scene.frames.start, scene.frames.end] { + if boundary <= state.frame { + frames.start = frames.start.max(boundary); + } else { + frames.end = frames.end.min(boundary); + } + } + } + TimelineCache { + image: surface.image_snapshot(), + bounds, + frames, + thumbnail_revision: model.thumbnail_revision, + audio_scroll: scroll.audio, + scale, + } + }); + } + if let Some(cache) = &self.timeline_cache { + canvas.save(); + canvas.reset_matrix(); + canvas.draw_image( + &cache.image, + (cache.bounds.left as f32, cache.bounds.top as f32), + None, + ); + canvas.restore(); + } else { + self.draw_tracks(ui, state, model, scroll); + } + } + + fn draw_tracks( + &self, + ui: &Ink<'_>, + state: &ControlsState, + model: &Studio, + scroll: &StudioScroll, + ) { + let canvas = ui.canvas; + ui.fill(self.timeline, SURFACE); let track = self.tracks(); if self.portrait { - return self.draw_portrait_tracks(&ui, state, model, scroll); + return self.draw_portrait_tracks(ui, state, model, scroll); } let scene_top = track.top + RULER; let duration = state.duration_in_frames; @@ -745,7 +837,21 @@ impl StudioLayout { } } } - let x = self.frame_position(state.frame, duration); + } + + fn draw_playhead(&self, ui: &Ink<'_>, state: &ControlsState) { + let track = self.tracks(); + if self.portrait { + let film = self.portrait_columns().0; + let y = self.frame_position(state.frame, state.duration_in_frames); + ui.fill( + Rect::from_ltrb(film.left - 2., y - 1., track.right, y + 1.), + ORANGE, + ); + ui.round(Rect::from_xywh(film.left - 4., y - 4., 7., 8.), ORANGE, 2.); + return; + } + let x = self.frame_position(state.frame, state.duration_in_frames); ui.fill( Rect::from_ltrb(x - 1., track.top, x + 1., track.bottom), ORANGE, @@ -768,13 +874,22 @@ fn tick_step(duration: usize, fps: usize, span: f32, spacing: f32) -> usize { .saturating_mul(unit) } +#[derive(Clone)] struct Ink<'a> { canvas: &'a Canvas, font: Font, medium: Font, } -impl Ink<'_> { +impl<'a> Ink<'a> { + fn new(canvas: &'a Canvas, controls: &Controls) -> Self { + Self { + canvas, + font: controls.font(false, 1.), + medium: controls.font(true, 1.), + } + } + fn paint(color: Color) -> Paint { let mut paint = Paint::default(); paint.set_anti_alias(true); @@ -862,6 +977,7 @@ mod tests { }, audio: vec![], thumbnails: vec![], + thumbnail_revision: 0, } } @@ -875,6 +991,92 @@ mod tests { assert_eq!(studio.adjacent_scene(0, false), None); } + #[test] + fn cached_timeline_matches_fresh_draws_after_scene_scroll_thumbnail_and_scale_changes() { + let controls = Controls::new(); + let mut model = studio(); + model.thumbnails = vec![None]; + model.audio = (0..4) + .map(|i| { + vec![Clip { + name: format!("Audio {i}"), + frames: i * 10..100, + waveform: None, + }] + }) + .collect(); + let mut scroll = StudioScroll::default(); + let mut state = ControlsState { + visible: true, + frame: 0, + duration_in_frames: 100, + fps: 30, + playing: true, + looping: true, + fullscreen: false, + hovered: None, + }; + let draw = |layout: &mut StudioLayout, + state: &ControlsState, + model: &Studio, + scroll: &StudioScroll, + scale: f32, + cached: bool| { + let size = (800. * scale) as i32; + let info = ImageInfo::new_n32_premul((size, size), None); + let mut surface = surfaces::raster(&info, None, None).unwrap(); + let canvas = surface.canvas(); + canvas.clear(Color::BLACK); + canvas.scale((scale, scale)); + if cached { + layout.draw(canvas, &controls, state, model, scroll); + } else { + let ui = Ink::new(canvas, &controls); + layout.draw_tracks(&ui, state, model, scroll); + layout.draw_transport(&ui, &controls, state); + layout.draw_playhead(&ui, state); + } + let mut pixels = vec![0; (size * size * 4) as usize]; + assert!(surface.read_pixels(&info, &mut pixels, size as usize * 4, (0, 0))); + pixels + }; + for aspect in [16. / 9., 9. / 16.] { + let mut layout = StudioLayout::new(800., 800., 4, aspect).unwrap(); + for scale in [1., 1.25, 2.] { + let mut first_id = None; + for frame in [0, 1, 49, 50, 59, 60, 99, 0] { + state.frame = frame; + assert!( + draw(&mut layout, &state, &model, &scroll, scale, true) + == draw(&mut layout, &state, &model, &scroll, scale, false), + "aspect={aspect}, frame={frame}, scale={scale}" + ); + let id = layout.timeline_cache.as_ref().unwrap().image.unique_id(); + if frame == 0 { + first_id = Some(id); + } else if frame == 1 { + assert_eq!(first_id, Some(id)); + } + } + } + scroll.audio = 1; + let mut thumbnail = surfaces::raster_n32_premul((20, 10)).unwrap(); + thumbnail.canvas().clear(Color::RED); + for updated_thumbnail in [false, true] { + if updated_thumbnail { + model.set_thumbnail(0, thumbnail.image_snapshot()); + } + assert!( + draw(&mut layout, &state, &model, &scroll, 2., true) + == draw(&mut layout, &state, &model, &scroll, 2., false), + "aspect={aspect}, updated_thumbnail={updated_thumbnail}" + ); + } + scroll.audio = 0; + model.thumbnails[0] = None; + } + } + #[test] fn timeline_seek_and_hit_regions_agree_across_sizes() { let model = studio(); diff --git a/fframes/src/media_provider.rs b/fframes/src/media_provider.rs index d9f8ad94..d60d0e2a 100644 --- a/fframes/src/media_provider.rs +++ b/fframes/src/media_provider.rs @@ -52,6 +52,11 @@ pub trait MediaProvider<'a>: Send + Sync + Debug { Vec::new() } + /// Whether the provider lists video files that need a decoder during playback. + fn has_video_sources(&self) -> bool { + !self.get_all_video_data().is_empty() + } + #[cfg(feature = "lottie")] fn get_all_lottie_data(&self) -> Vec<(&crate::Lottie, &str)> { Vec::new() @@ -241,6 +246,10 @@ impl<'a> MediaProvider<'a> for DynamicMediaProvider<'a> { .collect() } + fn has_video_sources(&self) -> bool { + !self.videos.is_empty() + } + fn get_all_video_data(&self) -> Vec<(&fframes_media::VideoMedia, &str)> { self.videos .iter() @@ -293,6 +302,10 @@ impl<'a, const N: usize> MediaProvider<'a> for CombinedMediaProvider<'a, N> { .find_map(|provider| provider.resolve_video(name)) } + fn has_video_sources(&self) -> bool { + self.0.iter().any(|provider| provider.has_video_sources()) + } + #[cfg(feature = "lottie")] fn resolve_lottie(&'a self, name: &str) -> Option<&'a crate::Lottie> { self.0 From 476ce22f245f00014bf253d2731cf893af79a98c Mon Sep 17 00:00:00 2001 From: dmtrKovalenko Date: Wed, 28 Oct 2026 20:02:33 +0000 Subject: [PATCH 05/21] perf(player): Keep prefetched frames when playback resumes --- fframes-native-player/src/app.rs | 50 ++++++++++++++++++++++++-------- 1 file changed, 38 insertions(+), 12 deletions(-) diff --git a/fframes-native-player/src/app.rs b/fframes-native-player/src/app.rs index 1a2ec683..40babbef 100644 --- a/fframes-native-player/src/app.rs +++ b/fframes-native-player/src/app.rs @@ -183,9 +183,12 @@ impl<'s> App<'s> { /// Restarts playback at `position`, all the prepared frames are discarded. fn seek_to(&mut self, position: u64, playing: bool) { self.scrub_target = None; - self.clock.set(position, playing); self.scheduler.seek(position, self.looping); + self.set_playback(position, playing); + } + fn set_playback(&mut self, position: u64, playing: bool) { + self.clock.set(position, playing); #[cfg(feature = "audio")] if let Some(audio) = self.audio { audio.reset(position, self.config.fps, playing, self.looping); @@ -203,12 +206,17 @@ impl<'s> App<'s> { fn toggle_play(&mut self) { let frame = self.playhead(); - if self.clock.is_playing() { - self.seek_to(frame as u64, false); - } else if !self.looping && frame >= self.last_frame() { + let playing = !self.clock.is_playing(); + let position = self.clock.position(); + if playing && !self.looping && frame >= self.last_frame() { self.seek_to(0, true); + } else if self.scrub_target.is_none() + && self.scheduler.frame_of(position, self.looping) == frame + { + // The workers have already prepared frames ahead of this position. + self.set_playback(position, playing); } else { - self.seek_to(frame as u64, true); + self.seek_to(frame as u64, playing); } } @@ -710,8 +718,7 @@ impl ApplicationHandler for App<'_> { mod tests { use super::*; - #[test] - fn failed_scrub_continues_at_the_latest_pointer_position() { + fn paused_app(scheduler: &Scheduler) -> App<'_> { let timeline = fframes::ResolvedRenderingTimeline { audio_map: None, scenes: None, @@ -721,11 +728,10 @@ mod tests { fps: 30, sample_rate: 44100, }; - let scheduler = Scheduler::new(100, 3, false, 0); - let mut app = App::new( + App::new( AppConfig { studio: Studio::new(time, &timeline, None, 0, 1.), - title: "scrub", + title: "test", window_size: LogicalSize::new(640, 480), fit_portrait_to_screen: false, backend: PlayerBackend::Cpu, @@ -737,10 +743,30 @@ mod tests { autoplay: false, start_frame: 0, }, - &scheduler, + scheduler, #[cfg(feature = "audio")] None, - ); + ) + } + + #[test] + fn resuming_keeps_prefetched_frames_unless_playback_restarts() { + let scheduler = Scheduler::new(100, 3, false, 0); + let mut app = paused_app(&scheduler); + app.seek_to(10, false); + app.toggle_play(); + assert!(app.clock.is_playing()); + assert!(scheduler.is_current_seek(10, 2)); + + app.seek_to(99, false); + app.toggle_play(); + assert!(scheduler.is_current_seek(0, 4)); + } + + #[test] + fn failed_scrub_continues_at_the_latest_pointer_position() { + let scheduler = Scheduler::new(100, 3, false, 0); + let mut app = paused_app(&scheduler); app.seek_to(10, false); app.scrub_target = Some(20); // A stale completion (even at the same position) and a failed lookahead From 23a429521da28e689b5d57d745116dc6f3da5aa0 Mon Sep 17 00:00:00 2001 From: dmtrKovalenko Date: Wed, 28 Oct 2026 20:43:23 +0000 Subject: [PATCH 06/21] perf(player): Buffer finished frames and share video decoders Prepare up to two seconds of preview frames while the current frame has priority. Playback workers share a decoder per video source and retain the decoded planes while another worker advances the stream. --- fframes-media/src/video_decoder.rs | 84 +++++--- fframes-media/tests/video_decoder_eof.rs | 30 +++ fframes-native-player/src/app.rs | 105 +++++++++- fframes-native-player/src/lib.rs | 3 +- fframes-native-player/src/options.rs | 26 ++- fframes-native-player/src/presenter/metal.rs | 9 + fframes-native-player/src/presenter/mod.rs | 156 ++++++++++---- fframes-native-player/src/presenter/vulkan.rs | 9 + fframes-native-player/src/scheduler.rs | 192 ++++++++++++++---- fframes/src/video_data.rs | 185 ++++++----------- 10 files changed, 555 insertions(+), 244 deletions(-) diff --git a/fframes-media/src/video_decoder.rs b/fframes-media/src/video_decoder.rs index 3cc81605..d0ae8e16 100644 --- a/fframes-media/src/video_decoder.rs +++ b/fframes-media/src/video_decoder.rs @@ -92,7 +92,6 @@ pub fn decode_video_resource(s: &str) -> Option<(&str, i64)> { struct VideoStreamInfo { hw_pix_fmt: Option, stream_index: i32, - codec_ctx: *mut AVCodecContext, width: i32, height: i32, display_width: i32, @@ -119,6 +118,9 @@ struct SwsScaler { linesize: [i32; 4], } +// The scaler exclusively owns its SwsContext. All conversions use it under one mutex. +unsafe impl Send for SwsScaler {} + impl Drop for SwsScaler { fn drop(&mut self) { unsafe { @@ -375,6 +377,7 @@ pub struct FFmpegDecoder { frame_buf: Arc, fmt_ctx: *mut AVFormatContext, video_stream_info: VideoStreamInfo, + codec_ctx: *mut AVCodecContext, pkt: *mut AVPacket, /// Offsets are counted in this time base: `speed / target_fps` seconds per offset. custom_time_base: AVRational, @@ -405,10 +408,10 @@ struct ConvertedFrames { #[derive(Debug)] pub struct FFmpegFrameBuf { - resource_name: String, + resource_name: Arc, latest_av_frame: *mut AVFrame, video_stream_info: VideoStreamInfo, - converted: Mutex, + converted: Arc>, } impl Drop for FFmpegFrameBuf { @@ -447,11 +450,11 @@ impl FFmpegFrameBuf { Ok(FFmpegFrameBuf { video_stream_info, latest_av_frame: av_frame, - resource_name, - converted: Mutex::new(ConvertedFrames { + resource_name: resource_name.into(), + converted: Arc::new(Mutex::new(ConvertedFrames { scaler: sws_ctx, images: VecDeque::with_capacity(buf_size.max(1)), - }), + })), }) } } @@ -559,6 +562,22 @@ impl FFmpegDecoder { Arc::clone(&self.frame_buf) } + /// Retains the decoded planes so this frame can be read after the decoder advances + /// or seeks. `FFmpeg` clones the frame header and reference-counts the pixel buffers. + pub fn retained_frame(&self) -> Result> { + let frame = &self.frame_buf; + let latest_av_frame = unsafe { av_frame_clone(frame.latest_av_frame) }; + if latest_av_frame.is_null() { + return Err(FFramesMediaError::LibAVAllocationError("frame")); + } + Ok(Arc::new(FFmpegFrameBuf { + latest_av_frame, + resource_name: Arc::clone(&frame.resource_name), + video_stream_info: self.video_stream_info, + converted: Arc::clone(&frame.converted), + })) + } + pub fn get_decoded_image_in_buf(&self, pts: i64) -> Option> { self.frame_buf .converted @@ -639,7 +658,7 @@ impl FFmpegDecoder { ))); } - let video_stream_info = Self::open_codec_context(fmt_ctx)?; + let (codec_ctx, video_stream_info) = Self::open_codec_context(fmt_ctx)?; let pkt = av_packet_alloc(); if pkt.is_null() { avformat_close_input(&raw mut fmt_ctx); @@ -674,6 +693,7 @@ impl FFmpegDecoder { }; Ok(FFmpegDecoder { + codec_ctx, pkt, fmt_ctx, hw_frame, @@ -694,7 +714,9 @@ impl FFmpegDecoder { } } - unsafe fn open_codec_context(fmt_ctx: *mut AVFormatContext) -> Result { + unsafe fn open_codec_context( + fmt_ctx: *mut AVFormatContext, + ) -> Result<(*mut AVCodecContext, VideoStreamInfo)> { unsafe { let ret = av_find_best_stream( fmt_ctx, @@ -772,21 +794,23 @@ impl FFmpegDecoder { } None => (width, height), }; - Ok(VideoStreamInfo { - hw_pix_fmt, - stream_index: video_stream_idx, - codec_ctx: video_dec_ctx, - width, - height, - display_width, - display_height, - pixel_format: (*video_dec_ctx).pix_fmt, - color_space: (*video_dec_ctx).colorspace, - color_range: (*video_dec_ctx).color_range, - time_base: (*stream).time_base, - duration: stream_duration(fmt_ctx, stream), - frame_rate: (*stream).r_frame_rate, - }) + Ok(( + video_dec_ctx, + VideoStreamInfo { + hw_pix_fmt, + stream_index: video_stream_idx, + width, + height, + display_width, + display_height, + pixel_format: (*video_dec_ctx).pix_fmt, + color_space: (*video_dec_ctx).colorspace, + color_range: (*video_dec_ctx).color_range, + time_base: (*stream).time_base, + duration: stream_duration(fmt_ctx, stream), + frame_rate: (*stream).r_frame_rate, + }, + )) } } @@ -814,7 +838,7 @@ impl FFmpegDecoder { } (*self.frame_buf.latest_av_frame).pts = -1; - avcodec_flush_buffers(self.video_stream_info.codec_ctx); + avcodec_flush_buffers(self.codec_ctx); self.draining = false; self.has_decoded_frame = false; self.last_pts = Some(timestamp); @@ -860,6 +884,11 @@ impl FFmpegDecoder { unsafe fn transfer_hardware_surface_data(&self, target_frame: *mut AVFrame) -> Result<()> { match self.video_stream_info.hw_pix_fmt { Some(hw_pix_fmt) if hw_pix_fmt as i32 == (*target_frame).format => { + // A retained frame can still own the last transfer's software planes. + let output = self.frame_buf.latest_av_frame; + if !(*output).buf[0].is_null() && av_frame_is_writable(output) == 0 { + av_frame_unref(output); + } if (*self.frame_buf.latest_av_frame).format == (AVPixelFormat::AV_PIX_FMT_NONE as i32) { @@ -938,7 +967,7 @@ impl FFmpegDecoder { loop { // A previous call may have returned with more decoded frames queued. // Consume them before sending another packet (which could return EAGAIN). - let ret = avcodec_receive_frame(self.video_stream_info.codec_ctx, self.recv_frame); + let ret = avcodec_receive_frame(self.codec_ctx, self.recv_frame); match ret { 0 => { av_frame_unref(target_frame); @@ -994,7 +1023,7 @@ impl FFmpegDecoder { } else { self.pkt.cast_const() }; - let ret = avcodec_send_packet(self.video_stream_info.codec_ctx, packet); + let ret = avcodec_send_packet(self.codec_ctx, packet); av_packet_unref(self.pkt); if ret < 0 { return Err(FFramesMediaError::LibAVAudioDecodingError(( @@ -1013,7 +1042,7 @@ impl Drop for FFmpegDecoder { unsafe { av_frame_free(&raw mut self.recv_frame); av_frame_free(&raw mut self.hw_frame); - avcodec_free_context(&raw mut self.video_stream_info.codec_ctx); + avcodec_free_context(&raw mut self.codec_ctx); avformat_close_input(&raw mut self.fmt_ctx); av_packet_free(&raw mut self.pkt); } @@ -1219,7 +1248,6 @@ mod color_tests { SwsScaler::new(VideoStreamInfo { hw_pix_fmt: None, stream_index: 0, - codec_ctx: ptr::null_mut(), width: WIDTH, height: HEIGHT, display_width: WIDTH, diff --git a/fframes-media/tests/video_decoder_eof.rs b/fframes-media/tests/video_decoder_eof.rs index b6a919a1..c70f321e 100644 --- a/fframes-media/tests/video_decoder_eof.rs +++ b/fframes-media/tests/video_decoder_eof.rs @@ -98,6 +98,36 @@ fn assert_frame(decoder: &mut FFmpegDecoder, index: i64) { } } +#[test] +fn retained_frame_keeps_its_pixels_and_timestamp_across_decode_and_seek() { + let video = VideoFixture::new(3); + let mut decoder = unsafe { FFmpegDecoder::new(&video.path(), FPS, 1) }.unwrap(); + unsafe { + assert!(decoder.decode_up_to(3).unwrap()); + let held = decoder.retained_frame().unwrap(); + let pts = held.get_pts(); + let expected = held + .convert_last_decoded_frame_into_svg_image(None) + .unwrap(); + + std::thread::scope(|scope| { + let decode = scope.spawn(move || { + assert_frame(&mut decoder, 5); + decoder.seek_to_offset(0).unwrap(); + assert_frame(&mut decoder, 0); + }); + assert_eq!(held.get_pts(), pts); + decode.join().unwrap(); + }); + + assert_eq!(held.get_pts(), pts); + let actual = held + .convert_last_decoded_frame_into_svg_image(None) + .unwrap(); + assert_eq!(actual.data, expected.data); + } +} + #[test] fn b_frames_are_drained_and_decoding_can_restart_after_eof() { let video = VideoFixture::new(3); diff --git a/fframes-native-player/src/app.rs b/fframes-native-player/src/app.rs index 40babbef..3d26946f 100644 --- a/fframes-native-player/src/app.rs +++ b/fframes-native-player/src/app.rs @@ -1,8 +1,8 @@ +use std::collections::BTreeMap; use std::fmt::Write as _; use std::rc::Rc; use std::time::{Duration, Instant}; -use fframes::usvgr; use winit::application::ApplicationHandler; use winit::dpi::{LogicalSize, PhysicalPosition}; use winit::event::{ElementState, KeyEvent, MouseButton, MouseScrollDelta, WindowEvent}; @@ -14,12 +14,14 @@ use crate::controls::{Control, ControlsState, format_time}; use crate::error::PlayerError; use crate::options::PlayerBackend; use crate::presenter::Presenter; -use crate::scheduler::Scheduler; +use crate::scheduler::{PlaybackFrame, Scheduler}; use crate::studio::{Studio, StudioLayout, StudioScroll}; -use fframes_skia_renderer::skia_safe::Point; +use fframes_skia_renderer::skia_safe::{Image, Point}; /// The control bar hides after this long without pointer movement while playing. const CONTROLS_IDLE_TIMEOUT: Duration = Duration::from_millis(2500); +/// Estimated RGBA storage for the current and buffered window-size images. +const BUFFER_BYTES: u64 = 512 << 20; /// A frame worker finished a playback or filmstrip job. pub(crate) enum FrameReady { @@ -78,6 +80,7 @@ pub(crate) struct AppConfig<'a> { pub looping: bool, pub autoplay: bool, pub start_frame: usize, + pub limit_prefetch_memory: bool, } pub(crate) struct App<'s> { @@ -87,12 +90,17 @@ pub(crate) struct App<'s> { audio: Option<&'s crate::audio::AudioOutput>, window: Option>, + // Drop texture-backed images before the presenter and its Skia context. + current: Option<(u64, PlaybackFrame)>, + buffered: BTreeMap, presenter: Option, + redraw_pending: bool, + draw_time: Duration, + snapshots_available: bool, pub(crate) error: Option, clock: Clock, looping: bool, - current: Option<(u64, usvgr::Tree)>, /// Toggled with `b`. controls_enabled: bool, controls_visible: bool, @@ -138,7 +146,11 @@ impl<'s> App<'s> { #[cfg(feature = "audio")] audio, window: None, + buffered: BTreeMap::new(), presenter: None, + redraw_pending: false, + draw_time: Duration::ZERO, + snapshots_available: true, error: None, clock, current: None, @@ -183,7 +195,13 @@ impl<'s> App<'s> { /// Restarts playback at `position`, all the prepared frames are discarded. fn seek_to(&mut self, position: u64, playing: bool) { self.scrub_target = None; - self.scheduler.seek(position, self.looping); + if !self + .scheduler + .seek_buffered(position, self.looping, &mut self.buffered) + { + self.buffered.clear(); + self.scheduler.seek(position, self.looping); + } self.set_playback(position, playing); } @@ -231,7 +249,7 @@ impl<'s> App<'s> { } } - if let Some(frame) = self.scheduler.take(position) { + if let Some(frame) = self.scheduler.take(position, &mut self.buffered) { let next_frame = self.scheduler.frame_of(frame.0, self.looping); self.current = Some(frame); self.needs_redraw = true; @@ -247,6 +265,7 @@ impl<'s> App<'s> { if self.needs_redraw { self.needs_redraw = false; if let Some(window) = &self.window { + self.redraw_pending = true; window.request_redraw(); } } @@ -383,7 +402,21 @@ impl<'s> App<'s> { Point::new(pos.x, pos.y) } + fn invalidate_view(&mut self) { + if !self.buffered.is_empty() || matches!(self.current, Some((_, PlaybackFrame::Image(_)))) { + self.buffered.clear(); + if matches!(self.current, Some((_, PlaybackFrame::Image(_)))) { + self.current = None; + self.presented = None; + } + self.scheduler.seek(self.clock.position(), self.looping); + } + self.snapshots_available = true; + self.needs_redraw = true; + } + fn toggle_studio(&mut self) { + self.invalidate_view(); self.studio_open = !self.studio_open; self.drag = None; self.hovered = None; @@ -555,7 +588,54 @@ impl<'s> App<'s> { } } + /// Start at most one speculative draw before returning to the event loop. + fn draw_ahead(&mut self) -> bool { + if self.redraw_pending + || self.drag.is_some() + || self.scrub_target.is_some() + || !self.snapshots_available + || self.presented != Some(self.clock.position()) + { + return false; + } + if let Some(deadline) = self.clock.next_frame_at() + && deadline.saturating_duration_since(Instant::now()) + <= self.draw_time + Duration::from_millis(2) + { + return false; + } + let Some(window) = &self.window else { + return false; + }; + let size = window.inner_size(); + let bytes = (u64::from(size.width) * u64::from(size.height)).saturating_mul(4); + // Leave room for the displayed image and keep large/high-DPI windows bounded. + if bytes == 0 + || (self.config.limit_prefetch_memory + && bytes.saturating_mul(self.buffered.len() as u64 + 2) > BUFFER_BYTES) + { + return false; + } + let Some(presenter) = &mut self.presenter else { + return false; + }; + let Some((position, tree)) = self.scheduler.next_to_draw() else { + return false; + }; + let start = Instant::now(); + if let Some(image) = presenter.snapshot(&tree, self.studio_open) { + self.draw_time = start.elapsed(); + self.buffered.insert(position, image); + true + } else { + self.scheduler.put_back(position, tree); + self.snapshots_available = false; + false + } + } + fn redraw(&mut self) -> Result<(), PlayerError> { + self.redraw_pending = false; let controls = ControlsState { visible: self.controls_visible, frame: self.playhead(), @@ -568,6 +648,7 @@ impl<'s> App<'s> { }; if let Some(presenter) = self.presenter.as_mut() { + let start = Instant::now(); presenter.present( self.current.as_ref().map(|(_, tree)| tree), &controls, @@ -577,6 +658,11 @@ impl<'s> App<'s> { )?; let position = self.current.as_ref().map(|(position, _)| *position); + if position != self.presented + && matches!(self.current, Some((_, PlaybackFrame::Tree(_)))) + { + self.draw_time = start.elapsed(); + } if position.is_some() && position != self.presented { self.frames_shown += 1; } @@ -654,7 +740,7 @@ impl ApplicationHandler for App<'_> { if let Some(Err(err)) = self.presenter.as_mut().map(Presenter::resize) { return self.fail(event_loop, err); } - self.needs_redraw = true; + self.invalidate_view(); } WindowEvent::RedrawRequested => { if let Err(err) = self.redraw() { @@ -706,6 +792,10 @@ impl ApplicationHandler for App<'_> { fn about_to_wait(&mut self, event_loop: &ActiveEventLoop) { self.tick(); + if self.draw_ahead() { + event_loop.set_control_flow(ControlFlow::Poll); + return; + } event_loop.set_control_flow(match self.clock.next_frame_at() { Some(next_frame) => ControlFlow::WaitUntil(next_frame), // Paused: sleep until input or a worker delivers the requested frame. @@ -742,6 +832,7 @@ mod tests { looping: false, autoplay: false, start_frame: 0, + limit_prefetch_memory: true, }, scheduler, #[cfg(feature = "audio")] diff --git a/fframes-native-player/src/lib.rs b/fframes-native-player/src/lib.rs index 1a7a0639..8a61845a 100644 --- a/fframes-native-player/src/lib.rs +++ b/fframes-native-player/src/lib.rs @@ -139,7 +139,7 @@ pub fn play<'a, 'media: 'a, TVideo: Video + Sync>( .min(timeline.duration_in_frames); let scheduler = Scheduler::new( timeline.duration_in_frames, - options.prefetch_frames, + options.lookahead(TVideo::FPS), options.looping, thumbnail_count, ); @@ -171,6 +171,7 @@ pub fn play<'a, 'media: 'a, TVideo: Video + Sync>( looping: options.looping, autoplay: options.autoplay, start_frame: options.start_frame, + limit_prefetch_memory: options.prefetch_frames == usize::MAX, }, &scheduler, #[cfg(feature = "audio")] diff --git a/fframes-native-player/src/options.rs b/fframes-native-player/src/options.rs index db8518fd..f62e8dd5 100644 --- a/fframes-native-player/src/options.rs +++ b/fframes-native-player/src/options.rs @@ -45,7 +45,10 @@ pub struct PlayerOptions<'a, 'media> { /// Defaults to one worker for video sources so the decoder reads frames in order. /// Other projects use half the available cores, up to eight workers. pub render_threads: Option, - /// How many frames ahead of the playhead may be prepared. + /// Maximum number of future frames to prepare. The default (`usize::MAX`) uses + /// two seconds at the video's frame rate. Zero disables preparation ahead of time. + /// The automatic setting also limits finished frames to an estimated 512 MiB. + /// An explicit frame count takes precedence over that limit. pub prefetch_frames: usize, /// Master bus of the audio mix, see `RenderOptions::audio_mix`. pub audio_mix: fframes::AudioMixOptions, @@ -66,13 +69,21 @@ impl Default for PlayerOptions<'_, '_> { start_frame: 0, audio: true, render_threads: None, - prefetch_frames: 12, + prefetch_frames: usize::MAX, audio_mix: fframes::AudioMixOptions::default(), } } } impl PlayerOptions<'_, '_> { + pub(crate) fn lookahead(&self, fps: usize) -> usize { + if self.prefetch_frames == usize::MAX { + fps.saturating_mul(2) + } else { + self.prefetch_frames + } + } + pub(crate) fn render_threads(&self) -> usize { if let Some(count) = self.render_threads { return count.max(1); @@ -88,6 +99,17 @@ impl PlayerOptions<'_, '_> { mod tests { use super::*; + #[test] + fn default_lookahead_tracks_the_video_frame_rate() { + let mut options = PlayerOptions::default(); + assert_eq!(options.lookahead(24), 48); + assert_eq!(options.lookahead(60), 120); + options.prefetch_frames = 8; + assert_eq!(options.lookahead(24), 8); + options.prefetch_frames = 0; + assert_eq!(options.lookahead(60), 0); + } + #[test] fn video_sources_default_to_one_worker() { let media = fframes::DynamicMediaProvider::default().with_video( diff --git a/fframes-native-player/src/presenter/metal.rs b/fframes-native-player/src/presenter/metal.rs index 66e4dcaf..3b9816ef 100644 --- a/fframes-native-player/src/presenter/metal.rs +++ b/fframes-native-player/src/presenter/metal.rs @@ -57,6 +57,15 @@ impl MetalTarget { } impl WindowTarget for MetalTarget { + fn gpu_context( + &mut self, + ) -> Option<( + &mut DirectContext, + Option<&fframes_skia_renderer::QueueLock>, + )> { + Some((&mut self.gpu, None)) + } + fn is_gpu(&self) -> bool { true } diff --git a/fframes-native-player/src/presenter/mod.rs b/fframes-native-player/src/presenter/mod.rs index 76b5d116..14df02cc 100644 --- a/fframes-native-player/src/presenter/mod.rs +++ b/fframes-native-player/src/presenter/mod.rs @@ -12,18 +12,24 @@ use std::rc::Rc; use fframes::usvgr; use fframes_skia_renderer::render::RenderCache; -use fframes_skia_renderer::skia_safe::{Canvas, Color, Rect}; +use fframes_skia_renderer::skia_safe::{Canvas, Color, Image, ImageInfo, Rect, gpu, surfaces}; +use fframes_skia_renderer::{QueueLock, lock_queue}; use winit::window::Window; use crate::controls::{Controls, ControlsLayout, ControlsState}; use crate::error::PlayerResult; use crate::options::PlayerBackend; +use crate::scheduler::PlaybackFrame; use crate::studio::{Studio, StudioLayout, StudioScroll}; /// The images of the window that Skia draws into. trait WindowTarget { fn is_gpu(&self) -> bool; + fn gpu_context(&mut self) -> Option<(&mut gpu::DirectContext, Option<&QueueLock>)> { + None + } + /// An empty cache for the canvas of this target. A target on a GPU that the backend /// shares with wgpu samples the wgpu layers of the frames in place. fn render_cache(&self) -> RenderCache { @@ -128,10 +134,47 @@ impl Presenter { Ok(()) } - /// Renders `tree` letterboxed into the window with the controls on top and presents it. + /// Draws just the scene. Controls and the timeline are composited when it is shown. + pub(crate) fn snapshot(&mut self, tree: &usvgr::Tree, studio_open: bool) -> Option { + let size = self.window.inner_size(); + let info = ImageInfo::new_n32_premul((size.width as i32, size.height as i32), None); + let mut context = self.target.gpu_context(); + let mut surface = match context.as_mut() { + Some((gpu, queue)) => { + let _lock = lock_queue(*queue); + gpu::surfaces::render_target( + gpu, + gpu::Budgeted::Yes, + &info, + None, + gpu::SurfaceOrigin::TopLeft, + None, + false, + None, + ) + } + None => surfaces::raster(&info, None, None), + }?; + draw_scene( + surface.canvas(), + Some(tree), + self.studio_layout.as_ref().filter(|_| studio_open), + self.window.scale_factor() as f32, + self.background, + &mut self.render_cache, + ); + let image = surface.image_snapshot(); + if let Some((gpu, queue)) = context { + let _lock = lock_queue(queue); + gpu.flush_and_submit_surface(&mut surface, gpu::SyncCpu::Yes); + } + Some(image) + } + + /// Draws the current frame and overlays the controls before presenting the window. pub(crate) fn present( &mut self, - tree: Option<&usvgr::Tree>, + frame: Option<&PlaybackFrame>, controls: &ControlsState, model: &Studio, scroll: &StudioScroll, @@ -149,49 +192,20 @@ impl Presenter { } = self; target.present(&mut |canvas| { - let size = canvas.base_layer_size(); - let (width, height) = (size.width as f32, size.height as f32); let view = studio_layout.as_ref().filter(|_| studio_open); - let viewport = view.map_or_else( - || Rect::from_wh(width, height), - |ui| { - let stage = ui.stage; - Rect::from_ltrb( - stage.left * pixel_scale, - stage.top * pixel_scale, - stage.right * pixel_scale, - stage.bottom * pixel_scale, - ) - }, - ); - canvas.clear(if view.is_some() { - Color::from_rgb(14, 14, 14) - } else { - Color::BLACK - }); - - if let Some(tree) = tree { - let video = tree.size(); - let scale = - (viewport.width() / video.width()).min(viewport.height() / video.height()); - let offset_x = viewport.center_x() - video.width() * scale / 2.; - let offset_y = if view.is_some_and(|ui| ui.portrait) { - viewport.top - } else { - viewport.center_y() - video.height() * scale / 2. - }; - - canvas.save(); - canvas.translate((offset_x, offset_y)); - canvas.scale((scale, scale)); - canvas.clip_rect( - Rect::from_wh(video.width(), video.height()), - None, - Some(true), - ); - canvas.draw_color(*background, None); - fframes_skia_renderer::render::render_tree(tree, canvas, render_cache); - canvas.restore(); + match frame { + Some(PlaybackFrame::Image(image)) => { + canvas.draw_image(image, (0, 0), None); + } + Some(PlaybackFrame::Tree(tree)) => draw_scene( + canvas, + Some(tree), + view, + pixel_scale, + *background, + render_cache, + ), + None => draw_scene(canvas, None, view, pixel_scale, *background, render_cache), } let show_bar = view.is_none(); @@ -216,6 +230,58 @@ impl Presenter { } } +fn draw_scene( + canvas: &Canvas, + tree: Option<&usvgr::Tree>, + view: Option<&StudioLayout>, + pixel_scale: f32, + background: Color, + render_cache: &mut RenderCache, +) { + let size = canvas.base_layer_size(); + let (width, height) = (size.width as f32, size.height as f32); + let viewport = view.map_or_else( + || Rect::from_wh(width, height), + |ui| { + let stage = ui.stage; + Rect::from_ltrb( + stage.left * pixel_scale, + stage.top * pixel_scale, + stage.right * pixel_scale, + stage.bottom * pixel_scale, + ) + }, + ); + canvas.clear(if view.is_some() { + Color::from_rgb(14, 14, 14) + } else { + Color::BLACK + }); + + if let Some(tree) = tree { + let video = tree.size(); + let scale = (viewport.width() / video.width()).min(viewport.height() / video.height()); + let offset_x = viewport.center_x() - video.width() * scale / 2.; + let offset_y = if view.is_some_and(|ui| ui.portrait) { + viewport.top + } else { + viewport.center_y() - video.height() * scale / 2. + }; + + canvas.save(); + canvas.translate((offset_x, offset_y)); + canvas.scale((scale, scale)); + canvas.clip_rect( + Rect::from_wh(video.width(), video.height()), + None, + Some(true), + ); + canvas.draw_color(background, None); + fframes_skia_renderer::render::render_tree(tree, canvas, render_cache); + canvas.restore(); + } +} + fn create_target( window: &Rc, backend: PlayerBackend, diff --git a/fframes-native-player/src/presenter/vulkan.rs b/fframes-native-player/src/presenter/vulkan.rs index 73740b6a..03f70e26 100644 --- a/fframes-native-player/src/presenter/vulkan.rs +++ b/fframes-native-player/src/presenter/vulkan.rs @@ -369,6 +369,15 @@ impl VulkanTarget { } impl WindowTarget for VulkanTarget { + fn gpu_context( + &mut self, + ) -> Option<( + &mut DirectContext, + Option<&fframes_skia_renderer::QueueLock>, + )> { + Some((&mut self.gpu, Some(&self.queue_lock))) + } + fn is_gpu(&self) -> bool { true } diff --git a/fframes-native-player/src/scheduler.rs b/fframes-native-player/src/scheduler.rs index 06f7ae25..0656e0ab 100644 --- a/fframes-native-player/src/scheduler.rs +++ b/fframes-native-player/src/scheduler.rs @@ -9,6 +9,18 @@ use std::sync::{Condvar, Mutex, MutexGuard}; use crate::{app::FrameReady, studio::thumbnail}; use fframes::{FFramesContext, TextCache, Video, VideoDecodersWorker, usvgr}; +use fframes_skia_renderer::skia_safe::Image; + +/// A tree can go straight to the window when its frame is due. Images were already +/// drawn at the window size and stay on the thread that owns the Skia context. +#[expect( + clippy::large_enum_variant, + reason = "The window holds one current frame. Keeping its tree inline avoids a per-frame allocation." +)] +pub(crate) enum PlaybackFrame { + Tree(usvgr::Tree), + Image(Image), +} pub(crate) struct Scheduler { state: Mutex, @@ -16,6 +28,7 @@ pub(crate) struct Scheduler { duration: u64, lookahead: u64, thumbnails: usize, + playback_decoders: VideoDecodersWorker, } struct State { @@ -69,8 +82,9 @@ impl Scheduler { }), workers_wakeup: Condvar::new(), duration: duration_in_frames.max(1) as u64, - lookahead: lookahead.max(1) as u64, + lookahead: lookahead.min(duration_in_frames) as u64, thumbnails, + playback_decoders: VideoDecodersWorker::default(), } } @@ -102,23 +116,43 @@ impl Scheduler { self.workers_wakeup.notify_all(); } + /// A forward seek within the finished buffer can show its target immediately. + pub(crate) fn seek_buffered( + &self, + position: u64, + looping: bool, + images: &mut BTreeMap, + ) -> bool { + let mut state = self.lock(); + if state.looping != looping || position < state.playhead || !images.contains_key(&position) + { + return false; + } + state.playhead = position; + state.ready.retain(|&p, _| p >= position); + drop(state); + images.retain(|&p, _| p >= position); + self.workers_wakeup.notify_all(); + true + } + /// Moves the playhead and returns the newest ready frame at or before it. /// Frames that are late are still returned so heavy videos degrade to a lower /// frame rate instead of freezing. - pub(crate) fn take(&self, playhead: u64) -> Option<(u64, usvgr::Tree)> { + pub(crate) fn take( + &self, + playhead: u64, + images: &mut BTreeMap, + ) -> Option<(u64, PlaybackFrame)> { + let image = take_due(images, playhead).map(|(p, image)| (p, PlaybackFrame::Image(image))); let mut state = self.lock(); let advanced = state.playhead != playhead; let first_frame = state.displayed.is_none(); state.playhead = playhead; - let mut taken = None; - while state - .ready - .first_key_value() - .is_some_and(|(&position, _)| position <= playhead) - { - taken = state.ready.pop_first(); - } + let tree = + take_due(&mut state.ready, playhead).map(|(p, tree)| (p, PlaybackFrame::Tree(tree))); + let taken = tree.into_iter().chain(image).max_by_key(|(p, _)| *p); if let Some((position, _)) = &taken { state.displayed = Some(*position); } @@ -130,6 +164,23 @@ impl Scheduler { taken } + /// Hands one future tree to the window for offscreen drawing. Leave any due + /// frame for `take`, even if it completed between the last tick and this call. + pub(crate) fn next_to_draw(&self) -> Option<(u64, usvgr::Tree)> { + let mut state = self.lock(); + if state.displayed.is_none() || *state.ready.first_key_value()?.0 <= state.playhead { + return None; + } + let next = state.ready.pop_first(); + drop(state); + self.workers_wakeup.notify_one(); + next + } + + pub(crate) fn put_back(&self, position: u64, tree: usvgr::Tree) { + self.lock().ready.insert(position, tree); + } + pub(crate) fn shutdown(&self) { self.lock().shutdown = true; self.workers_wakeup.notify_all(); @@ -142,25 +193,8 @@ impl Scheduler { return None; } - // Skip frames the playhead already passed: this is where frames get dropped - // when rendering can not keep up with real time. - if state.next < state.playhead { - state.next = state.playhead; - } - - let within_lookahead = state.next < state.playhead + self.lookahead; - let within_video = state.looping || state.next < self.duration; - if within_lookahead && within_video { - let position = state.next; - state.next += 1; - - return Some(Job { - frame: self.frame_of(position, state.looping), - kind: JobKind::Playback { - position, - epoch: state.epoch, - }, - }); + if let Some(job) = self.claim_playback(&mut state) { + return Some(job); } state = self @@ -170,6 +204,26 @@ impl Scheduler { } } + fn claim_playback(&self, state: &mut State) -> Option { + let position = state.next.max(state.playhead); + if position > state.playhead.saturating_add(self.lookahead) + || (!state.looping && position >= self.duration) + || (position > state.playhead && state.ready.len() >= 4) + { + return None; + } + // Keep only a few decoded trees waiting for the window. A new live position + // can bypass that limit if the clock jumped ahead of all assigned work. + state.next = position + 1; + Some(Job { + frame: self.frame_of(position, state.looping), + kind: JobKind::Playback { + position, + epoch: state.epoch, + }, + }) + } + // The filmstrip seeks across the video with its own decoder. fn next_thumbnail_job(&self) -> Option { let mut state = self.lock(); @@ -223,8 +277,10 @@ impl Scheduler { // Text caches are `Rc` based and therefore owned by the worker. let text_cache = TextCache::new(100); let mut converter_cache = usvgr::Cache::new_with_text_cache(100); - // Queued trees own their pixels. The decoder only needs its last conversion. - let video_decoders = VideoDecodersWorker::default(); + let video_decoders = match kind { + WorkerKind::Playback => self.playback_decoders.clone(), + WorkerKind::Thumbnails => VideoDecodersWorker::default(), + }; while let Some(job) = match kind { WorkerKind::Playback => self.next_job(), @@ -270,6 +326,14 @@ impl Scheduler { } } +fn take_due(ready: &mut BTreeMap, playhead: u64) -> Option<(u64, T)> { + let mut latest = None; + while ready.first_key_value().is_some_and(|(&p, _)| p <= playhead) { + latest = ready.pop_first(); + } + latest +} + #[cfg(test)] mod tests { use super::*; @@ -292,25 +356,81 @@ mod tests { for index in [2, 0, 1] { assert!(scheduler.complete(index, 0, tree())); } - assert_eq!(scheduler.take(1).unwrap().0, 1); - assert!(scheduler.take(1).is_none()); + assert_eq!(scheduler.take(1, &mut BTreeMap::new()).unwrap().0, 1); + assert!(scheduler.take(1, &mut BTreeMap::new()).is_none()); assert!(!scheduler.complete(0, 0, tree())); - assert_eq!(scheduler.take(2).unwrap().0, 2); + assert_eq!(scheduler.take(2, &mut BTreeMap::new()).unwrap().0, 2); scheduler.seek(50, false); assert!(!scheduler.complete(3, 0, tree())); assert_eq!(scheduler.next_job().unwrap().frame, 50); assert!(scheduler.complete(50, 1, tree())); - assert_eq!(scheduler.take(50).unwrap().0, 50); + assert_eq!(scheduler.take(50, &mut BTreeMap::new()).unwrap().0, 50); scheduler.shutdown(); assert!(scheduler.next_job().is_none()); } + #[test] + fn finished_frames_fill_the_window_and_a_seek_reuses_its_target() { + let scheduler = Scheduler::new(100, 48, false, 0); + let mut images = BTreeMap::new(); + let image = fframes_skia_renderer::skia_safe::surfaces::raster_n32_premul((8, 8)) + .unwrap() + .image_snapshot(); + assert_eq!(scheduler.next_job().unwrap().frame, 0); + scheduler.complete(0, 0, tree()); + assert!(scheduler.next_to_draw().is_none()); + assert!(matches!( + scheduler.take(0, &mut images), + Some((0, PlaybackFrame::Tree(_))) + )); + + for p in 1..=48 { + assert_eq!(scheduler.next_job().unwrap().frame as u64, p); + scheduler.complete(p, 0, tree()); + assert_eq!(scheduler.next_to_draw().unwrap().0, p); + images.insert(p, image.clone()); + } + assert!(scheduler.claim_playback(&mut scheduler.lock()).is_none()); + assert_eq!(images.len(), 48); + assert!(scheduler.seek_buffered(40, false, &mut images)); + assert!(matches!( + scheduler.take(40, &mut images), + Some((40, PlaybackFrame::Image(_))) + )); + assert_eq!(images.len(), 8); + assert!(!scheduler.seek_buffered(41, true, &mut images)); + assert!(!scheduler.seek_buffered(0, false, &mut images)); + assert_eq!(scheduler.next_job().unwrap().frame, 49); + } + + #[test] + fn live_frames_take_priority_over_queued_trees() { + let scheduler = Scheduler::new(100, 48, false, 0); + let mut images = BTreeMap::new(); + for p in 0..4 { + assert_eq!(scheduler.next_job().unwrap().frame as u64, p); + scheduler.complete(p, 0, tree()); + } + assert!(scheduler.claim_playback(&mut scheduler.lock()).is_none()); + assert!(matches!( + scheduler.take(30, &mut images), + Some((3, PlaybackFrame::Tree(_))) + )); + assert_eq!(scheduler.next_job().unwrap().frame, 30); + scheduler.complete(30, 0, tree()); + assert!(scheduler.next_to_draw().is_none()); + assert!(matches!( + scheduler.take(30, &mut images), + Some((30, PlaybackFrame::Tree(_))) + )); + } + #[test] fn thumbnails_do_not_consume_playback_jobs_and_stop_when_finished() { let scheduler = Scheduler::new(100, 3, false, 2); assert_eq!(scheduler.next_job().unwrap().frame, 0); scheduler.complete(0, 0, tree()); - scheduler.take(0).unwrap(); + scheduler.take(0, &mut BTreeMap::new()).unwrap(); assert_eq!(scheduler.next_thumbnail_job().unwrap().frame, 25); assert_eq!(scheduler.next_job().unwrap().frame, 1); assert_eq!(scheduler.next_thumbnail_job().unwrap().frame, 75); diff --git a/fframes/src/video_data.rs b/fframes/src/video_data.rs index 3fdf5553..5a4895f2 100644 --- a/fframes/src/video_data.rs +++ b/fframes/src/video_data.rs @@ -6,9 +6,12 @@ use std::sync::Arc; use crate::error::Result; #[cfg(not(target_arch = "wasm32"))] use fframes_media::ImageMetadata; -pub use fframes_media::{FrameConvertOptions, PlaybackSpeed, ResizeVideoFrame}; +pub use fframes_media::{FrameConvertOptions, ResizeVideoFrame}; #[cfg(not(target_arch = "wasm32"))] -use std::collections::HashMap; +use std::{ + collections::HashMap, + sync::{Mutex, RwLock}, +}; #[cfg(target_arch = "wasm32")] use std::cell::Cell; @@ -64,7 +67,7 @@ pub trait FFramesSyncedVideoFrame<'media> { fn stream_duration_in_seconds(&self) -> f32; } -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, Default)] pub struct SyncVideoFrameInput<'media> { /// The seconds timestamp in seconds to start playing the video from /// If not provided starts playing from the beginning of the video or scene. @@ -73,30 +76,11 @@ pub struct SyncVideoFrameInput<'media> { /// Also if `true` the `get_synced_video_frame` API should never return None, only in case of /// decoding errors, so in some cases it might be okay to panic if the frame is not available. pub looping: bool, - /// Playback speed of the video: `2.0` plays it twice as fast, `0.5` at half speed. - /// Every frame shows the source frame at `seconds since start_from * speed`, so the - /// motion stays even at any speed and frame rate. With `looping` a loop takes - /// `duration / speed` seconds. The speed is kept to 1/1000 (`PlaybackSpeed`); a value - /// that is not positive and finite plays at normal speed. Default `1.0`. - /// The speed is constant: each speed of a file has its own decoder, and the - /// renderer keeps the 4 used last. - pub speed: f32, /// The wasm preview editor is not supporting decoding video frames in runtime /// so this allows to provide a fallback image for the debugging pub editor_fallback_image: Option<&'media ImageData<'media>>, } -impl Default for SyncVideoFrameInput<'_> { - fn default() -> Self { - Self { - start_from: 0.0, - looping: false, - speed: 1.0, - editor_fallback_image: None, - } - } -} - /// This is used by the editor as a fallback for the static image #[derive(Debug, Clone)] pub struct WasmEditorVideoFrameFallback<'media> { @@ -196,26 +180,21 @@ impl<'media> FFramesSyncedVideoFrame<'media> for crate::media::FFmpegFrameBuf { } } +#[cfg(not(target_arch = "wasm32"))] +type DecoderMap = RwLock>>>; + #[derive(Clone)] #[allow(dead_code)] pub struct VideoDecodersWorker { buffer_size: usize, #[cfg(not(target_arch = "wasm32"))] - map: Arc>>>, + map: Arc, } -/// A decoder of a file at one playback speed. -#[cfg(not(target_arch = "wasm32"))] -type SpeedDecoder = (PlaybackSpeed, fframes_media::FFmpegDecoder); - -/// Each decoder keeps the file open and its frames in memory. A file played at more -/// speeds closes the decoder that was used least recently. -#[cfg(not(target_arch = "wasm32"))] -const MAX_DECODERS_PER_VIDEO: usize = 4; - impl std::fmt::Debug for VideoDecodersWorker { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("WorkerLocalDecoders").finish() + f.debug_struct("VideoDecodersWorker") + .finish_non_exhaustive() } } @@ -230,7 +209,7 @@ impl VideoDecodersWorker { VideoDecodersWorker { buffer_size, #[cfg(not(target_arch = "wasm32"))] - map: Arc::new(std::sync::RwLock::new(HashMap::new())), + map: Arc::new(RwLock::new(HashMap::new())), } } @@ -248,11 +227,6 @@ impl VideoDecodersWorker { return Ok(None); }; - // The editor decodes on the grid of the video's frames. - let speed = i64::from(PlaybackSpeed::new(options.speed).thousandths()); - let scale = i64::from(PlaybackSpeed::SCALE); - let offset = (offset * speed + scale - 1) / scale; - let last_frame_to_display_image = metadata.duration * ctx.time_base.fps as f32; if offset > last_frame_to_display_image as i64 && !options.looping { return Ok(None); @@ -306,20 +280,17 @@ impl VideoDecodersWorker { pts: i64, filename: &str, ) -> Result>> { - let frame = self.map.read().unwrap().get(filename).and_then(|decoders| { - decoders - .iter() - .find_map(|(_, decoder)| decoder.get_decoded_image_in_buf(pts)) - }); - - Ok(frame) + let Some(decoder) = self.map.read()?.get(filename).cloned() else { + return Ok(None); + }; + Ok(decoder.lock()?.get_decoded_image_in_buf(pts)) } #[cfg(not(target_arch = "wasm32"))] pub(crate) fn get_synced_frame<'media>( &self, VideoMedia { path, .. }: &VideoMedia, - offset: i64, + mut offset: i64, ctx: &FFramesContext<'_, 'media>, options: &SyncVideoFrameInput, ) -> Result + 'media>>> { @@ -331,86 +302,50 @@ impl VideoDecodersWorker { .ok_or_else(|| FFramesError::MediaError(FFramesMediaError::MediaDirectoryProvided))? .to_string_lossy(); - let speed = PlaybackSpeed::new(options.speed); - - // It is important to lock it for the whole duration of he function to - // avoid any parallel encoder creation. - let mut map_guard = self.map.write()?; - // Every speed gets its own decoder, offsets of different speeds point to - // different parts of the file. - let decoders = map_guard.entry(resource_name.into_owned()).or_default(); - let decoder = decoder_for_speed(decoders, speed, || unsafe { - fframes_media::FFmpegDecoder::new(path, ctx.time_base.fps, self.buffer_size) - .map(|decoder| decoder.with_playback_speed(speed)) - })?; + // Each source has one mutable decode cursor. Different sources can decode + // concurrently, and callers retain their frame after releasing this lock. + let existing = self.map.read()?.get(resource_name.as_ref()).cloned(); + let decoder = if let Some(decoder) = existing { + decoder + } else { + use std::collections::hash_map::Entry; + match self.map.write()?.entry(resource_name.into_owned()) { + Entry::Occupied(entry) => Arc::clone(entry.get()), + Entry::Vacant(entry) => { + let mut decoder = unsafe { + fframes_media::FFmpegDecoder::new( + path, + ctx.time_base.fps, + self.buffer_size, + )? + }; + if offset > 0 { + unsafe { decoder.seek_to_offset(offset)? }; + } + Arc::clone(entry.insert(Arc::new(Mutex::new(decoder)))) + } + } + }; + let mut decoder = decoder.lock()?; unsafe { - let has_frame = if options.looping { - decoder.decode_looping(offset)? - } else { - decoder.decode_up_to(offset)? - }; - Ok(has_frame.then(|| decoder.get_raw_frame())) - } - } -} - -/// The decoder of `speed`, made with `create` if there is none. The most recently used -/// decoder is first, and at most `MAX_DECODERS_PER_VIDEO` are kept. -#[cfg(not(target_arch = "wasm32"))] -fn decoder_for_speed( - decoders: &mut Vec<(PlaybackSpeed, D)>, - speed: PlaybackSpeed, - create: impl FnOnce() -> std::result::Result, -) -> std::result::Result<&mut D, E> { - if let Some(index) = decoders.iter().position(|(s, _)| *s == speed) { - decoders[..=index].rotate_right(1); - } else { - decoders.insert(0, (speed, create()?)); - decoders.truncate(MAX_DECODERS_PER_VIDEO); - } - Ok(&mut decoders[0].1) -} - -#[cfg(all(test, not(target_arch = "wasm32")))] -mod decoder_cache_tests { - use super::*; - - fn use_speed(decoders: &mut Vec<(PlaybackSpeed, f32)>, speed: f32, created: &mut usize) { - decoder_for_speed(decoders, PlaybackSpeed::new(speed), || { - *created += 1; - Ok::<_, ()>(speed) - }) - .unwrap(); - } - - #[test] - fn speeds_that_round_to_the_same_value_share_a_decoder() { - let (mut decoders, mut created) = (Vec::new(), 0); - for i in 0..100 { - use_speed(&mut decoders, 1.0 + i as f32 * 0.000_001, &mut created); - } - assert_eq!((decoders.len(), created), (1, 1)); - } + if options.looping { + offset = decoder.adjust_offset_for_looping(offset)?; + } - #[test] - fn least_recently_used_decoders_are_closed() { - let (mut decoders, mut created) = (Vec::new(), 0); - for i in 0..100 { - use_speed(&mut decoders, 1.0 + i as f32 * 0.1, &mut created); + let has_frame = decoder.decode_up_to(offset)?; + if has_frame { + Ok(Some(decoder.retained_frame()?)) + } else if options.looping { + decoder.seek_to_offset(0)?; + let has_first_frame = decoder.decode_up_to(0)?; + + decoder.current_loop += 1; + Ok(has_first_frame + .then(|| decoder.retained_frame()) + .transpose()?) + } else { + Ok(None) + } } - assert_eq!((decoders.len(), created), (MAX_DECODERS_PER_VIDEO, 100)); - - // A used decoder moves to the front and is kept. - use_speed(&mut decoders, 1.0 + 97.0 * 0.1, &mut created); - use_speed(&mut decoders, 0.25, &mut created); - use_speed(&mut decoders, 0.5, &mut created); - use_speed(&mut decoders, 1.0 + 97.0 * 0.1, &mut created); - assert_eq!(created, 102); - let speeds: Vec<_> = decoders.iter().map(|(s, _)| s.thousandths()).collect(); - assert_eq!(speeds, [10_700, 500, 250, 10_900]); } } - -// These implementations are already thread-safe due to RwLock -unsafe impl Send for VideoDecodersWorker {} -unsafe impl Sync for VideoDecodersWorker {} From 3747317c9d44ceb20cc4ae2c0290d74d83f23ee2 Mon Sep 17 00:00:00 2001 From: dmtrKovalenko Date: Wed, 28 Oct 2026 21:29:52 +0000 Subject: [PATCH 07/21] perf(player): Reduce preview memory and bound video workers --- examples/arm/src/main.rs | 4 +- fframes-native-player/src/app.rs | 12 +- fframes-native-player/src/lib.rs | 3 + fframes-native-player/src/options.rs | 21 ++- fframes-native-player/src/presenter/mod.rs | 168 ++++++++++++--------- fframes-native-player/src/scheduler.rs | 98 ++++++++---- fframes/src/renderer/cli.rs | 82 ++++++---- fframes/src/renderer/preview.rs | 32 ++++ fframes/src/renderer/renderer.rs | 41 +++-- 9 files changed, 304 insertions(+), 157 deletions(-) diff --git a/examples/arm/src/main.rs b/examples/arm/src/main.rs index 30fb08d9..49ac5fa1 100644 --- a/examples/arm/src/main.rs +++ b/examples/arm/src/main.rs @@ -59,12 +59,12 @@ fn main() -> ExitCode { ..Default::default() }; - let cpu = args.app.cpu; + let needs_export_gpu = !args.app.cpu && !matches!(args.command, Some(cli::Command::Preview(_))); let command = cli::new(&video, options) .args(args) .preview(fframes_native_player::cli_preview) .default_output("out.mp4"); - if cpu { + if !needs_export_gpu { return command.run(); } diff --git a/fframes-native-player/src/app.rs b/fframes-native-player/src/app.rs index 3d26946f..980430b4 100644 --- a/fframes-native-player/src/app.rs +++ b/fframes-native-player/src/app.rs @@ -13,14 +13,14 @@ use winit::window::{CursorIcon, Fullscreen, Window, WindowId}; use crate::controls::{Control, ControlsState, format_time}; use crate::error::PlayerError; use crate::options::PlayerBackend; -use crate::presenter::Presenter; +use crate::presenter::{BufferedFrame, Presenter}; use crate::scheduler::{PlaybackFrame, Scheduler}; use crate::studio::{Studio, StudioLayout, StudioScroll}; -use fframes_skia_renderer::skia_safe::{Image, Point}; +use fframes_skia_renderer::skia_safe::Point; /// The control bar hides after this long without pointer movement while playing. const CONTROLS_IDLE_TIMEOUT: Duration = Duration::from_millis(2500); -/// Estimated RGBA storage for the current and buffered window-size images. +/// Conservative RGBA budget using the whole window as an upper bound per image. const BUFFER_BYTES: u64 = 512 << 20; /// A frame worker finished a playback or filmstrip job. @@ -92,7 +92,7 @@ pub(crate) struct App<'s> { window: Option>, // Drop texture-backed images before the presenter and its Skia context. current: Option<(u64, PlaybackFrame)>, - buffered: BTreeMap, + buffered: BTreeMap, presenter: Option, redraw_pending: bool, draw_time: Duration, @@ -842,7 +842,7 @@ mod tests { #[test] fn resuming_keeps_prefetched_frames_unless_playback_restarts() { - let scheduler = Scheduler::new(100, 3, false, 0); + let scheduler = Scheduler::new(100, 3, false, 0, false); let mut app = paused_app(&scheduler); app.seek_to(10, false); app.toggle_play(); @@ -856,7 +856,7 @@ mod tests { #[test] fn failed_scrub_continues_at_the_latest_pointer_position() { - let scheduler = Scheduler::new(100, 3, false, 0); + let scheduler = Scheduler::new(100, 3, false, 0, false); let mut app = paused_app(&scheduler); app.seek_to(10, false); app.scrub_target = Some(20); diff --git a/fframes-native-player/src/lib.rs b/fframes-native-player/src/lib.rs index 8a61845a..2fefad41 100644 --- a/fframes-native-player/src/lib.rs +++ b/fframes-native-player/src/lib.rs @@ -142,6 +142,9 @@ pub fn play<'a, 'media: 'a, TVideo: Video + Sync>( options.lookahead(TVideo::FPS), options.looping, thumbnail_count, + options + .media + .is_some_and(fframes::MediaProvider::has_video_sources), ); let proxy = Mutex::new(event_loop.create_proxy()); let notify_frame_ready = |event| { diff --git a/fframes-native-player/src/options.rs b/fframes-native-player/src/options.rs index f62e8dd5..bb02c775 100644 --- a/fframes-native-player/src/options.rs +++ b/fframes-native-player/src/options.rs @@ -41,8 +41,9 @@ pub struct PlayerOptions<'a, 'media> { pub start_frame: usize, /// Play the video's audio map on the default output device. pub audio: bool, - /// Threads building frame trees (`Video::render_frame`) in parallel. + /// Maximum number of threads building frame trees (`Video::render_frame`) in parallel. /// Defaults to one worker for video sources so the decoder reads frames in order. + /// Video sources allow up to four staged trees and one worker for a newly due frame. /// Other projects use half the available cores, up to eight workers. pub render_threads: Option, /// Maximum number of future frames to prepare. The default (`usize::MAX`) uses @@ -85,13 +86,15 @@ impl PlayerOptions<'_, '_> { } pub(crate) fn render_threads(&self) -> usize { - if let Some(count) = self.render_threads { - return count.max(1); - } if self.media.is_some_and(MediaProvider::has_video_sources) { - return 1; + return self + .render_threads + .unwrap_or(1) + .clamp(1, crate::scheduler::MAX_STAGED_TREES + 1); } - (fframes::get_thread_count() / 2).clamp(1, 8) + self.render_threads + .unwrap_or_else(|| (fframes::get_thread_count() / 2).clamp(1, 8)) + .max(1) } } @@ -127,10 +130,14 @@ mod tests { assert_eq!(options.render_threads(), 1); options.render_threads = Some(4); assert_eq!(options.render_threads(), 4); + options.render_threads = Some(16); + assert_eq!(options.render_threads(), 5); options.render_threads = Some(0); assert_eq!(options.render_threads(), 1); - options.render_threads = None; options.media = None; + options.render_threads = Some(16); + assert_eq!(options.render_threads(), 16); + options.render_threads = None; assert_eq!( options.render_threads(), (fframes::get_thread_count() / 2).clamp(1, 8) diff --git a/fframes-native-player/src/presenter/mod.rs b/fframes-native-player/src/presenter/mod.rs index 14df02cc..c68473d1 100644 --- a/fframes-native-player/src/presenter/mod.rs +++ b/fframes-native-player/src/presenter/mod.rs @@ -12,7 +12,9 @@ use std::rc::Rc; use fframes::usvgr; use fframes_skia_renderer::render::RenderCache; -use fframes_skia_renderer::skia_safe::{Canvas, Color, Image, ImageInfo, Rect, gpu, surfaces}; +use fframes_skia_renderer::skia_safe::{ + Canvas, Color, IPoint, IRect, Image, ImageInfo, Point, Rect, RoundOut, Surface, gpu, surfaces, +}; use fframes_skia_renderer::{QueueLock, lock_queue}; use winit::window::Window; @@ -22,6 +24,12 @@ use crate::options::PlayerBackend; use crate::scheduler::PlaybackFrame; use crate::studio::{Studio, StudioLayout, StudioScroll}; +/// Only the video occupies this image. Its origin is in window pixels. +pub(crate) struct BufferedFrame { + pub(crate) origin: IPoint, + pub(crate) image: Image, +} + /// The images of the window that Skia draws into. trait WindowTarget { fn is_gpu(&self) -> bool; @@ -50,6 +58,7 @@ trait WindowTarget { pub(crate) struct Presenter { pub studio_layout: Option, render_cache: RenderCache, + snapshot_surface: Option, target: Box, window: Rc, background: Color, @@ -83,6 +92,7 @@ impl Presenter { let mut presenter = Self { render_cache: target.render_cache(), + snapshot_surface: None, target, window, background: Color::from_argb(background.a, background.r, background.g, background.b), @@ -115,6 +125,7 @@ impl Presenter { return Ok(()); } + self.snapshot_surface = None; self.target.resize(size.width, size.height)?; let logical = size.to_logical::(self.window.scale_factor()); self.studio_layout = StudioLayout::new( @@ -135,40 +146,59 @@ impl Presenter { } /// Draws just the scene. Controls and the timeline are composited when it is shown. - pub(crate) fn snapshot(&mut self, tree: &usvgr::Tree, studio_open: bool) -> Option { - let size = self.window.inner_size(); - let info = ImageInfo::new_n32_premul((size.width as i32, size.height as i32), None); + pub(crate) fn snapshot( + &mut self, + tree: &usvgr::Tree, + studio_open: bool, + ) -> Option { + let view = self.studio_layout.as_ref().filter(|_| studio_open); let mut context = self.target.gpu_context(); - let mut surface = match context.as_mut() { - Some((gpu, queue)) => { - let _lock = lock_queue(*queue); - gpu::surfaces::render_target( - gpu, - gpu::Budgeted::Yes, - &info, - None, - gpu::SurfaceOrigin::TopLeft, - None, - false, - None, - ) - } - None => surfaces::raster(&info, None, None), - }?; - draw_scene( - surface.canvas(), - Some(tree), - self.studio_layout.as_ref().filter(|_| studio_open), + // Keep window coordinates while rendering filters and shaders. Only the + // video rectangle is retained in each finished frame. + if self.snapshot_surface.is_none() { + let size = self.window.inner_size(); + let info = ImageInfo::new_n32_premul((size.width as i32, size.height as i32), None); + self.snapshot_surface = match context.as_mut() { + Some((gpu, queue)) => { + let _lock = lock_queue(*queue); + gpu::surfaces::render_target( + gpu, + gpu::Budgeted::Yes, + &info, + None, + gpu::SurfaceOrigin::TopLeft, + None, + false, + None, + ) + } + None => surfaces::raster(&info, None, None), + }; + } + let surface = self.snapshot_surface.as_mut()?; + let canvas = surface.canvas(); + canvas.clear(matte(view)); + let rect = draw_video( + canvas, + tree, + view, self.window.scale_factor() as f32, self.background, &mut self.render_cache, ); - let image = surface.image_snapshot(); + let bounds = IRect::intersect( + &rect.round_out(), + &IRect::from_size(canvas.base_layer_size()), + )?; + let image = surface.image_snapshot_with_bounds(bounds)?; if let Some((gpu, queue)) = context { let _lock = lock_queue(queue); - gpu.flush_and_submit_surface(&mut surface, gpu::SyncCpu::Yes); + gpu.flush_and_submit_surface(surface, gpu::SyncCpu::Yes); } - Some(image) + Some(BufferedFrame { + origin: IPoint::new(bounds.left, bounds.top), + image, + }) } /// Draws the current frame and overlays the controls before presenting the window. @@ -193,19 +223,15 @@ impl Presenter { target.present(&mut |canvas| { let view = studio_layout.as_ref().filter(|_| studio_open); + canvas.clear(matte(view)); match frame { - Some(PlaybackFrame::Image(image)) => { - canvas.draw_image(image, (0, 0), None); + Some(PlaybackFrame::Image(frame)) => { + canvas.draw_image(&frame.image, Point::from(frame.origin), None); } - Some(PlaybackFrame::Tree(tree)) => draw_scene( - canvas, - Some(tree), - view, - pixel_scale, - *background, - render_cache, - ), - None => draw_scene(canvas, None, view, pixel_scale, *background, render_cache), + Some(PlaybackFrame::Tree(tree)) => { + draw_video(canvas, tree, view, pixel_scale, *background, render_cache); + } + None => {} } let show_bar = view.is_none(); @@ -230,18 +256,25 @@ impl Presenter { } } -fn draw_scene( +fn matte(view: Option<&StudioLayout>) -> Color { + if view.is_some() { + Color::from_rgb(14, 14, 14) + } else { + Color::BLACK + } +} + +fn draw_video( canvas: &Canvas, - tree: Option<&usvgr::Tree>, + tree: &usvgr::Tree, view: Option<&StudioLayout>, pixel_scale: f32, background: Color, - render_cache: &mut RenderCache, -) { + cache: &mut RenderCache, +) -> Rect { let size = canvas.base_layer_size(); - let (width, height) = (size.width as f32, size.height as f32); let viewport = view.map_or_else( - || Rect::from_wh(width, height), + || Rect::from_iwh(size.width, size.height), |ui| { let stage = ui.stage; Rect::from_ltrb( @@ -252,34 +285,29 @@ fn draw_scene( ) }, ); - canvas.clear(if view.is_some() { - Color::from_rgb(14, 14, 14) + let video = tree.size(); + let scale = (viewport.width() / video.width()).min(viewport.height() / video.height()); + let width = video.width() * scale; + let height = video.height() * scale; + let x = viewport.center_x() - width / 2.; + let y = if view.is_some_and(|ui| ui.portrait) { + viewport.top } else { - Color::BLACK - }); + viewport.center_y() - height / 2. + }; - if let Some(tree) = tree { - let video = tree.size(); - let scale = (viewport.width() / video.width()).min(viewport.height() / video.height()); - let offset_x = viewport.center_x() - video.width() * scale / 2.; - let offset_y = if view.is_some_and(|ui| ui.portrait) { - viewport.top - } else { - viewport.center_y() - video.height() * scale / 2. - }; - - canvas.save(); - canvas.translate((offset_x, offset_y)); - canvas.scale((scale, scale)); - canvas.clip_rect( - Rect::from_wh(video.width(), video.height()), - None, - Some(true), - ); - canvas.draw_color(background, None); - fframes_skia_renderer::render::render_tree(tree, canvas, render_cache); - canvas.restore(); - } + canvas.save(); + canvas.translate((x, y)); + canvas.scale((scale, scale)); + canvas.clip_rect( + Rect::from_wh(video.width(), video.height()), + None, + Some(true), + ); + canvas.draw_color(background, None); + fframes_skia_renderer::render::render_tree(tree, canvas, cache); + canvas.restore(); + Rect::from_xywh(x, y, width, height) } fn create_target( diff --git a/fframes-native-player/src/scheduler.rs b/fframes-native-player/src/scheduler.rs index 0656e0ab..bbf4ce88 100644 --- a/fframes-native-player/src/scheduler.rs +++ b/fframes-native-player/src/scheduler.rs @@ -7,19 +7,20 @@ use std::collections::BTreeMap; use std::sync::{Condvar, Mutex, MutexGuard}; -use crate::{app::FrameReady, studio::thumbnail}; +use crate::{app::FrameReady, presenter::BufferedFrame, studio::thumbnail}; use fframes::{FFramesContext, TextCache, Video, VideoDecodersWorker, usvgr}; -use fframes_skia_renderer::skia_safe::Image; + +pub(crate) const MAX_STAGED_TREES: usize = 4; /// A tree can go straight to the window when its frame is due. Images were already -/// drawn at the window size and stay on the thread that owns the Skia context. +/// drawn at the window scale and stay on the thread that owns the Skia context. #[expect( clippy::large_enum_variant, reason = "The window holds one current frame. Keeping its tree inline avoids a per-frame allocation." )] pub(crate) enum PlaybackFrame { Tree(usvgr::Tree), - Image(Image), + Image(BufferedFrame), } pub(crate) struct Scheduler { @@ -29,6 +30,7 @@ pub(crate) struct Scheduler { lookahead: u64, thumbnails: usize, playback_decoders: VideoDecodersWorker, + video_sources: bool, } struct State { @@ -42,6 +44,7 @@ struct State { displayed: Option, looping: bool, ready: BTreeMap, + in_flight: usize, shutdown: bool, next_thumbnail: usize, } @@ -68,6 +71,7 @@ impl Scheduler { lookahead: usize, looping: bool, thumbnails: usize, + video_sources: bool, ) -> Self { Self { state: Mutex::new(State { @@ -77,6 +81,7 @@ impl Scheduler { displayed: None, looping, ready: BTreeMap::new(), + in_flight: 0, shutdown: false, next_thumbnail: 0, }), @@ -85,6 +90,7 @@ impl Scheduler { lookahead: lookahead.min(duration_in_frames) as u64, thumbnails, playback_decoders: VideoDecodersWorker::default(), + video_sources, } } @@ -121,7 +127,7 @@ impl Scheduler { &self, position: u64, looping: bool, - images: &mut BTreeMap, + images: &mut BTreeMap, ) -> bool { let mut state = self.lock(); if state.looping != looping || position < state.playhead || !images.contains_key(&position) @@ -142,7 +148,7 @@ impl Scheduler { pub(crate) fn take( &self, playhead: u64, - images: &mut BTreeMap, + images: &mut BTreeMap, ) -> Option<(u64, PlaybackFrame)> { let image = take_due(images, playhead).map(|(p, image)| (p, PlaybackFrame::Image(image))); let mut state = self.lock(); @@ -206,15 +212,23 @@ impl Scheduler { fn claim_playback(&self, state: &mut State) -> Option { let position = state.next.max(state.playhead); + // Decoding workers can each hold full-size source frames before they return a tree. + let staged = state.ready.len() + + if self.video_sources { + state.in_flight + } else { + 0 + }; if position > state.playhead.saturating_add(self.lookahead) || (!state.looping && position >= self.duration) - || (position > state.playhead && state.ready.len() >= 4) + || (position > state.playhead && staged >= MAX_STAGED_TREES) { return None; } // Keep only a few decoded trees waiting for the window. A new live position // can bypass that limit if the clock jumped ahead of all assigned work. state.next = position + 1; + state.in_flight += 1; Some(Job { frame: self.frame_of(position, state.looping), kind: JobKind::Playback { @@ -251,18 +265,21 @@ impl Scheduler { state.playhead == position && state.epoch == epoch } - fn complete(&self, position: u64, epoch: u64, tree: usvgr::Tree) -> bool { + fn complete(&self, position: u64, epoch: u64, tree: Option) -> bool { let mut state = self.lock(); + state.in_flight -= 1; let is_stale = state.epoch != epoch || state .displayed .is_some_and(|displayed| position <= displayed); - if !is_stale { + if !is_stale && let Some(tree) = tree { state.ready.insert(position, tree); + return true; } - - !is_stale + drop(state); + self.workers_wakeup.notify_one(); + false } pub(crate) fn run_worker<'a, 'media: 'a, TVideo: Video + Sync>( @@ -308,7 +325,7 @@ impl Scheduler { } } (Ok(tree), JobKind::Playback { position, epoch }) => { - if self.complete(position, epoch, tree) { + if self.complete(position, epoch, Some(tree)) { on_frame_ready(FrameReady::Playback); } } @@ -318,6 +335,7 @@ impl Scheduler { job.frame ); if let JobKind::Playback { position, epoch } = kind { + self.complete(position, epoch, None); on_frame_ready(FrameReady::Failed { position, epoch }); } } @@ -349,21 +367,21 @@ mod tests { #[test] fn taking_out_of_order_frames_preserves_future_frames_and_rejects_stale_jobs() { - let scheduler = Scheduler::new(100, 3, false, 2); - for index in 0..3 { + let scheduler = Scheduler::new(100, 3, false, 2, false); + for index in 0..=3 { assert_eq!(scheduler.next_job().unwrap().frame, index); } - for index in [2, 0, 1] { - assert!(scheduler.complete(index, 0, tree())); + for index in [2, 1] { + assert!(scheduler.complete(index, 0, Some(tree()))); } assert_eq!(scheduler.take(1, &mut BTreeMap::new()).unwrap().0, 1); assert!(scheduler.take(1, &mut BTreeMap::new()).is_none()); - assert!(!scheduler.complete(0, 0, tree())); + assert!(!scheduler.complete(0, 0, Some(tree()))); assert_eq!(scheduler.take(2, &mut BTreeMap::new()).unwrap().0, 2); scheduler.seek(50, false); - assert!(!scheduler.complete(3, 0, tree())); + assert!(!scheduler.complete(3, 0, Some(tree()))); assert_eq!(scheduler.next_job().unwrap().frame, 50); - assert!(scheduler.complete(50, 1, tree())); + assert!(scheduler.complete(50, 1, Some(tree()))); assert_eq!(scheduler.take(50, &mut BTreeMap::new()).unwrap().0, 50); scheduler.shutdown(); assert!(scheduler.next_job().is_none()); @@ -371,13 +389,13 @@ mod tests { #[test] fn finished_frames_fill_the_window_and_a_seek_reuses_its_target() { - let scheduler = Scheduler::new(100, 48, false, 0); + let scheduler = Scheduler::new(100, 48, false, 0, false); let mut images = BTreeMap::new(); let image = fframes_skia_renderer::skia_safe::surfaces::raster_n32_premul((8, 8)) .unwrap() .image_snapshot(); assert_eq!(scheduler.next_job().unwrap().frame, 0); - scheduler.complete(0, 0, tree()); + scheduler.complete(0, 0, Some(tree())); assert!(scheduler.next_to_draw().is_none()); assert!(matches!( scheduler.take(0, &mut images), @@ -386,9 +404,15 @@ mod tests { for p in 1..=48 { assert_eq!(scheduler.next_job().unwrap().frame as u64, p); - scheduler.complete(p, 0, tree()); + scheduler.complete(p, 0, Some(tree())); assert_eq!(scheduler.next_to_draw().unwrap().0, p); - images.insert(p, image.clone()); + images.insert( + p, + BufferedFrame { + image: image.clone(), + origin: fframes_skia_renderer::skia_safe::IPoint::default(), + }, + ); } assert!(scheduler.claim_playback(&mut scheduler.lock()).is_none()); assert_eq!(images.len(), 48); @@ -405,11 +429,11 @@ mod tests { #[test] fn live_frames_take_priority_over_queued_trees() { - let scheduler = Scheduler::new(100, 48, false, 0); + let scheduler = Scheduler::new(100, 48, false, 0, false); let mut images = BTreeMap::new(); for p in 0..4 { assert_eq!(scheduler.next_job().unwrap().frame as u64, p); - scheduler.complete(p, 0, tree()); + scheduler.complete(p, 0, Some(tree())); } assert!(scheduler.claim_playback(&mut scheduler.lock()).is_none()); assert!(matches!( @@ -417,7 +441,7 @@ mod tests { Some((3, PlaybackFrame::Tree(_))) )); assert_eq!(scheduler.next_job().unwrap().frame, 30); - scheduler.complete(30, 0, tree()); + scheduler.complete(30, 0, Some(tree())); assert!(scheduler.next_to_draw().is_none()); assert!(matches!( scheduler.take(30, &mut images), @@ -425,18 +449,34 @@ mod tests { )); } + #[test] + fn video_workers_bound_in_flight_frames_and_release_failed_jobs() { + let scheduler = Scheduler::new(100, 48, false, 0, true); + for p in 0..4 { + assert_eq!(scheduler.next_job().unwrap().frame, p); + } + assert!(scheduler.claim_playback(&mut scheduler.lock()).is_none()); + scheduler.complete(2, 0, None); + assert_eq!(scheduler.next_job().unwrap().frame, 4); + scheduler.take(30, &mut BTreeMap::new()); + assert_eq!(scheduler.next_job().unwrap().frame, 30); + scheduler.seek(60, false); + assert!(!scheduler.complete(1, 0, Some(tree()))); + assert_eq!(scheduler.next_job().unwrap().frame, 60); + } + #[test] fn thumbnails_do_not_consume_playback_jobs_and_stop_when_finished() { - let scheduler = Scheduler::new(100, 3, false, 2); + let scheduler = Scheduler::new(100, 3, false, 2, false); assert_eq!(scheduler.next_job().unwrap().frame, 0); - scheduler.complete(0, 0, tree()); + scheduler.complete(0, 0, Some(tree())); scheduler.take(0, &mut BTreeMap::new()).unwrap(); assert_eq!(scheduler.next_thumbnail_job().unwrap().frame, 25); assert_eq!(scheduler.next_job().unwrap().frame, 1); assert_eq!(scheduler.next_thumbnail_job().unwrap().frame, 75); assert!(scheduler.next_thumbnail_job().is_none()); assert_eq!(scheduler.next_job().unwrap().frame, 2); - let stopped = Scheduler::new(100, 3, false, 2); + let stopped = Scheduler::new(100, 3, false, 2, false); stopped.shutdown(); assert!(stopped.next_thumbnail_job().is_none()); } diff --git a/fframes/src/renderer/cli.rs b/fframes/src/renderer/cli.rs index 29b64a5e..9d74b284 100644 --- a/fframes/src/renderer/cli.rs +++ b/fframes/src/renderer/cli.rs @@ -494,12 +494,15 @@ impl<'r, 'a, 'media: 'a, V: Video + Send + Sync, B: FFramesRenderBackend, A: Arg /// Parses the command line (unless `args` was given) and runs the command. pub fn run(mut self) -> ExitCode { - let renderer = self - .args - .get_or_insert_with(Cli::::parse) + let args = self.args.get_or_insert_with(Cli::::parse); + if matches!(args.command, Some(Command::Preview(_))) { + // The player opens its own window backend. + return self.execute(); + } + let renderer = args .renderer .take() - .or(self.renderers.default_renderer()); + .or_else(|| self.renderers.default_renderer()); let runner = Runner { video: self.video, options: self.options, @@ -550,6 +553,10 @@ impl<'r, 'a, 'media: 'a, V: Video + Send + Sync, B: FFramesRenderBackend, A: Arg args.output = args.output.or(Some(default_output)); render(json, cli.scale, video, options, args, backend) } + Command::Preview(args) => { + drop(backend); + preview(video, &options, player, args) + } command => { let mut frame_renderer = backend.frame_renderer(); let mut fallback; @@ -575,29 +582,7 @@ impl<'r, 'a, 'media: 'a, V: Video + Send + Sync, B: FFramesRenderBackend, A: Arg inspect(json, &mut previewer, frame_renderer, args) } Command::Audio(args) => audio(json, &previewer, args), - Command::Preview(args) => { - let Some(play) = player else { - return Err("this binary has no real-time player: add the \ - `fframes_native_player` crate and \ - `.preview(fframes_native_player::cli_preview)`. Without a \ - window use `strip`, `frame` or `render --draft`." - .to_owned()); - }; - let request = super::PreviewRequest { - start_frame: previewer - .timeline() - .resolve_frame(&args.at) - .map_err(err)?, - autoplay: !args.paused, - looping: !args.no_loop, - audio: !args.mute, - backend: args.backend, - }; - // Fonts and caches of the previewer are not needed by the player. - drop(previewer); - play(video, &options, &request).map(|()| ExitCode::SUCCESS) - } - Command::Render(_) => unreachable!(), + Command::Preview(_) | Command::Render(_) => unreachable!(), }) } }; @@ -630,6 +615,49 @@ fn print(json: bool, value: &T, text: impl FnOnce() -> String) { } } +fn preview<'a, 'media: 'a, V: Video>( + video: &'a V, + options: &RenderOptions<'a, 'media>, + player: Option>, + args: PreviewArgs, +) -> Result { + let Some(play) = player else { + return Err("this binary has no real-time player: add the \ + `fframes_native_player` crate and \ + `.preview(fframes_native_player::cli_preview)`. Without a \ + window use `strip`, `frame` or `render --draft`." + .to_owned()); + }; + let start_frame = { + let time_base = crate::TimeBase { + fps: V::FPS, + sample_rate: options.audio_encoder_options.sample_rate, + }; + let timeline = crate::FFramesRendererRuntime::resolve_video_timeline( + time_base, + video, + &video.define_scenes(), + options.media, + ) + .map_err(err)?; + crate::TimelineIndex::new( + V::FPS, + timeline.duration_in_frames, + timeline.scenes.as_ref(), + ) + .resolve_frame(&args.at) + .map_err(err)? + }; + let request = super::PreviewRequest { + start_frame, + autoplay: !args.paused, + looping: !args.no_loop, + audio: !args.mute, + backend: args.backend, + }; + play(video, options, &request).map(|()| ExitCode::SUCCESS) +} + fn scenes_label(scenes: &[String]) -> String { if scenes.is_empty() { String::new() diff --git a/fframes/src/renderer/preview.rs b/fframes/src/renderer/preview.rs index 37fc45c9..a97ff9c0 100644 --- a/fframes/src/renderer/preview.rs +++ b/fframes/src/renderer/preview.rs @@ -975,6 +975,38 @@ mod batch_tests { } } + #[cfg(feature = "cli")] + #[test] + fn native_preview_resolves_the_start_without_an_export_backend() { + use crate::cli::{self, clap::Parser}; + + let video = Changing { panic_at: None }; + let args = cli::Cli::::parse_from([ + "video", + "--renderer", + "gpu", + "preview", + "1s", + "--backend", + "cpu", + "--paused", + "--mute", + ]); + let mut opened = false; + let result = cli::new(&video, RenderOptions::default()) + .args(args) + .preview(|_, _, request| { + assert_eq!(request.start_frame, 10); + assert_eq!(request.backend, "cpu"); + assert!(!request.autoplay && !request.audio); + opened = true; + Ok(()) + }) + .run(); + assert_eq!(result, std::process::ExitCode::SUCCESS); + assert!(opened); + } + /// Out of order, with a repeated frame. const FRAMES: [usize; 9] = [23, 5, 0, 12, 12, 19, 20, 3, 8]; diff --git a/fframes/src/renderer/renderer.rs b/fframes/src/renderer/renderer.rs index 1219bf30..4bca6216 100644 --- a/fframes/src/renderer/renderer.rs +++ b/fframes/src/renderer/renderer.rs @@ -105,7 +105,30 @@ impl<'a, 'media: 'a> FFramesRendererRuntime<'a> { scenes: &Scenes<'a>, media: Option<&'media dyn MediaProvider<'media>>, ) -> FFramesRendererResult { - let timeline = crate::resolve_timeline( + let timeline = Self::resolve_video_timeline(time_base, video, scenes, media)?; + + let mut font_source = RendererFontSource { + fontdb: fontdb::Database::new(), + }; + + if let Some(media) = media { + media.populate_font_source(&mut font_source); + } + + Ok(Self { + time_base, + timeline, + font_source, + }) + } + + pub(super) fn resolve_video_timeline( + time_base: TimeBase, + video: &'a TVideo, + scenes: &Scenes<'a>, + media: Option<&'media dyn MediaProvider<'media>>, + ) -> FFramesRendererResult> { + Ok(crate::resolve_timeline( &video.duration(), &ScenesWithAudio::new(scenes), &time_base, @@ -137,21 +160,7 @@ impl<'a, 'media: 'a> FFramesRendererRuntime<'a> { crate::error::FFramesError::CanNotProcessAudioDuration(name.to_owned()) }) }, - )?; - - let mut font_source = RendererFontSource { - fontdb: fontdb::Database::new(), - }; - - if let Some(media) = media { - media.populate_font_source(&mut font_source); - } - - Ok(Self { - time_base, - timeline, - font_source, - }) + )?) } } From 098c57f369803feac6141371e3790dc6442a6e8f Mon Sep 17 00:00:00 2001 From: dmtrKovalenko Date: Wed, 28 Oct 2026 22:36:21 +0000 Subject: [PATCH 08/21] fix(media): Release hardware devices when decoders close --- fframes-media/src/video_decoder.rs | 255 ++++++++++++----------------- 1 file changed, 105 insertions(+), 150 deletions(-) diff --git a/fframes-media/src/video_decoder.rs b/fframes-media/src/video_decoder.rs index d0ae8e16..2f8059d5 100644 --- a/fframes-media/src/video_decoder.rs +++ b/fframes-media/src/video_decoder.rs @@ -1,6 +1,6 @@ use crate::FFramesMediaError; use crate::error::Result; -use crate::video_types::{FrameConvertOptions, PlaybackSpeed}; +use crate::video_types::FrameConvertOptions; use ffmpeg_sys_fframes::SwsFlags::{SWS_BICUBIC, SWS_BILINEAR}; use ffmpeg_sys_fframes::*; use std::borrow::Cow; @@ -96,7 +96,6 @@ struct VideoStreamInfo { height: i32, display_width: i32, display_height: i32, - pixel_format: AVPixelFormat, color_space: AVColorSpace, color_range: AVColorRange, time_base: AVRational, @@ -379,11 +378,10 @@ pub struct FFmpegDecoder { video_stream_info: VideoStreamInfo, codec_ctx: *mut AVCodecContext, pkt: *mut AVPacket, - /// Offsets are counted in this time base: `speed / target_fps` seconds per offset. custom_time_base: AVRational, - target_fps: i32, - /// The stream position of the previous decode or seek, in the stream's time base. - last_pts: Option, + duration_in_frames: i64, + /// The offset of the previous `decode_up_to` call, in `custom_time_base` units. + last_offset: Option, /// A null packet has been sent; receive delayed frames until decoder EOF. draining: bool, /// The target frame holds a frame decoded since the last seek. @@ -590,35 +588,6 @@ impl FFmpegDecoder { .map(|slot| Arc::clone(&slot.image)) } - /// Plays the stream `speed` times faster: offset `n` shows the frame at - /// `n * speed / target_fps` seconds of the stream. - pub fn with_playback_speed(mut self, speed: PlaybackSpeed) -> Self { - self.custom_time_base = AVRational { - num: speed.thousandths(), - den: self.target_fps * PlaybackSpeed::SCALE, - }; - self.last_pts = None; - self.current_loop = 0; - self - } - - /// The stream position of the offset, in the stream's time base. - fn offset_pts(&self, offset: i64) -> i64 { - unsafe { - av_rescale_q( - offset, - self.custom_time_base, - self.video_stream_info.time_base, - ) - } - } - - /// The exact stream duration, in the stream's time base. `None` if it is unknown. - fn stream_duration(&self) -> Option { - let duration = self.video_stream_info.duration; - (duration > 0).then_some(duration) - } - /// Creates a new decoder for the video file at the specified path /// # Safety /// Generally safe but uses libav functions @@ -658,59 +627,60 @@ impl FFmpegDecoder { ))); } - let (codec_ctx, video_stream_info) = Self::open_codec_context(fmt_ctx)?; - let pkt = av_packet_alloc(); - if pkt.is_null() { - avformat_close_input(&raw mut fmt_ctx); - return Err(FFramesMediaError::LibAVAllocationError("packet")); - } - - let hw_frame = if video_stream_info.hw_pix_fmt.is_some() { - let hw_frame = av_frame_alloc(); - if hw_frame.is_null() { - avformat_close_input(&raw mut fmt_ctx); - return Err(FFramesMediaError::LibAVAllocationError("av_frame")); - } - - (*hw_frame).width = video_stream_info.width; - (*hw_frame).height = video_stream_info.height; - (*hw_frame).format = video_stream_info.pixel_format as i32; - - hw_frame - } else { - ptr::null_mut() - }; - - let recv_frame = av_frame_alloc(); - if recv_frame.is_null() { + let (mut codec_ctx, video_stream_info) = Self::open_codec_context(fmt_ctx) + .inspect_err(|_| avformat_close_input(&raw mut fmt_ctx))?; + let frame_buf = FFmpegFrameBuf::new( + filename.to_string_lossy().into_owned(), + video_stream_info, + buffer_size, + ) + .inspect_err(|_| { + avcodec_free_context(&raw mut codec_ctx); avformat_close_input(&raw mut fmt_ctx); - return Err(FFramesMediaError::LibAVAllocationError("av_frame")); - } + })?; let custom_time_base = AVRational { num: 1, den: target_fps as i32, }; - Ok(FFmpegDecoder { + let duration_in_frames = av_rescale_q( + video_stream_info.duration, + video_stream_info.time_base, + custom_time_base, + ); + + let mut decoder = Self { codec_ctx, - pkt, fmt_ctx, - hw_frame, - recv_frame, - frame_buf: Arc::new(FFmpegFrameBuf::new( - filename.to_string_lossy().to_string(), - video_stream_info, - buffer_size, - )?), + pkt: ptr::null_mut(), + hw_frame: ptr::null_mut(), + recv_frame: ptr::null_mut(), + frame_buf: Arc::new(frame_buf), video_stream_info, custom_time_base, - target_fps: target_fps as i32, + duration_in_frames, current_loop: 0, - last_pts: None, + last_offset: None, draining: false, has_decoded_frame: false, - }) + }; + + decoder.pkt = av_packet_alloc(); + if decoder.pkt.is_null() { + return Err(FFramesMediaError::LibAVAllocationError("packet")); + } + decoder.recv_frame = av_frame_alloc(); + if decoder.recv_frame.is_null() { + return Err(FFramesMediaError::LibAVAllocationError("av_frame")); + } + if video_stream_info.hw_pix_fmt.is_some() { + decoder.hw_frame = av_frame_alloc(); + if decoder.hw_frame.is_null() { + return Err(FFramesMediaError::LibAVAllocationError("av_frame")); + } + } + Ok(decoder) } } @@ -737,35 +707,37 @@ impl FFmpegDecoder { let stream = *(*fmt_ctx).streams.offset(video_stream_idx as isize); let dec = avcodec_find_decoder((*(*stream).codecpar).codec_id); - let mut hw_device_ctx: *mut AVBufferRef = ptr::null_mut(); - let hw_pix_fmt = find_hw_accelleleration_for_codec(&mut hw_device_ctx, dec); - if dec.is_null() { return Err(FFramesMediaError::LibAVAudioDecodingError(( - ret, + AVERROR_DECODER_NOT_FOUND, "Could not find decoder".to_string(), ))); } - let video_dec_ctx = avcodec_alloc_context3(dec); + let mut video_dec_ctx = avcodec_alloc_context3(dec); if video_dec_ctx.is_null() { - return Err(FFramesMediaError::LibAVAllocationError("avcocdec_context")); - } - - if let Some(hw_pix_fmt) = hw_pix_fmt { - (*video_dec_ctx).opaque = hw_pix_fmt as i32 as *mut c_void; - (*video_dec_ctx).get_format = Some(get_hw_format); - (*video_dec_ctx).hw_device_ctx = av_buffer_ref(hw_device_ctx); + return Err(FFramesMediaError::LibAVAllocationError("avcodec_context")); } - if avcodec_parameters_to_context(video_dec_ctx, (*stream).codecpar) < 0 { + let ret = avcodec_parameters_to_context(video_dec_ctx, (*stream).codecpar); + if ret < 0 { + avcodec_free_context(&raw mut video_dec_ctx); return Err(FFramesMediaError::LibAVAudioDecodingError(( ret, "Failed to copy video codec parameters to decoder context".to_string(), ))); } - if avcodec_open2(video_dec_ctx, dec, ptr::null_mut()) < 0 { + let hw_pix_fmt = + find_hw_acceleration_for_codec(&mut (*video_dec_ctx).hw_device_ctx, dec); + if let Some(hw_pix_fmt) = hw_pix_fmt { + (*video_dec_ctx).opaque = hw_pix_fmt as i32 as *mut c_void; + (*video_dec_ctx).get_format = Some(get_hw_format); + } + + let ret = avcodec_open2(video_dec_ctx, dec, ptr::null_mut()); + if ret < 0 { + avcodec_free_context(&raw mut video_dec_ctx); return Err(FFramesMediaError::LibAVAudioDecodingError(( ret, "Failed to open video codec".to_string(), @@ -803,7 +775,6 @@ impl FFmpegDecoder { height, display_width, display_height, - pixel_format: (*video_dec_ctx).pix_fmt, color_space: (*video_dec_ctx).colorspace, color_range: (*video_dec_ctx).color_range, time_base: (*stream).time_base, @@ -818,11 +789,13 @@ impl FFmpegDecoder { /// # Safety /// Generally safe but uses libav functions pub unsafe fn seek_to_offset(&mut self, offset: i64) -> Result<()> { - unsafe { self.seek_to_pts(self.offset_pts(offset)) } - } - - unsafe fn seek_to_pts(&mut self, timestamp: i64) -> Result<()> { unsafe { + let timestamp = av_rescale_q( + offset, + self.custom_time_base, + self.video_stream_info.time_base, + ); + let ret = av_seek_frame( self.fmt_ctx, self.video_stream_info.stream_index, @@ -841,42 +814,33 @@ impl FFmpegDecoder { avcodec_flush_buffers(self.codec_ctx); self.draining = false; self.has_decoded_frame = false; - self.last_pts = Some(timestamp); av_packet_unref(self.pkt); Ok(()) } } - /// Decodes the frame at the offset of the video played in a loop. The position in - /// the loop comes from the exact stream duration, so loops at a fractional speed - /// do not drift and do not end early. + /// Moves offset to the start of the video if need to loop /// # Safety /// Generally safe but uses libav functions - pub unsafe fn decode_looping(&mut self, offset: i64) -> Result { - unsafe { - let pts = self.offset_pts(offset); - // An unknown duration can't be looped over (and would divide by zero below). - let Some(duration) = self.stream_duration() else { - return self.decode_up_to_pts(pts); - }; - - let loop_index = pts.div_euclid(duration); - let pts = pts.rem_euclid(duration); - if loop_index != self.current_loop { - self.current_loop = loop_index; - self.seek_to_pts(pts)?; - } + pub unsafe fn adjust_offset_for_looping(&mut self, offset: i64) -> Result { + // An unknown or empty duration can't be looped over (and would divide by zero below). + if self.duration_in_frames <= 0 || offset < self.duration_in_frames { + return Ok(offset); + } - if self.decode_up_to_pts(pts)? { - return Ok(true); - } + let new_loop_index = offset / self.duration_in_frames; + let new_offset = offset % self.duration_in_frames; + if new_loop_index == self.current_loop { + return Ok(new_offset); + } - // The frames end before the stream's duration: start the next loop. - self.current_loop += 1; - self.seek_to_pts(0)?; - self.decode_up_to_pts(0) + self.current_loop = new_loop_index; + unsafe { + self.seek_to_offset(new_offset)?; } + + Ok(new_offset) } /// If we are actually using hardware accelerated frames it is required to correctly @@ -934,25 +898,25 @@ impl FFmpegDecoder { /// # Safety /// Generally safe but uses libav functions pub unsafe fn decode_up_to(&mut self, offset: i64) -> Result { - unsafe { self.decode_up_to_pts(self.offset_pts(offset)) } - } - - unsafe fn decode_up_to_pts(&mut self, target_pts: i64) -> Result { unsafe { // Going back needs a seek, otherwise the newer frame that was already decoded - // would be returned. Far jumps forward (more than 2 seconds of the stream) seek - // to the closest keyframe instead of decoding every frame in between (frames can - // be requested out of order by parallel renderers). A new decoder is at the start. - let seek_ahead = av_rescale_q( - 2, - AVRational { num: 1, den: 1 }, + // would be returned. Far jumps forward seek to the closest keyframe instead of + // decoding every frame in between (frames can be requested out of order by + // parallel renderers). + let seek_ahead_frames = 2 * i64::from(self.custom_time_base.den.max(1)); + let needs_seek = self.last_offset.is_some_and(|last_offset| { + offset < last_offset || offset - last_offset > seek_ahead_frames + }); + self.last_offset = Some(offset); + if needs_seek { + self.seek_to_offset(offset)?; + } + + let target_pts = av_rescale_q( + offset, + self.custom_time_base, self.video_stream_info.time_base, ); - let last_pts = self.last_pts.unwrap_or(0); - if target_pts < last_pts || target_pts - last_pts > seek_ahead { - self.seek_to_pts(target_pts)?; - } - self.last_pts = Some(target_pts); if (*self.frame_buf.latest_av_frame).pts >= target_pts { return Ok(true); @@ -983,10 +947,8 @@ impl FFmpegDecoder { // The target lies after the last frame's timestamp but before the end of // the stream (e.g. a 24 fps clip sampled at 30 fps): the last frame is // still the one on screen, so return it instead of reporting the end. - let shows_last_frame = self.has_decoded_frame - && self - .stream_duration() - .is_some_and(|duration| target_pts < duration); + let shows_last_frame = + self.has_decoded_frame && offset < self.duration_in_frames; if shows_last_frame { self.transfer_hardware_surface_data(target_frame)?; } @@ -1049,7 +1011,7 @@ impl Drop for FFmpegDecoder { } } -unsafe fn find_hw_accelleleration_for_codec( +unsafe fn find_hw_acceleration_for_codec( hw_device_ctx: &mut *mut AVBufferRef, codec: *const AVCodec, ) -> Option { @@ -1074,18 +1036,10 @@ unsafe fn find_hw_accelleleration_for_codec( ]; for &hw_type in &hw_types { - let create_result = - av_hwdevice_ctx_create(hw_device_ctx, hw_type, ptr::null(), ptr::null_mut(), 0); - - if create_result < 0 { - continue; - } - let mut hw_pix_fmt = AVPixelFormat::AV_PIX_FMT_NONE; let mut i = 0; loop { - // Check if decoder supports this hardware if yes we return the available pix fmt let config = avcodec_get_hw_config(codec, i); if config.is_null() { break; @@ -1101,11 +1055,13 @@ unsafe fn find_hw_accelleleration_for_codec( i += 1; } - if hw_pix_fmt != AVPixelFormat::AV_PIX_FMT_NONE { + // The codec context owns the reference returned on success. + if hw_pix_fmt != AVPixelFormat::AV_PIX_FMT_NONE + && av_hwdevice_ctx_create(hw_device_ctx, hw_type, ptr::null(), ptr::null_mut(), 0) + >= 0 + { return Some(hw_pix_fmt); } - - av_buffer_unref(hw_device_ctx); } } @@ -1252,7 +1208,6 @@ mod color_tests { height: HEIGHT, display_width: WIDTH, display_height: HEIGHT, - pixel_format: AVPixelFormat::AV_PIX_FMT_YUV444P, color_space: space, color_range: range, time_base: AVRational { num: 1, den: 30 }, From 71505d80ce8932b1d589f64d0b174a2a5d2b47c7 Mon Sep 17 00:00:00 2001 From: dmtrKovalenko Date: Thu, 29 Oct 2026 11:03:59 -0700 Subject: [PATCH 09/21] perf(player): Release idle decoders and map VideoToolbox frames in preview --- README.md | 6 + fframes-media/Cargo.toml | 4 +- fframes-media/src/video_decoder.rs | 81 +++++++++-- fframes-media/tests/video_decoder_eof.rs | 140 +++++++++--------- fframes-native-player/Cargo.toml | 1 + fframes-native-player/src/app.rs | 46 +++++- fframes-native-player/src/lib.rs | 2 + fframes-native-player/src/options.rs | 6 +- fframes-native-player/src/scheduler.rs | 26 ++-- fframes/Cargo.toml | 1 + fframes/build.rs | 8 +- fframes/src/renderer/cli.rs | 4 + fframes/src/renderer/preview.rs | 1 + fframes/src/video_data.rs | 176 +++++++++++++++++------ 14 files changed, 366 insertions(+), 136 deletions(-) diff --git a/README.md b/README.md index 112a88ff..c2d97408 100644 --- a/README.md +++ b/README.md @@ -344,6 +344,12 @@ Drag the filmstrip to scrub, or click a scene marker to pause on its first frame to jump between scenes and `,` and `.` to step one frame. Press `Space` to play or pause, or `T` to hide the timeline and expand the video. +On macOS the studio tries VideoToolbox for supported source videos and falls back to software +for unsupported formats. `preview --software-decode` uses the CPU for decoding while keeping +the selected drawing backend. Playback and filmstrip workers release idle source decoders. +The automatic lookahead covers up to two seconds. The CPU draws directly into the window. +The GPU budgets 96 MiB for finished frames when playing source videos or expensive scenes. + `cargo fframes new` takes `--template single-scene|multi-scene`, `--format landscape|portrait|square|uhd`, `--fps`, `--title` and `--backend`. On macOS and Linux (arm64, x86_64) the first build downloads prebuilt Skia and ffmpeg libraries and takes under a minute on a fast machine; other targets and feature combinations diff --git a/fframes-media/Cargo.toml b/fframes-media/Cargo.toml index 4f4b69cf..b9705edf 100644 --- a/fframes-media/Cargo.toml +++ b/fframes-media/Cargo.toml @@ -62,7 +62,9 @@ mp3lame = ["ffmpeg-sys-fframes/build-lib-mp3lame"] opus = ["ffmpeg-sys-fframes/build-lib-opus"] vpx = ["ffmpeg-sys-fframes/build-lib-vpx"] # enable various hardware accelerations -videotoolbox = ["ffmpeg-sys-fframes/build-videotoolbox"] +# The player opts in on macOS without changing decoding policy for exports. +preview-videotoolbox = ["ffmpeg-sys-fframes/build-videotoolbox"] +videotoolbox = ["preview-videotoolbox"] audiotoolbox = ["ffmpeg-sys-fframes/build-audiotoolbox"] vaapi = ["ffmpeg-sys-fframes/build-vaapi"] nvidia = ["ffmpeg-sys-fframes/build-nvidia"] diff --git a/fframes-media/src/video_decoder.rs b/fframes-media/src/video_decoder.rs index 2f8059d5..d276b420 100644 --- a/fframes-media/src/video_decoder.rs +++ b/fframes-media/src/video_decoder.rs @@ -366,10 +366,23 @@ impl SwsScaler { } } +/// Selects the decode path without changing encoder or presentation settings. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum DecodeMode { + /// Hardware accelerators explicitly enabled by this project. + #[default] + Configured, + /// Also try the platform accelerator included with the native studio. + Preview, + /// Use the codec's software implementation. + Software, +} + #[derive(Debug)] pub struct FFmpegDecoder { pub current_loop: i64, hw_frame: *mut AVFrame, + map_videotoolbox: bool, /// Frames are received here and moved into the target frame, so the newest decoded frame /// survives `avcodec_receive_frame` returning EOF (which unrefs the frame it is given). recv_frame: *mut AVFrame, @@ -556,6 +569,8 @@ impl FFmpegFrameBuf { } impl FFmpegDecoder { + /// Live view of this decoder's cursor. Use [`Self::retained_frame`] for a frame + /// that will outlive the next decode or seek. pub fn get_raw_frame(&self) -> Arc { Arc::clone(&self.frame_buf) } @@ -592,6 +607,18 @@ impl FFmpegDecoder { /// # Safety /// Generally safe but uses libav functions pub unsafe fn new(path: &Path, target_fps: usize, buffer_size: usize) -> Result { + unsafe { Self::new_with_mode(path, target_fps, buffer_size, DecodeMode::Configured) } + } + + /// Opens a decoder using the requested hardware policy. + /// # Safety + /// Has the same libav initialization requirements as [`Self::new`]. + pub unsafe fn new_with_mode( + path: &Path, + target_fps: usize, + buffer_size: usize, + mode: DecodeMode, + ) -> Result { unsafe { let filename = path .file_name() @@ -627,7 +654,7 @@ impl FFmpegDecoder { ))); } - let (mut codec_ctx, video_stream_info) = Self::open_codec_context(fmt_ctx) + let (mut codec_ctx, video_stream_info) = Self::open_codec_context(fmt_ctx, mode) .inspect_err(|_| avformat_close_input(&raw mut fmt_ctx))?; let frame_buf = FFmpegFrameBuf::new( filename.to_string_lossy().into_owned(), @@ -655,6 +682,7 @@ impl FFmpegDecoder { fmt_ctx, pkt: ptr::null_mut(), hw_frame: ptr::null_mut(), + map_videotoolbox: cfg!(target_os = "macos") && mode == DecodeMode::Preview, recv_frame: ptr::null_mut(), frame_buf: Arc::new(frame_buf), video_stream_info, @@ -686,6 +714,7 @@ impl FFmpegDecoder { unsafe fn open_codec_context( fmt_ctx: *mut AVFormatContext, + mode: DecodeMode, ) -> Result<(*mut AVCodecContext, VideoStreamInfo)> { unsafe { let ret = av_find_best_stream( @@ -729,7 +758,7 @@ impl FFmpegDecoder { } let hw_pix_fmt = - find_hw_acceleration_for_codec(&mut (*video_dec_ctx).hw_device_ctx, dec); + find_hw_acceleration_for_codec(&mut (*video_dec_ctx).hw_device_ctx, dec, mode); if let Some(hw_pix_fmt) = hw_pix_fmt { (*video_dec_ctx).opaque = hw_pix_fmt as i32 as *mut c_void; (*video_dec_ctx).get_format = Some(get_hw_format); @@ -812,6 +841,7 @@ impl FFmpegDecoder { (*self.frame_buf.latest_av_frame).pts = -1; avcodec_flush_buffers(self.codec_ctx); + self.last_offset = Some(offset); self.draining = false; self.has_decoded_frame = false; av_packet_unref(self.pkt); @@ -845,12 +875,27 @@ impl FFmpegDecoder { /// If we are actually using hardware accelerated frames it is required to correctly /// move the data from the hardware surface to the software frame - unsafe fn transfer_hardware_surface_data(&self, target_frame: *mut AVFrame) -> Result<()> { + unsafe fn transfer_hardware_surface_data(&mut self, target_frame: *mut AVFrame) -> Result<()> { match self.video_stream_info.hw_pix_fmt { Some(hw_pix_fmt) if hw_pix_fmt as i32 == (*target_frame).format => { // A retained frame can still own the last transfer's software planes. let output = self.frame_buf.latest_av_frame; - if !(*output).buf[0].is_null() && av_frame_is_writable(output) == 0 { + if self.map_videotoolbox && hw_pix_fmt == AVPixelFormat::AV_PIX_FMT_VIDEOTOOLBOX { + // FFmpeg holds a read lock on the CVPixelBuffer until the last + // mapped frame reference drops. swscale can read its planes directly. + av_frame_unref(output); + if av_hwframe_map(output, target_frame, AV_HWFRAME_MAP_READ as i32) == 0 { + return Ok(()); + } + av_frame_unref(output); + self.map_videotoolbox = false; + } + // copy_props appends side data and merges dictionaries. Retire those + // along with the pixels if this destination already has frame metadata. + if (!(*output).buf[0].is_null() && av_frame_is_writable(output) == 0) + || (*output).nb_side_data != 0 + || !(*output).metadata.is_null() + { av_frame_unref(output); } if (*self.frame_buf.latest_av_frame).format @@ -873,7 +918,13 @@ impl FFmpegDecoder { ))); } - av_frame_copy_props(self.frame_buf.latest_av_frame, self.hw_frame); + let ret = av_frame_copy_props(output, target_frame); + if ret < 0 { + return Err(FFramesMediaError::LibAVAudioDecodingError(( + ret, + "Error copying hardware frame properties".to_string(), + ))); + } } // The decoder fell back to software decoding: the frame it returned already // holds the pixels. Keep a reference in the target too, since EOF may need @@ -904,9 +955,11 @@ impl FFmpegDecoder { // decoding every frame in between (frames can be requested out of order by // parallel renderers). let seek_ahead_frames = 2 * i64::from(self.custom_time_base.den.max(1)); - let needs_seek = self.last_offset.is_some_and(|last_offset| { - offset < last_offset || offset - last_offset > seek_ahead_frames - }); + let needs_seek = self + .last_offset + .map_or(offset > seek_ahead_frames, |last_offset| { + offset < last_offset || offset - last_offset > seek_ahead_frames + }); self.last_offset = Some(offset); if needs_seek { self.seek_to_offset(offset)?; @@ -1014,7 +1067,11 @@ impl Drop for FFmpegDecoder { unsafe fn find_hw_acceleration_for_codec( hw_device_ctx: &mut *mut AVBufferRef, codec: *const AVCodec, + mode: DecodeMode, ) -> Option { + if mode == DecodeMode::Software { + return None; + } unsafe { let hw_types = [ #[cfg(feature = "vaapi")] @@ -1023,7 +1080,7 @@ unsafe fn find_hw_acceleration_for_codec( AVHWDeviceType::AV_HWDEVICE_TYPE_D3D11VA, #[cfg(feature = "direct3d9")] AVHWDeviceType::AV_HWDEVICE_TYPE_DXVA2, - #[cfg(feature = "videotoolbox")] + #[cfg(any(feature = "preview-videotoolbox", feature = "videotoolbox"))] AVHWDeviceType::AV_HWDEVICE_TYPE_VIDEOTOOLBOX, #[cfg(feature = "nvidia")] AVHWDeviceType::AV_HWDEVICE_TYPE_CUDA, @@ -1036,6 +1093,12 @@ unsafe fn find_hw_acceleration_for_codec( ]; for &hw_type in &hw_types { + #[cfg(all(feature = "preview-videotoolbox", not(feature = "videotoolbox")))] + if hw_type == AVHWDeviceType::AV_HWDEVICE_TYPE_VIDEOTOOLBOX + && mode != DecodeMode::Preview + { + continue; + } let mut hw_pix_fmt = AVPixelFormat::AV_PIX_FMT_NONE; let mut i = 0; diff --git a/fframes-media/tests/video_decoder_eof.rs b/fframes-media/tests/video_decoder_eof.rs index c70f321e..9bd1a61d 100644 --- a/fframes-media/tests/video_decoder_eof.rs +++ b/fframes-media/tests/video_decoder_eof.rs @@ -6,7 +6,7 @@ use std::process::Command; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; -use fframes_media::{FFmpegDecoder, FrameConvertOptions, PlaybackSpeed, ResizeVideoFrame}; +use fframes_media::{FFmpegDecoder, FrameConvertOptions, ResizeVideoFrame}; const FPS: usize = 30; const FRAME_COUNT: i64 = 300; @@ -149,8 +149,9 @@ fn b_frames_are_drained_and_decoding_can_restart_after_eof() { assert_frame(&mut decoder, FRAME_COUNT - 1); // Looping also seeks after decoding the delayed tail. - assert!(unsafe { decoder.decode_looping(FRAME_COUNT) }.unwrap()); - assert_frame(&mut decoder, 0); + let offset = unsafe { decoder.adjust_offset_for_looping(FRAME_COUNT) }.unwrap(); + assert_eq!(offset, 0); + assert_frame(&mut decoder, offset); assert_frame(&mut decoder, FRAME_COUNT - 1); } @@ -317,7 +318,8 @@ fn last_frame_is_shown_until_the_stream_ends_at_a_higher_fps() { for _ in 0..2 { assert!(!unsafe { decoder.decode_up_to(60) }.unwrap()); } - assert!(unsafe { decoder.decode_looping(60) }.unwrap()); + let offset = unsafe { decoder.adjust_offset_for_looping(60) }.unwrap(); + assert_eq!(offset, 0); assert_frame(&mut decoder, 0); // A seek straight to the tail must not return a frame left over from before the seek. @@ -389,70 +391,80 @@ fn frames_can_be_converted_at_widths_that_are_not_a_multiple_of_16() { } } -/// The source frame that offset `n` shows at `num / den` speed: the first frame at or -/// after `n * num / den` (the last frame until the stream ends), in a loop if `looping`. -fn expected_frame(offset: i64, num: i64, den: i64, looping: bool) -> i64 { - let mut position = offset * num; - if looping { - position %= FRAME_COUNT * den; +#[test] +fn preview_decoding_preserves_frames_and_color_across_seeks() { + use fframes_media::DecodeMode; + let fixture = VideoFixture::with_size(3, FPS, 56, (320, 192)); + let file = fixture.path(); + let mut preview = + unsafe { FFmpegDecoder::new_with_mode(&file, FPS, 1, DecodeMode::Preview) }.unwrap(); + let mut software = + unsafe { FFmpegDecoder::new_with_mode(&file, FPS, 1, DecodeMode::Software) }.unwrap(); + for i in [0, 1, 16, 9, 55, 2] { + unsafe { + assert!(preview.decode_up_to(i).unwrap() && software.decode_up_to(i).unwrap()); + let held = preview.retained_frame().unwrap(); + let reference = software.retained_frame().unwrap(); + assert_eq!(held.get_pts(), reference.get_pts()); + // Changing the cursor must not invalidate a mapped VideoToolbox surface. + assert!(preview.decode_up_to((i + 1) % 56).unwrap()); + let a = held + .convert_last_decoded_frame_into_svg_image(None) + .unwrap(); + let b = reference + .convert_last_decoded_frame_into_svg_image(None) + .unwrap(); + assert_eq!((a.width, a.height), (320, 192)); + assert_eq!(a.data.len(), b.data.len()); + // Hardware can reconstruct a channel a little differently than libavcodec. + let error: usize = a + .data + .iter() + .zip(b.data.iter()) + .map(|(x, y)| x.abs_diff(*y) as usize) + .sum(); + assert!( + error < a.data.len() * 2, + "frame {i}: mean channel difference exceeds two" + ); + } } - ((position + den - 1) / den).min(FRAME_COUNT - 1) } -fn assert_shows(decoder: &FFmpegDecoder, speed: f32, offset: i64, frame: i64) { - let seconds = unsafe { decoder.get_raw_frame().timestamp_seconds() }; +#[test] +fn preview_falls_back_for_lossless_alpha_video() { + use fframes_media::DecodeMode; + let fixture = VideoFixture::with_rate(0, FPS, 8); + let file = fixture.path().with_extension("mkv"); + let output = Command::new("ffmpeg") + .args([ + "-v", + "error", + "-y", + "-f", + "lavfi", + "-i", + "testsrc2=size=32x32:rate=30,format=bgra,colorchannelmixer=aa=0.5", + "-frames:v", + "8", + "-c:v", + "ffv1", + ]) + .arg(&file) + .output() + .unwrap(); assert!( - (seconds - frame as f32 / FPS as f32).abs() < 0.0001, - "{speed}x: offset {offset} shows {seconds}s, expected frame {frame}" + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) ); -} - -fn assert_speed(speed: f32, num: i64, den: i64) { - let video = VideoFixture::new(3); - let speed_of = |speed| { - unsafe { FFmpegDecoder::new(&video.path(), FPS, 1) } - .unwrap() - .with_playback_speed(PlaybackSpeed::new(speed)) - }; - - // Every offset before the end of the stream has a frame, also when the stream's - // duration is not a whole number of offsets. - let mut decoder = speed_of(speed); - let offsets = (FRAME_COUNT * den + num - 1) / num; - for offset in 0..offsets { - assert!( - unsafe { decoder.decode_up_to(offset) }.unwrap(), - "{speed}x: offset {offset}" - ); - assert_shows( - &decoder, - speed, - offset, - expected_frame(offset, num, den, false), - ); - } - assert!(!unsafe { decoder.decode_up_to(offsets) }.unwrap()); - - // Loops follow the exact duration and do not drift. - let mut decoder = speed_of(speed); - for offset in 0..3 * offsets { - assert!( - unsafe { decoder.decode_looping(offset) }.unwrap(), - "{speed}x looping: offset {offset}" - ); - assert_shows( - &decoder, - speed, - offset, - expected_frame(offset, num, den, true), - ); + let mut preview = + unsafe { FFmpegDecoder::new_with_mode(&file, FPS, 1, DecodeMode::Preview) }.unwrap(); + let mut software = + unsafe { FFmpegDecoder::new_with_mode(&file, FPS, 1, DecodeMode::Software) }.unwrap(); + for i in [0, 6, 3] { + let pixels = convert_frame(&mut preview, i, None); + assert_eq!(pixels, convert_frame(&mut software, i, None)); + assert!((127..=128).contains(&pixels[3])); } } - -#[test] -fn playback_speed_scales_the_source_time() { - assert_speed(2.0, 2, 1); - assert_speed(1.5, 3, 2); - assert_speed(1.4, 7, 5); - assert_speed(0.5, 1, 2); -} diff --git a/fframes-native-player/Cargo.toml b/fframes-native-player/Cargo.toml index ba65e10b..b8016315 100644 --- a/fframes-native-player/Cargo.toml +++ b/fframes-native-player/Cargo.toml @@ -20,6 +20,7 @@ cpal = { version = "0.18", optional = true } # The CAMetalLayer of the window, presented to by Metal and by Vulkan (MoltenVK). [target.'cfg(target_os = "macos")'.dependencies] +fframes = { workspace = true, features = ["preview-videotoolbox"] } raw-window-metal = { version = "1.1", optional = true } core-graphics-types = { version = "0.2", optional = true, default-features = false } diff --git a/fframes-native-player/src/app.rs b/fframes-native-player/src/app.rs index 980430b4..afe121c5 100644 --- a/fframes-native-player/src/app.rs +++ b/fframes-native-player/src/app.rs @@ -21,7 +21,7 @@ use fframes_skia_renderer::skia_safe::Point; /// The control bar hides after this long without pointer movement while playing. const CONTROLS_IDLE_TIMEOUT: Duration = Duration::from_millis(2500); /// Conservative RGBA budget using the whole window as an upper bound per image. -const BUFFER_BYTES: u64 = 512 << 20; +const BUFFER_BYTES: u64 = 96 << 20; /// A frame worker finished a playback or filmstrip job. pub(crate) enum FrameReady { @@ -83,6 +83,18 @@ pub(crate) struct AppConfig<'a> { pub limit_prefetch_memory: bool, } +impl AppConfig<'_> { + fn should_snapshot(&self, gpu: bool, video_sources: bool, draw_time: Duration) -> bool { + if !self.limit_prefetch_memory { + return true; + } + // Rasterizing ahead competes with presentation on the same CPU. Cheap GPU + // scenes also fit their deadline without retaining extra surfaces. + let frame_interval = Duration::from_secs_f64(1. / self.fps.max(1) as f64); + gpu && (video_sources || draw_time >= frame_interval / 2) + } +} + pub(crate) struct App<'s> { config: AppConfig<'s>, scheduler: &'s Scheduler, @@ -598,6 +610,16 @@ impl<'s> App<'s> { { return false; } + let Some(presenter) = self.presenter.as_mut() else { + return false; + }; + if !self.config.should_snapshot( + presenter.is_gpu(), + self.scheduler.video_sources, + self.draw_time, + ) { + return false; + } if let Some(deadline) = self.clock.next_frame_at() && deadline.saturating_duration_since(Instant::now()) <= self.draw_time + Duration::from_millis(2) @@ -616,9 +638,6 @@ impl<'s> App<'s> { { return false; } - let Some(presenter) = &mut self.presenter else { - return false; - }; let Some((position, tree)) = self.scheduler.next_to_draw() else { return false; }; @@ -840,9 +859,24 @@ mod tests { ) } + #[test] + fn automatic_buffer_reserves_surfaces_for_gpu_work_that_needs_them() { + let scheduler = Scheduler::new(100, 3, false, 0, false, false); + let mut app = paused_app(&scheduler); + let config = &mut app.config; + let fast = Duration::from_millis(1); + let slow = Duration::from_millis(25); + assert!(!config.should_snapshot(false, true, slow)); + assert!(!config.should_snapshot(true, false, fast)); + assert!(config.should_snapshot(true, true, fast)); + assert!(config.should_snapshot(true, false, slow)); + config.limit_prefetch_memory = false; + assert!(config.should_snapshot(false, false, fast)); + } + #[test] fn resuming_keeps_prefetched_frames_unless_playback_restarts() { - let scheduler = Scheduler::new(100, 3, false, 0, false); + let scheduler = Scheduler::new(100, 3, false, 0, false, false); let mut app = paused_app(&scheduler); app.seek_to(10, false); app.toggle_play(); @@ -856,7 +890,7 @@ mod tests { #[test] fn failed_scrub_continues_at_the_latest_pointer_position() { - let scheduler = Scheduler::new(100, 3, false, 0, false); + let scheduler = Scheduler::new(100, 3, false, 0, false, false); let mut app = paused_app(&scheduler); app.seek_to(10, false); app.scrub_target = Some(20); diff --git a/fframes-native-player/src/lib.rs b/fframes-native-player/src/lib.rs index 2fefad41..7f82d941 100644 --- a/fframes-native-player/src/lib.rs +++ b/fframes-native-player/src/lib.rs @@ -145,6 +145,7 @@ pub fn play<'a, 'media: 'a, TVideo: Video + Sync>( options .media .is_some_and(fframes::MediaProvider::has_video_sources), + options.software_decode, ); let proxy = Mutex::new(event_loop.create_proxy()); let notify_frame_ready = |event| { @@ -271,6 +272,7 @@ pub fn cli_preview<'a, 'media: 'a, TVideo: Video + Sync>( looping: request.looping, start_frame: request.start_frame, audio: request.audio, + software_decode: request.software_decode, audio_mix: options.audio_mix, ..Default::default() }, diff --git a/fframes-native-player/src/options.rs b/fframes-native-player/src/options.rs index bb02c775..94a7ec23 100644 --- a/fframes-native-player/src/options.rs +++ b/fframes-native-player/src/options.rs @@ -30,6 +30,8 @@ pub struct PlayerOptions<'a, 'media> { /// to the size of the display. pub window_size: Option<(u32, u32)>, pub backend: PlayerBackend, + /// Decode source videos in software even when a hardware decoder is available. + pub software_decode: bool, /// Wait for the display refresh to show a frame. With `false` a frame is shown as soon /// as it is drawn, up to the frame rate of the video: the picture can tear and the GPU /// stays busy. Only the GPU backends wait for the display. @@ -48,7 +50,8 @@ pub struct PlayerOptions<'a, 'media> { pub render_threads: Option, /// Maximum number of future frames to prepare. The default (`usize::MAX`) uses /// two seconds at the video's frame rate. Zero disables preparation ahead of time. - /// The automatic setting also limits finished frames to an estimated 512 MiB. + /// On automatic, the CPU prepares frame trees and draws directly into the window. + /// GPU pre-rendering for slow scenes or source videos budgets 96 MiB of finished frames. /// An explicit frame count takes precedence over that limit. pub prefetch_frames: usize, /// Master bus of the audio mix, see `RenderOptions::audio_mix`. @@ -64,6 +67,7 @@ impl Default for PlayerOptions<'_, '_> { title: "fframes", window_size: None, backend: PlayerBackend::Auto, + software_decode: false, vsync: true, autoplay: true, looping: true, diff --git a/fframes-native-player/src/scheduler.rs b/fframes-native-player/src/scheduler.rs index bbf4ce88..1095dd10 100644 --- a/fframes-native-player/src/scheduler.rs +++ b/fframes-native-player/src/scheduler.rs @@ -30,7 +30,8 @@ pub(crate) struct Scheduler { lookahead: u64, thumbnails: usize, playback_decoders: VideoDecodersWorker, - video_sources: bool, + pub(crate) video_sources: bool, + software_decode: bool, } struct State { @@ -72,6 +73,7 @@ impl Scheduler { looping: bool, thumbnails: usize, video_sources: bool, + software_decode: bool, ) -> Self { Self { state: Mutex::new(State { @@ -89,8 +91,9 @@ impl Scheduler { duration: duration_in_frames.max(1) as u64, lookahead: lookahead.min(duration_in_frames) as u64, thumbnails, - playback_decoders: VideoDecodersWorker::default(), + playback_decoders: VideoDecodersWorker::for_preview(software_decode), video_sources, + software_decode, } } @@ -296,7 +299,7 @@ impl Scheduler { let mut converter_cache = usvgr::Cache::new_with_text_cache(100); let video_decoders = match kind { WorkerKind::Playback => self.playback_decoders.clone(), - WorkerKind::Thumbnails => VideoDecodersWorker::default(), + WorkerKind::Thumbnails => VideoDecodersWorker::for_preview(self.software_decode), }; while let Some(job) = match kind { @@ -318,6 +321,11 @@ impl Scheduler { Ok(svgr.into_svg_tree(usvg_options, &mut converter_cache, font_db)?) }); + let concurrent_jobs = match kind { + WorkerKind::Playback => self.lock().in_flight, + WorkerKind::Thumbnails => 1, + }; + video_decoders.finish_preview_frame(concurrent_jobs); match (tree, job.kind) { (Ok(tree), JobKind::Thumbnail(slot)) => { if let Some(image) = thumbnail(&tree, TVideo::BACKGROUND_COLOR) { @@ -367,7 +375,7 @@ mod tests { #[test] fn taking_out_of_order_frames_preserves_future_frames_and_rejects_stale_jobs() { - let scheduler = Scheduler::new(100, 3, false, 2, false); + let scheduler = Scheduler::new(100, 3, false, 2, false, false); for index in 0..=3 { assert_eq!(scheduler.next_job().unwrap().frame, index); } @@ -389,7 +397,7 @@ mod tests { #[test] fn finished_frames_fill_the_window_and_a_seek_reuses_its_target() { - let scheduler = Scheduler::new(100, 48, false, 0, false); + let scheduler = Scheduler::new(100, 48, false, 0, false, false); let mut images = BTreeMap::new(); let image = fframes_skia_renderer::skia_safe::surfaces::raster_n32_premul((8, 8)) .unwrap() @@ -429,7 +437,7 @@ mod tests { #[test] fn live_frames_take_priority_over_queued_trees() { - let scheduler = Scheduler::new(100, 48, false, 0, false); + let scheduler = Scheduler::new(100, 48, false, 0, false, false); let mut images = BTreeMap::new(); for p in 0..4 { assert_eq!(scheduler.next_job().unwrap().frame as u64, p); @@ -451,7 +459,7 @@ mod tests { #[test] fn video_workers_bound_in_flight_frames_and_release_failed_jobs() { - let scheduler = Scheduler::new(100, 48, false, 0, true); + let scheduler = Scheduler::new(100, 48, false, 0, true, false); for p in 0..4 { assert_eq!(scheduler.next_job().unwrap().frame, p); } @@ -467,7 +475,7 @@ mod tests { #[test] fn thumbnails_do_not_consume_playback_jobs_and_stop_when_finished() { - let scheduler = Scheduler::new(100, 3, false, 2, false); + let scheduler = Scheduler::new(100, 3, false, 2, false, false); assert_eq!(scheduler.next_job().unwrap().frame, 0); scheduler.complete(0, 0, Some(tree())); scheduler.take(0, &mut BTreeMap::new()).unwrap(); @@ -476,7 +484,7 @@ mod tests { assert_eq!(scheduler.next_thumbnail_job().unwrap().frame, 75); assert!(scheduler.next_thumbnail_job().is_none()); assert_eq!(scheduler.next_job().unwrap().frame, 2); - let stopped = Scheduler::new(100, 3, false, 2, false); + let stopped = Scheduler::new(100, 3, false, 2, false, false); stopped.shutdown(); assert!(stopped.next_thumbnail_job().is_none()); } diff --git a/fframes/Cargo.toml b/fframes/Cargo.toml index d78ed2be..1924687d 100644 --- a/fframes/Cargo.toml +++ b/fframes/Cargo.toml @@ -91,6 +91,7 @@ mp3lame = ["fframes-media/mp3lame"] opus = ["fframes-media/opus"] vpx = ["fframes-media/vpx"] # enable various hardware accelerations +preview-videotoolbox = ["fframes-media/preview-videotoolbox"] videotoolbox = ["fframes-media/videotoolbox"] audiotoolbox = ["fframes-media/audiotoolbox"] vaapi = ["fframes-media/vaapi"] diff --git a/fframes/build.rs b/fframes/build.rs index 1e7cbb5d..3607dccc 100644 --- a/fframes/build.rs +++ b/fframes/build.rs @@ -1,13 +1,9 @@ -use std::env::VarError; - fn main() { let target_os = std::env::var("CARGO_CFG_TARGET_OS").unwrap_or_default(); if (target_os == "macos" || target_os == "ios") - && matches!( - std::env::var("CARGO_FEATURE_VIDEOTOOLBOX"), - Err(VarError::NotPresent) - ) + && std::env::var_os("CARGO_FEATURE_VIDEOTOOLBOX").is_none() + && std::env::var_os("CARGO_FEATURE_PREVIEW_VIDEOTOOLBOX").is_none() { println!( "cargo:warning=\x1B[1m⚠️ WARNING: Apple target arch without videotoolbox enabled ⚠️\x1B[0m" diff --git a/fframes/src/renderer/cli.rs b/fframes/src/renderer/cli.rs index 9d74b284..bb2cdf6e 100644 --- a/fframes/src/renderer/cli.rs +++ b/fframes/src/renderer/cli.rs @@ -131,6 +131,9 @@ pub struct PreviewArgs { /// Do not play the audio. #[arg(long)] pub mute: bool, + /// Decode source video on the CPU even if hardware decoding is available. + #[arg(long)] + pub software_decode: bool, /// Skia backend of the window: auto, metal, vulkan or cpu. #[arg(long, default_value = "auto")] pub backend: String, @@ -653,6 +656,7 @@ fn preview<'a, 'media: 'a, V: Video>( autoplay: !args.paused, looping: !args.no_loop, audio: !args.mute, + software_decode: args.software_decode, backend: args.backend, }; play(video, options, &request).map(|()| ExitCode::SUCCESS) diff --git a/fframes/src/renderer/preview.rs b/fframes/src/renderer/preview.rs index a97ff9c0..c5a91f75 100644 --- a/fframes/src/renderer/preview.rs +++ b/fframes/src/renderer/preview.rs @@ -211,6 +211,7 @@ pub struct PreviewRequest { pub autoplay: bool, pub looping: bool, pub audio: bool, + pub software_decode: bool, /// `auto`, `metal`, `vulkan` or `cpu`. pub backend: String, } diff --git a/fframes/src/video_data.rs b/fframes/src/video_data.rs index 5a4895f2..86dd0d3f 100644 --- a/fframes/src/video_data.rs +++ b/fframes/src/video_data.rs @@ -10,7 +10,8 @@ pub use fframes_media::{FrameConvertOptions, ResizeVideoFrame}; #[cfg(not(target_arch = "wasm32"))] use std::{ collections::HashMap, - sync::{Mutex, RwLock}, + path::{Path, PathBuf}, + sync::Mutex, }; #[cfg(target_arch = "wasm32")] @@ -181,14 +182,56 @@ impl<'media> FFramesSyncedVideoFrame<'media> for crate::media::FFmpegFrameBuf { } #[cfg(not(target_arch = "wasm32"))] -type DecoderMap = RwLock>>>; +type DecoderSlot = Arc>>; + +#[cfg(not(target_arch = "wasm32"))] +#[derive(Default)] +struct DecoderEntry { + last_used: u64, + slot: DecoderSlot, +} + +#[cfg(not(target_arch = "wasm32"))] +#[derive(Default)] +struct DecoderPool { + sources: HashMap, + generation: u64, +} + +#[cfg(not(target_arch = "wasm32"))] +impl DecoderPool { + fn slot(&mut self, path: &Path) -> DecoderSlot { + let entry = if let Some(entry) = self.sources.get_mut(path) { + entry + } else { + self.sources.entry(path.to_owned()).or_default() + }; + entry.last_used = self.generation; + Arc::clone(&entry.slot) + } + + fn finish_frame(&mut self, concurrent_jobs: usize) -> Vec { + // Count completed jobs, not timeline positions. Scrubs, loops and out-of-order + // workers share the same grace period. Keep all layers of an active composition. + self.generation += 1; + self.sources + .extract_if(|_, entry| { + self.generation - entry.last_used > concurrent_jobs.max(1) as u64 + 1 + && Arc::strong_count(&entry.slot) == 1 + }) + .map(|(_, entry)| entry.slot) + .collect() + } +} #[derive(Clone)] #[allow(dead_code)] pub struct VideoDecodersWorker { buffer_size: usize, #[cfg(not(target_arch = "wasm32"))] - map: Arc, + pool: Arc>, + #[cfg(not(target_arch = "wasm32"))] + mode: fframes_media::DecodeMode, } impl std::fmt::Debug for VideoDecodersWorker { @@ -209,10 +252,37 @@ impl VideoDecodersWorker { VideoDecodersWorker { buffer_size, #[cfg(not(target_arch = "wasm32"))] - map: Arc::new(RwLock::new(HashMap::new())), + pool: Arc::default(), + #[cfg(not(target_arch = "wasm32"))] + mode: fframes_media::DecodeMode::Configured, } } + /// The native studio keeps only recently used source decoders between frames. + #[cfg(not(target_arch = "wasm32"))] + pub fn for_preview(software_decode: bool) -> Self { + Self { + mode: if software_decode { + fframes_media::DecodeMode::Software + } else { + fframes_media::DecodeMode::Preview + }, + ..Self::default() + } + } + + /// Retire idle decoder sessions outside the pool lock, including after a seek + /// into a scene that contains no video files. + #[cfg(not(target_arch = "wasm32"))] + pub fn finish_preview_frame(&self, concurrent_jobs: usize) { + let retired = self + .pool + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .finish_frame(concurrent_jobs); + drop(retired); + } + #[cfg(target_arch = "wasm32")] pub(crate) fn get_synced_frame<'media>( &self, @@ -280,10 +350,27 @@ impl VideoDecodersWorker { pts: i64, filename: &str, ) -> Result>> { - let Some(decoder) = self.map.read()?.get(filename).cloned() else { - return Ok(None); + let decoder = { + let pool = self.pool.lock()?; + let path = Path::new(filename); + pool.sources + .get(path) + .or_else(|| { + // Older callers use a bare filename. Only resolve an unambiguous source. + let mut matches = pool + .sources + .iter() + .filter(|(source, _)| source.file_name() == Some(path.as_os_str())); + let (_, entry) = matches.next()?; + matches.next().is_none().then_some(entry) + }) + .map(|entry| Arc::clone(&entry.slot)) }; - Ok(decoder.lock()?.get_decoded_image_in_buf(pts)) + let Some(slot) = decoder else { return Ok(None) }; + Ok(slot + .lock()? + .as_ref() + .and_then(|decoder| decoder.get_decoded_image_in_buf(pts))) } #[cfg(not(target_arch = "wasm32"))] @@ -294,40 +381,20 @@ impl VideoDecodersWorker { ctx: &FFramesContext<'_, 'media>, options: &SyncVideoFrameInput, ) -> Result + 'media>>> { - use crate::error::FFramesError; - use fframes_media::FFramesMediaError; - - let resource_name = path - .file_name() - .ok_or_else(|| FFramesError::MediaError(FFramesMediaError::MediaDirectoryProvided))? - .to_string_lossy(); - - // Each source has one mutable decode cursor. Different sources can decode - // concurrently, and callers retain their frame after releasing this lock. - let existing = self.map.read()?.get(resource_name.as_ref()).cloned(); - let decoder = if let Some(decoder) = existing { - decoder - } else { - use std::collections::hash_map::Entry; - match self.map.write()?.entry(resource_name.into_owned()) { - Entry::Occupied(entry) => Arc::clone(entry.get()), - Entry::Vacant(entry) => { - let mut decoder = unsafe { - fframes_media::FFmpegDecoder::new( - path, - ctx.time_base.fps, - self.buffer_size, - )? - }; - if offset > 0 { - unsafe { decoder.seek_to_offset(offset)? }; - } - Arc::clone(entry.insert(Arc::new(Mutex::new(decoder)))) - } - } - }; - let mut decoder = decoder.lock()?; + // A slot serializes this source's cursor. Opening a file, decoding a frame + // or tearing down a hardware session never holds the shared pool lock. + let slot = self.pool.lock()?.slot(path); + let mut guard = slot.lock()?; unsafe { + let decoder = match guard.as_mut() { + Some(decoder) => decoder, + None => guard.insert(fframes_media::FFmpegDecoder::new_with_mode( + path, + ctx.time_base.fps, + self.buffer_size, + self.mode, + )?), + }; if options.looping { offset = decoder.adjust_offset_for_looping(offset)?; } @@ -349,3 +416,32 @@ impl VideoDecodersWorker { } } } + +#[cfg(all(test, not(target_arch = "wasm32")))] +mod decoder_pool_tests { + use super::*; + + #[test] + fn clips_age_out_without_evicting_active_layers_or_in_flight_work() { + let mut pool = DecoderPool::default(); + let first = Path::new("first/clip.mp4"); + let second = Path::new("second/clip.mp4"); + let held = pool.slot(first); + assert!(!Arc::ptr_eq(&held, &pool.slot(second))); + for _ in 0..24 { + // Overlapping sources in a composite must all stay warm. + for i in 0..12 { + pool.slot(Path::new(&format!("layer-{i}.mp4"))); + } + drop(pool.finish_frame(5)); + } + assert!(!pool.sources.contains_key(second)); + assert!(Arc::ptr_eq(&held, &pool.slot(first))); + assert_eq!(pool.sources.len(), 13); + drop(held); + for _ in 0..9 { + drop(pool.finish_frame(5)); + } + assert!(pool.sources.is_empty()); + } +} From ae9c9afde8d6d2ff59d92e95c4f7b43dcb59f87b Mon Sep 17 00:00:00 2001 From: dmtrKovalenko Date: Thu, 29 Oct 2026 14:01:41 -0700 Subject: [PATCH 10/21] perf(player): Reduce memory use for vector previews on Mac MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Draw small vector scenes into one full-resolution raster surface on unified-memory Macs. Use the GPU for source video, filters, images, and sustained slow frames. Limit text cache size and open the audio device only for projects with audio tracks. --- Hello World at 2560×1440 on one M5 Max, median of three hidden 30-second playbacks. | Metric | Previous Metal preview | Automatic preview | | --- | ---: | ---: | | Physical footprint | 413.0 MiB | 79.4 MiB | | Process RSS | 110.5 MiB | 140.3 MiB | | Process CPU time | 2.75 s | 6.17 s | --- README.md | 4 + fframes-native-player/src/app.rs | 10 +- fframes-native-player/src/lib.rs | 29 ++-- fframes-native-player/src/presenter/metal.rs | 162 ++++++++++++++++++- fframes-native-player/src/presenter/mod.rs | 30 ++-- fframes-native-player/src/scheduler.rs | 6 +- fframes/src/renderer/renderer.rs | 48 +++++- fframes/src/renderer/tests/preview_tests.rs | 41 +++++ 8 files changed, 287 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index c2d97408..fb1e03d5 100644 --- a/README.md +++ b/README.md @@ -347,6 +347,10 @@ to hide the timeline and expand the video. On macOS the studio tries VideoToolbox for supported source videos and falls back to software for unsupported formats. `preview --software-decode` uses the CPU for decoding while keeping the selected drawing backend. Playback and filmstrip workers release idle source decoders. +On unified-memory Macs, automatic playback draws simple vector scenes into a reusable +full-resolution raster surface and presents it through Metal. Source videos, images, effects, +large scenes, and frames that repeatedly exceed their drawing budget use GPU rendering. +Use `preview --backend metal` to render every frame on the GPU. The automatic lookahead covers up to two seconds. The CPU draws directly into the window. The GPU budgets 96 MiB for finished frames when playing source videos or expensive scenes. diff --git a/fframes-native-player/src/app.rs b/fframes-native-player/src/app.rs index afe121c5..6bb37558 100644 --- a/fframes-native-player/src/app.rs +++ b/fframes-native-player/src/app.rs @@ -730,15 +730,7 @@ impl ApplicationHandler for App<'_> { Err(err) => return self.fail(event_loop, err.into()), }; - let duration_label = format_time(self.config.duration_in_frames, self.config.fps); - match Presenter::new( - window.clone(), - self.config.backend, - self.config.vsync, - self.config.background, - duration_label, - &self.config.studio, - ) { + match Presenter::new(window.clone(), &self.config, self.scheduler.video_sources) { Ok(presenter) => self.presenter = Some(presenter), Err(err) => return self.fail(event_loop, err), } diff --git a/fframes-native-player/src/lib.rs b/fframes-native-player/src/lib.rs index 7f82d941..2b6b8488 100644 --- a/fframes-native-player/src/lib.rs +++ b/fframes-native-player/src/lib.rs @@ -65,32 +65,33 @@ pub fn play<'a, 'media: 'a, TVideo: Video + Sync>( let event_loop = EventLoop::::with_user_event().build()?; #[cfg(feature = "audio")] - let audio = options - .audio - .then(|| { - crate::audio::AudioOutput::new() + let mut audio = None; + let audio_sample_rate = || { + #[cfg(feature = "audio")] + if options.audio { + audio = crate::audio::AudioOutput::new() .inspect_err(|err| eprintln!("fframes player: audio disabled ({err})")) - .ok() - }) - .flatten(); - #[cfg(feature = "audio")] - let sample_rate = audio.as_ref().map_or(44100, |audio| audio.sample_rate); - #[cfg(not(feature = "audio"))] - let sample_rate = 44100; - + .ok(); + if let Some(output) = &audio { + return output.sample_rate; + } + } + 44100 + }; let scenes = video.define_scenes(); let FFramesRendererRuntime { time_base, timeline, mut font_source, - } = FFramesRendererRuntime::new( + } = FFramesRendererRuntime::with_audio_sample_rate( TimeBase { fps: TVideo::FPS, - sample_rate, + sample_rate: 44100, }, video, &scenes, options.media, + audio_sample_rate, )?; if options.load_system_fonts { diff --git a/fframes-native-player/src/presenter/metal.rs b/fframes-native-player/src/presenter/metal.rs index 3b9816ef..7cd2f65a 100644 --- a/fframes-native-player/src/presenter/metal.rs +++ b/fframes-native-player/src/presenter/metal.rs @@ -1,31 +1,43 @@ use std::ffi::c_void; +use std::time::{Duration, Instant}; use core_graphics_types::geometry::CGSize; +use fframes::usvgr; use fframes_skia_renderer::metal::SkiaMetalCtx; use fframes_skia_renderer::metal::metal_rs::foreign_types::{ForeignType, ForeignTypeRef}; use fframes_skia_renderer::metal::metal_rs::objc::Message; use fframes_skia_renderer::metal::metal_rs::objc::rc::autoreleasepool; use fframes_skia_renderer::metal::metal_rs::objc::runtime::Sel; -use fframes_skia_renderer::metal::metal_rs::{MTLPixelFormat, MetalLayer, MetalLayerRef}; +use fframes_skia_renderer::metal::metal_rs::{ + MTLPixelFormat, MTLRegion, MTLStorageMode, MetalLayer, MetalLayerRef, TextureRef, +}; use fframes_skia_renderer::skia_safe::gpu::{ self, DirectContext, SurfaceOrigin, SyncCpu, backend_render_targets, mtl, }; -use fframes_skia_renderer::skia_safe::{Canvas, ColorType}; +use fframes_skia_renderer::skia_safe::{ + AlphaType, Canvas, ColorType, ImageInfo, Surface, surfaces, +}; use winit::raw_window_handle::{HasWindowHandle, RawWindowHandle}; use winit::window::Window; use super::WindowTarget; use crate::error::{PlayerError, PlayerResult}; +use crate::scheduler::PlaybackFrame; /// Skia draws into the drawables of a `CAMetalLayer` of the window. pub(super) struct MetalTarget { + raster: Option, gpu: DirectContext, layer: MetalLayer, backend: SkiaMetalCtx, } impl MetalTarget { - pub(super) fn new(window: &Window, vsync: bool) -> PlayerResult { + pub(super) fn new( + window: &Window, + vsync: bool, + raster_fps: Option, + ) -> PlayerResult { // The size is for the backend's own surfaces, which the player does not use. let backend = SkiaMetalCtx::new(16, 16)?; let gpu = backend.create_context()?; @@ -48,7 +60,12 @@ impl MetalTarget { layer.set_display_sync_enabled(vsync); set_srgb_colorspace(&layer); + let raster = raster_fps + .filter(|_| backend.device().has_unified_memory()) + .map(RasterPreview::new); + Ok(Self { + raster, gpu, layer, backend, @@ -63,11 +80,24 @@ impl WindowTarget for MetalTarget { &mut DirectContext, Option<&fframes_skia_renderer::QueueLock>, )> { + self.raster = None; Some((&mut self.gpu, None)) } fn is_gpu(&self) -> bool { - true + self.raster.is_none() + } + + fn prepare_frame(&mut self, frame: Option<&PlaybackFrame>) { + if self.raster.is_none() { + return; + } + let mut remaining = MAX_RASTER_NODES; + if let Some(frame) = frame + && !matches!(frame, PlaybackFrame::Tree(tree) if simple_group(tree.root(), &mut remaining)) + { + self.raster = None; + } } fn resize(&mut self, width: u32, height: u32) -> PlayerResult<()> { @@ -85,6 +115,18 @@ impl WindowTarget for MetalTarget { }; let texture = drawable.texture(); + if let Some(raster) = &mut self.raster { + if raster.draw(texture, draw) { + drawable.present(); + if raster.slow_frames >= 6 { + self.raster = None; + } + return Ok(()); + } + // Some devices expose private drawable textures. Draw this frame on the GPU. + self.raster = None; + } + // SAFETY: the texture lives as long as the drawable, which outlives the surface. let texture_info = unsafe { mtl::TextureInfo::new(texture.as_ptr() as mtl::Handle) }; let render_target = backend_render_targets::make_mtl( @@ -116,6 +158,97 @@ impl WindowTarget for MetalTarget { } } +/// On unified-memory Macs we can present a CPU-written drawable without submitting a +/// Metal render or blit pass. Keep one full-resolution raster surface between frames. +struct RasterPreview { + surface: Option, + budget: Duration, + slow_frames: u8, +} + +impl RasterPreview { + fn new(fps: usize) -> Self { + Self { + surface: None, + budget: Duration::from_secs_f64(1. / (3. * fps.max(1) as f64)), + slow_frames: 0, + } + } + + fn draw(&mut self, texture: &TextureRef, draw: &mut dyn FnMut(&Canvas)) -> bool { + if !matches!( + texture.storage_mode(), + MTLStorageMode::Shared | MTLStorageMode::Managed + ) { + return false; + } + + let start = Instant::now(); + let (width, height) = (texture.width(), texture.height()); + let Ok(w) = i32::try_from(width) else { + return false; + }; + let Ok(h) = i32::try_from(height) else { + return false; + }; + if self + .surface + .as_ref() + .is_none_or(|s| s.width() != w || s.height() != h) + { + let info = ImageInfo::new((w, h), ColorType::BGRA8888, AlphaType::Premul, None); + self.surface = surfaces::raster(&info, None, None); + } + let Some(surface) = self.surface.as_mut() else { + return false; + }; + draw(surface.canvas()); + // A raster surface owns contiguous pixels of the drawable's size and BGRA format. + let Some(pixels) = surface.peek_pixels() else { + return false; + }; + texture.replace_region( + MTLRegion::new_2d(0, 0, width, height), + 0, + pixels.addr(), + pixels.row_bytes() as u64, + ); + if start.elapsed() > self.budget { + self.slow_frames += 1; + } else { + self.slow_frames = 0; + } + true + } +} + +const MAX_RASTER_NODES: usize = 128; + +/// Keep images (including video, shaders and wgpu layers), complex scenes and blend +/// effects on the GPU. Bound the work of checking a large scene before promotion. +fn simple_group(group: &usvgr::Group, remaining: &mut usize) -> bool { + !group.should_isolate() + && group.children().iter().all(|node| { + if *remaining == 0 { + return false; + } + *remaining -= 1; + let path = match node { + usvgr::Node::Group(child) => return simple_group(child, remaining), + usvgr::Node::Text(text) => return simple_group(text.flattened(), remaining), + usvgr::Node::Image(_) => return false, + usvgr::Node::Path(path) => path, + usvgr::Node::FastShape(shape) => shape.path(), + }; + // A pattern can contain an arbitrary subtree, including GPU layers. + path.fill() + .is_none_or(|fill| !matches!(fill.paint(), usvgr::Paint::Pattern(_))) + && path + .stroke() + .is_none_or(|stroke| !matches!(stroke.paint(), usvgr::Paint::Pattern(_))) + }) +} + /// Skia draws sRGB. Tagging the layer makes macOS convert it to the color space of the /// display, an untagged layer is shown as is (oversaturated on wide gamut displays). fn set_srgb_colorspace(layer: &MetalLayerRef) { @@ -137,3 +270,24 @@ fn set_srgb_colorspace(layer: &MetalLayerRef) { CGColorSpaceRelease(space); } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn only_small_vector_frames_use_the_raster_path() { + let simple = r#""#; + for (content, budget, expected) in [ + (simple.to_owned(), 1, true), + (simple.repeat(2), 1, false), + (format!(r#"{simple}"#), 128, false), + (r#""#.to_owned(), 128, false), + ] { + let svg = format!(r#"{content}"#); + let tree = usvgr::Tree::from_str(&svg, &usvgr::Options::default(), &usvgr::fontdb::Database::new()).unwrap(); + let mut remaining = budget; + assert_eq!(simple_group(tree.root(), &mut remaining), expected, "{content}"); + } + } +} diff --git a/fframes-native-player/src/presenter/mod.rs b/fframes-native-player/src/presenter/mod.rs index c68473d1..d621cca4 100644 --- a/fframes-native-player/src/presenter/mod.rs +++ b/fframes-native-player/src/presenter/mod.rs @@ -34,6 +34,9 @@ pub(crate) struct BufferedFrame { trait WindowTarget { fn is_gpu(&self) -> bool; + /// Allow an automatic target to switch to GPU rendering before drawing this frame. + fn prepare_frame(&mut self, _frame: Option<&PlaybackFrame>) {} + fn gpu_context(&mut self) -> Option<(&mut gpu::DirectContext, Option<&QueueLock>)> { None } @@ -73,13 +76,14 @@ pub(crate) struct Presenter { impl Presenter { pub(crate) fn new( window: Rc, - backend: PlayerBackend, - vsync: bool, - background: fframes::Color, - duration_label: String, - model: &Studio, + config: &crate::app::AppConfig<'_>, + video_sources: bool, ) -> PlayerResult { - let target = create_target(&window, backend, vsync)?; + let raster_fps = (config.limit_prefetch_memory && !video_sources).then_some(config.fps); + let target = create_target(&window, config.backend, config.vsync, raster_fps)?; + let duration_label = crate::controls::format_time(config.duration_in_frames, config.fps); + let background = config.background; + let model = &config.studio; let controls = Controls::new(); let size = window.inner_size(); let layout = ControlsLayout::new( @@ -221,6 +225,7 @@ impl Presenter { .. } = self; + target.prepare_frame(frame); target.present(&mut |canvas| { let view = studio_layout.as_ref().filter(|_| studio_open); canvas.clear(matte(view)); @@ -314,26 +319,29 @@ fn create_target( window: &Rc, backend: PlayerBackend, vsync: bool, + raster_fps: Option, ) -> PlayerResult> { + #[cfg(not(feature = "metal"))] + let _ = raster_fps; match backend { PlayerBackend::Auto => { #[cfg(feature = "metal")] - match create_target(window, PlayerBackend::Metal, vsync) { - Ok(target) => return Ok(target), + match metal::MetalTarget::new(window, vsync, raster_fps) { + Ok(target) => return Ok(Box::new(target)), Err(err) => eprintln!("fframes player: metal is not available ({err})"), } #[cfg(feature = "vulkan")] - match create_target(window, PlayerBackend::Vulkan, vsync) { + match create_target(window, PlayerBackend::Vulkan, vsync, None) { Ok(target) => return Ok(target), Err(err) => eprintln!("fframes player: vulkan is not available ({err})"), } eprintln!("fframes player: falling back to the Skia CPU renderer"); - create_target(window, PlayerBackend::Cpu, vsync) + create_target(window, PlayerBackend::Cpu, vsync, None) } #[cfg(feature = "metal")] - PlayerBackend::Metal => Ok(Box::new(metal::MetalTarget::new(window, vsync)?)), + PlayerBackend::Metal => Ok(Box::new(metal::MetalTarget::new(window, vsync, None)?)), #[cfg(feature = "vulkan")] PlayerBackend::Vulkan => Ok(Box::new(vulkan::VulkanTarget::new(window.clone(), vsync)?)), PlayerBackend::Cpu => Ok(Box::new(cpu::CpuTarget::new(window.clone())?)), diff --git a/fframes-native-player/src/scheduler.rs b/fframes-native-player/src/scheduler.rs index 1095dd10..7e5f0e04 100644 --- a/fframes-native-player/src/scheduler.rs +++ b/fframes-native-player/src/scheduler.rs @@ -295,8 +295,10 @@ impl Scheduler { on_frame_ready: &(dyn Fn(FrameReady) + Sync), ) { // Text caches are `Rc` based and therefore owned by the worker. - let text_cache = TextCache::new(100); - let mut converter_cache = usvgr::Cache::new_with_text_cache(100); + // A changing frame counter would otherwise retain 100 sets of glyph outlines per worker. + const TEXT_CACHE_ENTRIES: usize = 8; + let text_cache = TextCache::new(TEXT_CACHE_ENTRIES); + let mut converter_cache = usvgr::Cache::new_with_text_cache(TEXT_CACHE_ENTRIES); let video_decoders = match kind { WorkerKind::Playback => self.playback_decoders.clone(), WorkerKind::Thumbnails => VideoDecodersWorker::for_preview(self.software_decode), diff --git a/fframes/src/renderer/renderer.rs b/fframes/src/renderer/renderer.rs index 4bca6216..87d580c3 100644 --- a/fframes/src/renderer/renderer.rs +++ b/fframes/src/renderer/renderer.rs @@ -105,7 +105,32 @@ impl<'a, 'media: 'a> FFramesRendererRuntime<'a> { scenes: &Scenes<'a>, media: Option<&'media dyn MediaProvider<'media>>, ) -> FFramesRendererResult { - let timeline = Self::resolve_video_timeline(time_base, video, scenes, media)?; + Self::with_audio_sample_rate(time_base, video, scenes, media, || time_base.sample_rate) + } + + /// Choose the playback device's sample rate only for projects with audio tracks. + /// Silent projects keep the supplied time base and never need to open a device. + pub fn with_audio_sample_rate( + mut time_base: TimeBase, + video: &'a TVideo, + scenes: &Scenes<'a>, + media: Option<&'media dyn MediaProvider<'media>>, + audio_sample_rate: impl FnOnce() -> usize, + ) -> FFramesRendererResult { + let scenes = ScenesWithAudio::new(scenes); + let audio = video.audio(); + let has_tracks = + |map: &crate::AudioMap<'_>| map.0.as_ref().is_some_and(|tracks| !tracks.is_empty()); + if has_tracks(&audio) + || scenes + .0 + .iter() + .flatten() + .any(|scene| has_tracks(&scene.audio_map)) + { + time_base.sample_rate = audio_sample_rate(); + } + let timeline = Self::resolve_audio_timeline(time_base, video, &scenes, &audio, media)?; let mut font_source = RendererFontSource { fontdb: fontdb::Database::new(), @@ -122,17 +147,34 @@ impl<'a, 'media: 'a> FFramesRendererRuntime<'a> { }) } + #[cfg(feature = "cli")] pub(super) fn resolve_video_timeline( time_base: TimeBase, video: &'a TVideo, scenes: &Scenes<'a>, media: Option<&'media dyn MediaProvider<'media>>, + ) -> FFramesRendererResult> { + Self::resolve_audio_timeline( + time_base, + video, + &ScenesWithAudio::new(scenes), + &video.audio(), + media, + ) + } + + fn resolve_audio_timeline( + time_base: TimeBase, + video: &'a TVideo, + scenes: &ScenesWithAudio<'a>, + audio: &crate::AudioMap<'_>, + media: Option<&'media dyn MediaProvider<'media>>, ) -> FFramesRendererResult> { Ok(crate::resolve_timeline( &video.duration(), - &ScenesWithAudio::new(scenes), + scenes, &time_base, - &video.audio(), + audio, |name| { let media = media.ok_or_else(|| { crate::error::FFramesError::RequiredAudioNotFound(name.to_owned()) diff --git a/fframes/src/renderer/tests/preview_tests.rs b/fframes/src/renderer/tests/preview_tests.rs index f2a767fd..24d36e95 100644 --- a/fframes/src/renderer/tests/preview_tests.rs +++ b/fframes/src/renderer/tests/preview_tests.rs @@ -201,3 +201,44 @@ fn frames_crop_to_a_clamped_rectangle() { let corner = frame.crop(190, 90, 50, 50); assert_eq!((corner.width, corner.height), (10, 10)); } + +#[test] +fn audio_device_rate_is_only_requested_for_playable_tracks() { + use crate::{AudioTimestamp::Second, FFramesRendererRuntime, Scenes, TimeBase}; + let time = TimeBase { + fps: 10, + sample_rate: 44100, + }; + let silent = FFramesRendererRuntime::with_audio_sample_rate( + time, + &TwoScenes, + &TwoScenes.define_scenes(), + None, + || panic!("silent scenes opened the audio device"), + ) + .unwrap(); + assert_eq!(silent.time_base.sample_rate, 44100); + + #[derive(Debug)] + struct Sound; + impl Scene for Sound { + fn audio(&self) -> AudioMap<'_> { + AudioMap::from([("clip.wav", Second(0.)..Second(1.))]) + } + fn duration(&self) -> crate::Duration<'_> { + crate::Duration::Seconds(1.) + } + fn render_frame<'a>(&'a self, _: Frame, _: &FFramesContext) -> Svgr<'a> { + unreachable!() + } + } + let scenes = Scenes::from(vec![&Sound as &dyn Scene]); + let voiced = + FFramesRendererRuntime::with_audio_sample_rate(time, &TwoScenes, &scenes, None, || 48000) + .unwrap(); + assert_eq!(voiced.time_base.sample_rate, 48000); + assert_eq!( + voiced.timeline.audio_map.unwrap().tracks()[0].range.end.0, + 48000 + ); +} From f5245f02a0e97badf16f5688c3ae562c68e71c92 Mon Sep 17 00:00:00 2001 From: dmtrKovalenko Date: Thu, 29 Oct 2026 14:04:37 -0700 Subject: [PATCH 11/21] test(renderer): Define the audio fixture before playback assertions --- fframes/src/renderer/tests/preview_tests.rs | 27 +++++++++++---------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/fframes/src/renderer/tests/preview_tests.rs b/fframes/src/renderer/tests/preview_tests.rs index 24d36e95..f4314e74 100644 --- a/fframes/src/renderer/tests/preview_tests.rs +++ b/fframes/src/renderer/tests/preview_tests.rs @@ -205,6 +205,20 @@ fn frames_crop_to_a_clamped_rectangle() { #[test] fn audio_device_rate_is_only_requested_for_playable_tracks() { use crate::{AudioTimestamp::Second, FFramesRendererRuntime, Scenes, TimeBase}; + #[derive(Debug)] + struct Sound; + impl Scene for Sound { + fn audio(&self) -> AudioMap<'_> { + AudioMap::from([("clip.wav", Second(0.)..Second(1.))]) + } + fn duration(&self) -> crate::Duration<'_> { + crate::Duration::Seconds(1.) + } + fn render_frame<'a>(&'a self, _: Frame, _: &FFramesContext) -> Svgr<'a> { + unreachable!() + } + } + let time = TimeBase { fps: 10, sample_rate: 44100, @@ -219,19 +233,6 @@ fn audio_device_rate_is_only_requested_for_playable_tracks() { .unwrap(); assert_eq!(silent.time_base.sample_rate, 44100); - #[derive(Debug)] - struct Sound; - impl Scene for Sound { - fn audio(&self) -> AudioMap<'_> { - AudioMap::from([("clip.wav", Second(0.)..Second(1.))]) - } - fn duration(&self) -> crate::Duration<'_> { - crate::Duration::Seconds(1.) - } - fn render_frame<'a>(&'a self, _: Frame, _: &FFramesContext) -> Svgr<'a> { - unreachable!() - } - } let scenes = Scenes::from(vec![&Sound as &dyn Scene]); let voiced = FFramesRendererRuntime::with_audio_sample_rate(time, &TwoScenes, &scenes, None, || 48000) From 7992844e0e475d40f702c9b008e16b90da32ca62 Mon Sep 17 00:00:00 2001 From: dmtrKovalenko Date: Thu, 29 Oct 2026 15:47:05 -0700 Subject: [PATCH 12/21] perf(player): Release unused Skia GPU resources during preview MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ask Skia once per second to release unlocked GPU resources that the Metal preview has not used for a second. Active resources and buffered frames stay available. --- Beta played at 2560×1440 on an M5 Max. Values are medians of three hidden runs per configuration. | Measurement | Before | With cleanup | | --- | ---: | ---: | | Highest sampled Skia resource cache | 250.9 MiB | 153.4 MiB | | Peak process physical footprint | 1017.3 MiB | 891.5 MiB | | Steady process physical footprint | 715.4 MiB | 680.6 MiB | | Steady process RSS | 374.0 MiB | 354.9 MiB | | Timeline positions presented out of 1920 | 1914 | 1914 | --- fframes-native-player/src/presenter/metal.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/fframes-native-player/src/presenter/metal.rs b/fframes-native-player/src/presenter/metal.rs index 7cd2f65a..2c533420 100644 --- a/fframes-native-player/src/presenter/metal.rs +++ b/fframes-native-player/src/presenter/metal.rs @@ -24,10 +24,13 @@ use super::WindowTarget; use crate::error::{PlayerError, PlayerResult}; use crate::scheduler::PlaybackFrame; +const GPU_CACHE_IDLE: Duration = Duration::from_secs(1); + /// Skia draws into the drawables of a `CAMetalLayer` of the window. pub(super) struct MetalTarget { raster: Option, gpu: DirectContext, + last_cache_cleanup: Instant, layer: MetalLayer, backend: SkiaMetalCtx, } @@ -67,6 +70,7 @@ impl MetalTarget { Ok(Self { raster, gpu, + last_cache_cleanup: Instant::now(), layer, backend, }) @@ -127,6 +131,12 @@ impl WindowTarget for MetalTarget { self.raster = None; } + // Release unlocked Skia resources from earlier scenes while retaining active frames. + if self.last_cache_cleanup.elapsed() >= GPU_CACHE_IDLE { + self.gpu.perform_deferred_cleanup(GPU_CACHE_IDLE, None); + self.last_cache_cleanup = Instant::now(); + } + // SAFETY: the texture lives as long as the drawable, which outlives the surface. let texture_info = unsafe { mtl::TextureInfo::new(texture.as_ptr() as mtl::Handle) }; let render_target = backend_render_targets::make_mtl( From 40ab586da1c3ac32baac7599f6eee10d76afb273 Mon Sep 17 00:00:00 2001 From: dmtrKovalenko Date: Thu, 29 Oct 2026 17:35:22 -0700 Subject: [PATCH 13/21] perf(player): Select preview workers without starting Rayon Count available CPUs without initializing Rayon's global thread pool. Preserve the explicit preview worker count and the FFRAMES_NUM_THREADS and RAYON_NUM_THREADS environment settings. --- The hidden Hello World preview on the M5 Max used 27 process threads instead of 45 with the same eight playback workers. --- fframes-native-player/src/options.rs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/fframes-native-player/src/options.rs b/fframes-native-player/src/options.rs index 94a7ec23..3d287d22 100644 --- a/fframes-native-player/src/options.rs +++ b/fframes-native-player/src/options.rs @@ -97,7 +97,20 @@ impl PlayerOptions<'_, '_> { .clamp(1, crate::scheduler::MAX_STAGED_TREES + 1); } self.render_threads - .unwrap_or_else(|| (fframes::get_thread_count() / 2).clamp(1, 8)) + .unwrap_or_else(|| { + // The preview owns its workers. Querying Rayon here starts a second idle pool. + let cores = std::env::var("FFRAMES_NUM_THREADS") + .ok() + .and_then(|value| value.parse::().ok()) + .or_else(|| { + std::env::var("RAYON_NUM_THREADS") + .ok() + .and_then(|value| value.parse::().ok()) + .filter(|&threads| threads > 0) + }) + .unwrap_or_else(|| std::thread::available_parallelism().map_or(1, usize::from)); + (cores / 2).clamp(1, 8) + }) .max(1) } } From 164d9469ef4842336814e1c86994ad14bc96d5a9 Mon Sep 17 00:00:00 2001 From: dmtrKovalenko Date: Thu, 29 Oct 2026 17:43:37 -0700 Subject: [PATCH 14/21] refactor(player): Draw scenes directly with Skia and buffer prepared trees Draw each due scene directly into the window drawable or swapchain. Buffer up to four scene trees by default and reuse ready trees for forward seeks. Remove CPU raster uploads, prerendered frame images, and speculative GPU draws. --- | Playback state | Before | After | | --- | --- | --- | | Prepared frames | Scene trees and rendered images | Scene trees | | Automatic GPU presentation | Raster uploads or direct drawing | Direct Skia drawing | | Default lookahead | Up to two seconds plus image staging | Up to four scene trees | --- README.md | 9 +- fframes-native-player/src/app.rs | 134 +------------ fframes-native-player/src/lib.rs | 6 +- fframes-native-player/src/options.rs | 23 +-- fframes-native-player/src/presenter/metal.rs | 170 +--------------- fframes-native-player/src/presenter/mod.rs | 109 +---------- fframes-native-player/src/presenter/vulkan.rs | 9 - fframes-native-player/src/scheduler.rs | 183 +++++++----------- 8 files changed, 104 insertions(+), 539 deletions(-) diff --git a/README.md b/README.md index fb1e03d5..ff2bfb1f 100644 --- a/README.md +++ b/README.md @@ -347,12 +347,9 @@ to hide the timeline and expand the video. On macOS the studio tries VideoToolbox for supported source videos and falls back to software for unsupported formats. `preview --software-decode` uses the CPU for decoding while keeping the selected drawing backend. Playback and filmstrip workers release idle source decoders. -On unified-memory Macs, automatic playback draws simple vector scenes into a reusable -full-resolution raster surface and presents it through Metal. Source videos, images, effects, -large scenes, and frames that repeatedly exceed their drawing budget use GPU rendering. -Use `preview --backend metal` to render every frame on the GPU. -The automatic lookahead covers up to two seconds. The CPU draws directly into the window. -The GPU budgets 96 MiB for finished frames when playing source videos or expensive scenes. +The GPU preview draws each scene through Skia directly into the window drawable or swapchain. +The default buffer holds up to four prepared scene trees. Source video frames remain attached +to their trees until drawing finishes. The CPU backend draws directly into the window buffer. `cargo fframes new` takes `--template single-scene|multi-scene`, `--format landscape|portrait|square|uhd`, `--fps`, `--title` and `--backend`. On macOS and Linux (arm64, x86_64) the first build downloads prebuilt Skia and ffmpeg diff --git a/fframes-native-player/src/app.rs b/fframes-native-player/src/app.rs index 6bb37558..bf0ac97d 100644 --- a/fframes-native-player/src/app.rs +++ b/fframes-native-player/src/app.rs @@ -1,4 +1,3 @@ -use std::collections::BTreeMap; use std::fmt::Write as _; use std::rc::Rc; use std::time::{Duration, Instant}; @@ -13,15 +12,13 @@ use winit::window::{CursorIcon, Fullscreen, Window, WindowId}; use crate::controls::{Control, ControlsState, format_time}; use crate::error::PlayerError; use crate::options::PlayerBackend; -use crate::presenter::{BufferedFrame, Presenter}; -use crate::scheduler::{PlaybackFrame, Scheduler}; +use crate::presenter::Presenter; +use crate::scheduler::Scheduler; use crate::studio::{Studio, StudioLayout, StudioScroll}; use fframes_skia_renderer::skia_safe::Point; /// The control bar hides after this long without pointer movement while playing. const CONTROLS_IDLE_TIMEOUT: Duration = Duration::from_millis(2500); -/// Conservative RGBA budget using the whole window as an upper bound per image. -const BUFFER_BYTES: u64 = 96 << 20; /// A frame worker finished a playback or filmstrip job. pub(crate) enum FrameReady { @@ -80,19 +77,6 @@ pub(crate) struct AppConfig<'a> { pub looping: bool, pub autoplay: bool, pub start_frame: usize, - pub limit_prefetch_memory: bool, -} - -impl AppConfig<'_> { - fn should_snapshot(&self, gpu: bool, video_sources: bool, draw_time: Duration) -> bool { - if !self.limit_prefetch_memory { - return true; - } - // Rasterizing ahead competes with presentation on the same CPU. Cheap GPU - // scenes also fit their deadline without retaining extra surfaces. - let frame_interval = Duration::from_secs_f64(1. / self.fps.max(1) as f64); - gpu && (video_sources || draw_time >= frame_interval / 2) - } } pub(crate) struct App<'s> { @@ -102,13 +86,10 @@ pub(crate) struct App<'s> { audio: Option<&'s crate::audio::AudioOutput>, window: Option>, - // Drop texture-backed images before the presenter and its Skia context. - current: Option<(u64, PlaybackFrame)>, - buffered: BTreeMap, + // Keep source frames alive until Skia finishes drawing them. + current: Option<(u64, fframes::usvgr::Tree)>, presenter: Option, redraw_pending: bool, - draw_time: Duration, - snapshots_available: bool, pub(crate) error: Option, clock: Clock, @@ -158,11 +139,8 @@ impl<'s> App<'s> { #[cfg(feature = "audio")] audio, window: None, - buffered: BTreeMap::new(), presenter: None, redraw_pending: false, - draw_time: Duration::ZERO, - snapshots_available: true, error: None, clock, current: None, @@ -204,14 +182,10 @@ impl<'s> App<'s> { self.scheduler.frame_of(position, self.looping) } - /// Restarts playback at `position`, all the prepared frames are discarded. + /// Seeks to `position`, reusing prepared scenes when possible. fn seek_to(&mut self, position: u64, playing: bool) { self.scrub_target = None; - if !self - .scheduler - .seek_buffered(position, self.looping, &mut self.buffered) - { - self.buffered.clear(); + if !self.scheduler.seek_buffered(position, self.looping) { self.scheduler.seek(position, self.looping); } self.set_playback(position, playing); @@ -261,7 +235,7 @@ impl<'s> App<'s> { } } - if let Some(frame) = self.scheduler.take(position, &mut self.buffered) { + if let Some(frame) = self.scheduler.take(position) { let next_frame = self.scheduler.frame_of(frame.0, self.looping); self.current = Some(frame); self.needs_redraw = true; @@ -415,15 +389,6 @@ impl<'s> App<'s> { } fn invalidate_view(&mut self) { - if !self.buffered.is_empty() || matches!(self.current, Some((_, PlaybackFrame::Image(_)))) { - self.buffered.clear(); - if matches!(self.current, Some((_, PlaybackFrame::Image(_)))) { - self.current = None; - self.presented = None; - } - self.scheduler.seek(self.clock.position(), self.looping); - } - self.snapshots_available = true; self.needs_redraw = true; } @@ -600,59 +565,6 @@ impl<'s> App<'s> { } } - /// Start at most one speculative draw before returning to the event loop. - fn draw_ahead(&mut self) -> bool { - if self.redraw_pending - || self.drag.is_some() - || self.scrub_target.is_some() - || !self.snapshots_available - || self.presented != Some(self.clock.position()) - { - return false; - } - let Some(presenter) = self.presenter.as_mut() else { - return false; - }; - if !self.config.should_snapshot( - presenter.is_gpu(), - self.scheduler.video_sources, - self.draw_time, - ) { - return false; - } - if let Some(deadline) = self.clock.next_frame_at() - && deadline.saturating_duration_since(Instant::now()) - <= self.draw_time + Duration::from_millis(2) - { - return false; - } - let Some(window) = &self.window else { - return false; - }; - let size = window.inner_size(); - let bytes = (u64::from(size.width) * u64::from(size.height)).saturating_mul(4); - // Leave room for the displayed image and keep large/high-DPI windows bounded. - if bytes == 0 - || (self.config.limit_prefetch_memory - && bytes.saturating_mul(self.buffered.len() as u64 + 2) > BUFFER_BYTES) - { - return false; - } - let Some((position, tree)) = self.scheduler.next_to_draw() else { - return false; - }; - let start = Instant::now(); - if let Some(image) = presenter.snapshot(&tree, self.studio_open) { - self.draw_time = start.elapsed(); - self.buffered.insert(position, image); - true - } else { - self.scheduler.put_back(position, tree); - self.snapshots_available = false; - false - } - } - fn redraw(&mut self) -> Result<(), PlayerError> { self.redraw_pending = false; let controls = ControlsState { @@ -667,7 +579,6 @@ impl<'s> App<'s> { }; if let Some(presenter) = self.presenter.as_mut() { - let start = Instant::now(); presenter.present( self.current.as_ref().map(|(_, tree)| tree), &controls, @@ -677,11 +588,6 @@ impl<'s> App<'s> { )?; let position = self.current.as_ref().map(|(position, _)| *position); - if position != self.presented - && matches!(self.current, Some((_, PlaybackFrame::Tree(_)))) - { - self.draw_time = start.elapsed(); - } if position.is_some() && position != self.presented { self.frames_shown += 1; } @@ -730,7 +636,7 @@ impl ApplicationHandler for App<'_> { Err(err) => return self.fail(event_loop, err.into()), }; - match Presenter::new(window.clone(), &self.config, self.scheduler.video_sources) { + match Presenter::new(window.clone(), &self.config) { Ok(presenter) => self.presenter = Some(presenter), Err(err) => return self.fail(event_loop, err), } @@ -803,10 +709,6 @@ impl ApplicationHandler for App<'_> { fn about_to_wait(&mut self, event_loop: &ActiveEventLoop) { self.tick(); - if self.draw_ahead() { - event_loop.set_control_flow(ControlFlow::Poll); - return; - } event_loop.set_control_flow(match self.clock.next_frame_at() { Some(next_frame) => ControlFlow::WaitUntil(next_frame), // Paused: sleep until input or a worker delivers the requested frame. @@ -843,7 +745,6 @@ mod tests { looping: false, autoplay: false, start_frame: 0, - limit_prefetch_memory: true, }, scheduler, #[cfg(feature = "audio")] @@ -851,24 +752,9 @@ mod tests { ) } - #[test] - fn automatic_buffer_reserves_surfaces_for_gpu_work_that_needs_them() { - let scheduler = Scheduler::new(100, 3, false, 0, false, false); - let mut app = paused_app(&scheduler); - let config = &mut app.config; - let fast = Duration::from_millis(1); - let slow = Duration::from_millis(25); - assert!(!config.should_snapshot(false, true, slow)); - assert!(!config.should_snapshot(true, false, fast)); - assert!(config.should_snapshot(true, true, fast)); - assert!(config.should_snapshot(true, false, slow)); - config.limit_prefetch_memory = false; - assert!(config.should_snapshot(false, false, fast)); - } - #[test] fn resuming_keeps_prefetched_frames_unless_playback_restarts() { - let scheduler = Scheduler::new(100, 3, false, 0, false, false); + let scheduler = Scheduler::new(100, 3, false, 0, false); let mut app = paused_app(&scheduler); app.seek_to(10, false); app.toggle_play(); @@ -882,7 +768,7 @@ mod tests { #[test] fn failed_scrub_continues_at_the_latest_pointer_position() { - let scheduler = Scheduler::new(100, 3, false, 0, false, false); + let scheduler = Scheduler::new(100, 3, false, 0, false); let mut app = paused_app(&scheduler); app.seek_to(10, false); app.scrub_target = Some(20); diff --git a/fframes-native-player/src/lib.rs b/fframes-native-player/src/lib.rs index 2b6b8488..227b4c3d 100644 --- a/fframes-native-player/src/lib.rs +++ b/fframes-native-player/src/lib.rs @@ -140,12 +140,9 @@ pub fn play<'a, 'media: 'a, TVideo: Video + Sync>( .min(timeline.duration_in_frames); let scheduler = Scheduler::new( timeline.duration_in_frames, - options.lookahead(TVideo::FPS), + options.lookahead(), options.looping, thumbnail_count, - options - .media - .is_some_and(fframes::MediaProvider::has_video_sources), options.software_decode, ); let proxy = Mutex::new(event_loop.create_proxy()); @@ -176,7 +173,6 @@ pub fn play<'a, 'media: 'a, TVideo: Video + Sync>( looping: options.looping, autoplay: options.autoplay, start_frame: options.start_frame, - limit_prefetch_memory: options.prefetch_frames == usize::MAX, }, &scheduler, #[cfg(feature = "audio")] diff --git a/fframes-native-player/src/options.rs b/fframes-native-player/src/options.rs index 3d287d22..83656080 100644 --- a/fframes-native-player/src/options.rs +++ b/fframes-native-player/src/options.rs @@ -45,14 +45,12 @@ pub struct PlayerOptions<'a, 'media> { pub audio: bool, /// Maximum number of threads building frame trees (`Video::render_frame`) in parallel. /// Defaults to one worker for video sources so the decoder reads frames in order. - /// Video sources allow up to four staged trees and one worker for a newly due frame. + /// Source video workers are limited to the default buffer size plus one live frame. /// Other projects use half the available cores, up to eight workers. pub render_threads: Option, - /// Maximum number of future frames to prepare. The default (`usize::MAX`) uses - /// two seconds at the video's frame rate. Zero disables preparation ahead of time. - /// On automatic, the CPU prepares frame trees and draws directly into the window. - /// GPU pre-rendering for slow scenes or source videos budgets 96 MiB of finished frames. - /// An explicit frame count takes precedence over that limit. + /// Maximum number of future frame trees to prepare. The default (`usize::MAX`) + /// buffers up to four trees. Zero prepares only the currently due frame. + /// Skia draws each tree directly into the window when it is due. pub prefetch_frames: usize, /// Master bus of the audio mix, see `RenderOptions::audio_mix`. pub audio_mix: fframes::AudioMixOptions, @@ -81,9 +79,9 @@ impl Default for PlayerOptions<'_, '_> { } impl PlayerOptions<'_, '_> { - pub(crate) fn lookahead(&self, fps: usize) -> usize { + pub(crate) fn lookahead(&self) -> usize { if self.prefetch_frames == usize::MAX { - fps.saturating_mul(2) + crate::scheduler::MAX_STAGED_TREES } else { self.prefetch_frames } @@ -120,14 +118,13 @@ mod tests { use super::*; #[test] - fn default_lookahead_tracks_the_video_frame_rate() { + fn lookahead_defaults_to_a_small_tree_buffer() { let mut options = PlayerOptions::default(); - assert_eq!(options.lookahead(24), 48); - assert_eq!(options.lookahead(60), 120); + assert_eq!(options.lookahead(), 4); options.prefetch_frames = 8; - assert_eq!(options.lookahead(24), 8); + assert_eq!(options.lookahead(), 8); options.prefetch_frames = 0; - assert_eq!(options.lookahead(60), 0); + assert_eq!(options.lookahead(), 0); } #[test] diff --git a/fframes-native-player/src/presenter/metal.rs b/fframes-native-player/src/presenter/metal.rs index 2c533420..fedc8141 100644 --- a/fframes-native-player/src/presenter/metal.rs +++ b/fframes-native-player/src/presenter/metal.rs @@ -2,33 +2,26 @@ use std::ffi::c_void; use std::time::{Duration, Instant}; use core_graphics_types::geometry::CGSize; -use fframes::usvgr; use fframes_skia_renderer::metal::SkiaMetalCtx; use fframes_skia_renderer::metal::metal_rs::foreign_types::{ForeignType, ForeignTypeRef}; use fframes_skia_renderer::metal::metal_rs::objc::Message; use fframes_skia_renderer::metal::metal_rs::objc::rc::autoreleasepool; use fframes_skia_renderer::metal::metal_rs::objc::runtime::Sel; -use fframes_skia_renderer::metal::metal_rs::{ - MTLPixelFormat, MTLRegion, MTLStorageMode, MetalLayer, MetalLayerRef, TextureRef, -}; +use fframes_skia_renderer::metal::metal_rs::{MTLPixelFormat, MetalLayer, MetalLayerRef}; use fframes_skia_renderer::skia_safe::gpu::{ self, DirectContext, SurfaceOrigin, SyncCpu, backend_render_targets, mtl, }; -use fframes_skia_renderer::skia_safe::{ - AlphaType, Canvas, ColorType, ImageInfo, Surface, surfaces, -}; +use fframes_skia_renderer::skia_safe::{Canvas, ColorType}; use winit::raw_window_handle::{HasWindowHandle, RawWindowHandle}; use winit::window::Window; use super::WindowTarget; use crate::error::{PlayerError, PlayerResult}; -use crate::scheduler::PlaybackFrame; const GPU_CACHE_IDLE: Duration = Duration::from_secs(1); /// Skia draws into the drawables of a `CAMetalLayer` of the window. pub(super) struct MetalTarget { - raster: Option, gpu: DirectContext, last_cache_cleanup: Instant, layer: MetalLayer, @@ -36,11 +29,7 @@ pub(super) struct MetalTarget { } impl MetalTarget { - pub(super) fn new( - window: &Window, - vsync: bool, - raster_fps: Option, - ) -> PlayerResult { + pub(super) fn new(window: &Window, vsync: bool) -> PlayerResult { // The size is for the backend's own surfaces, which the player does not use. let backend = SkiaMetalCtx::new(16, 16)?; let gpu = backend.create_context()?; @@ -63,12 +52,7 @@ impl MetalTarget { layer.set_display_sync_enabled(vsync); set_srgb_colorspace(&layer); - let raster = raster_fps - .filter(|_| backend.device().has_unified_memory()) - .map(RasterPreview::new); - Ok(Self { - raster, gpu, last_cache_cleanup: Instant::now(), layer, @@ -78,30 +62,8 @@ impl MetalTarget { } impl WindowTarget for MetalTarget { - fn gpu_context( - &mut self, - ) -> Option<( - &mut DirectContext, - Option<&fframes_skia_renderer::QueueLock>, - )> { - self.raster = None; - Some((&mut self.gpu, None)) - } - fn is_gpu(&self) -> bool { - self.raster.is_none() - } - - fn prepare_frame(&mut self, frame: Option<&PlaybackFrame>) { - if self.raster.is_none() { - return; - } - let mut remaining = MAX_RASTER_NODES; - if let Some(frame) = frame - && !matches!(frame, PlaybackFrame::Tree(tree) if simple_group(tree.root(), &mut remaining)) - { - self.raster = None; - } + true } fn resize(&mut self, width: u32, height: u32) -> PlayerResult<()> { @@ -119,18 +81,6 @@ impl WindowTarget for MetalTarget { }; let texture = drawable.texture(); - if let Some(raster) = &mut self.raster { - if raster.draw(texture, draw) { - drawable.present(); - if raster.slow_frames >= 6 { - self.raster = None; - } - return Ok(()); - } - // Some devices expose private drawable textures. Draw this frame on the GPU. - self.raster = None; - } - // Release unlocked Skia resources from earlier scenes while retaining active frames. if self.last_cache_cleanup.elapsed() >= GPU_CACHE_IDLE { self.gpu.perform_deferred_cleanup(GPU_CACHE_IDLE, None); @@ -168,97 +118,6 @@ impl WindowTarget for MetalTarget { } } -/// On unified-memory Macs we can present a CPU-written drawable without submitting a -/// Metal render or blit pass. Keep one full-resolution raster surface between frames. -struct RasterPreview { - surface: Option, - budget: Duration, - slow_frames: u8, -} - -impl RasterPreview { - fn new(fps: usize) -> Self { - Self { - surface: None, - budget: Duration::from_secs_f64(1. / (3. * fps.max(1) as f64)), - slow_frames: 0, - } - } - - fn draw(&mut self, texture: &TextureRef, draw: &mut dyn FnMut(&Canvas)) -> bool { - if !matches!( - texture.storage_mode(), - MTLStorageMode::Shared | MTLStorageMode::Managed - ) { - return false; - } - - let start = Instant::now(); - let (width, height) = (texture.width(), texture.height()); - let Ok(w) = i32::try_from(width) else { - return false; - }; - let Ok(h) = i32::try_from(height) else { - return false; - }; - if self - .surface - .as_ref() - .is_none_or(|s| s.width() != w || s.height() != h) - { - let info = ImageInfo::new((w, h), ColorType::BGRA8888, AlphaType::Premul, None); - self.surface = surfaces::raster(&info, None, None); - } - let Some(surface) = self.surface.as_mut() else { - return false; - }; - draw(surface.canvas()); - // A raster surface owns contiguous pixels of the drawable's size and BGRA format. - let Some(pixels) = surface.peek_pixels() else { - return false; - }; - texture.replace_region( - MTLRegion::new_2d(0, 0, width, height), - 0, - pixels.addr(), - pixels.row_bytes() as u64, - ); - if start.elapsed() > self.budget { - self.slow_frames += 1; - } else { - self.slow_frames = 0; - } - true - } -} - -const MAX_RASTER_NODES: usize = 128; - -/// Keep images (including video, shaders and wgpu layers), complex scenes and blend -/// effects on the GPU. Bound the work of checking a large scene before promotion. -fn simple_group(group: &usvgr::Group, remaining: &mut usize) -> bool { - !group.should_isolate() - && group.children().iter().all(|node| { - if *remaining == 0 { - return false; - } - *remaining -= 1; - let path = match node { - usvgr::Node::Group(child) => return simple_group(child, remaining), - usvgr::Node::Text(text) => return simple_group(text.flattened(), remaining), - usvgr::Node::Image(_) => return false, - usvgr::Node::Path(path) => path, - usvgr::Node::FastShape(shape) => shape.path(), - }; - // A pattern can contain an arbitrary subtree, including GPU layers. - path.fill() - .is_none_or(|fill| !matches!(fill.paint(), usvgr::Paint::Pattern(_))) - && path - .stroke() - .is_none_or(|stroke| !matches!(stroke.paint(), usvgr::Paint::Pattern(_))) - }) -} - /// Skia draws sRGB. Tagging the layer makes macOS convert it to the color space of the /// display, an untagged layer is shown as is (oversaturated on wide gamut displays). fn set_srgb_colorspace(layer: &MetalLayerRef) { @@ -280,24 +139,3 @@ fn set_srgb_colorspace(layer: &MetalLayerRef) { CGColorSpaceRelease(space); } } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn only_small_vector_frames_use_the_raster_path() { - let simple = r#""#; - for (content, budget, expected) in [ - (simple.to_owned(), 1, true), - (simple.repeat(2), 1, false), - (format!(r#"{simple}"#), 128, false), - (r#""#.to_owned(), 128, false), - ] { - let svg = format!(r#"{content}"#); - let tree = usvgr::Tree::from_str(&svg, &usvgr::Options::default(), &usvgr::fontdb::Database::new()).unwrap(); - let mut remaining = budget; - assert_eq!(simple_group(tree.root(), &mut remaining), expected, "{content}"); - } - } -} diff --git a/fframes-native-player/src/presenter/mod.rs b/fframes-native-player/src/presenter/mod.rs index d621cca4..a215e4b7 100644 --- a/fframes-native-player/src/presenter/mod.rs +++ b/fframes-native-player/src/presenter/mod.rs @@ -12,35 +12,18 @@ use std::rc::Rc; use fframes::usvgr; use fframes_skia_renderer::render::RenderCache; -use fframes_skia_renderer::skia_safe::{ - Canvas, Color, IPoint, IRect, Image, ImageInfo, Point, Rect, RoundOut, Surface, gpu, surfaces, -}; -use fframes_skia_renderer::{QueueLock, lock_queue}; +use fframes_skia_renderer::skia_safe::{Canvas, Color, Rect}; use winit::window::Window; use crate::controls::{Controls, ControlsLayout, ControlsState}; use crate::error::PlayerResult; use crate::options::PlayerBackend; -use crate::scheduler::PlaybackFrame; use crate::studio::{Studio, StudioLayout, StudioScroll}; -/// Only the video occupies this image. Its origin is in window pixels. -pub(crate) struct BufferedFrame { - pub(crate) origin: IPoint, - pub(crate) image: Image, -} - /// The images of the window that Skia draws into. trait WindowTarget { fn is_gpu(&self) -> bool; - /// Allow an automatic target to switch to GPU rendering before drawing this frame. - fn prepare_frame(&mut self, _frame: Option<&PlaybackFrame>) {} - - fn gpu_context(&mut self) -> Option<(&mut gpu::DirectContext, Option<&QueueLock>)> { - None - } - /// An empty cache for the canvas of this target. A target on a GPU that the backend /// shares with wgpu samples the wgpu layers of the frames in place. fn render_cache(&self) -> RenderCache { @@ -61,7 +44,6 @@ trait WindowTarget { pub(crate) struct Presenter { pub studio_layout: Option, render_cache: RenderCache, - snapshot_surface: Option, target: Box, window: Rc, background: Color, @@ -77,10 +59,8 @@ impl Presenter { pub(crate) fn new( window: Rc, config: &crate::app::AppConfig<'_>, - video_sources: bool, ) -> PlayerResult { - let raster_fps = (config.limit_prefetch_memory && !video_sources).then_some(config.fps); - let target = create_target(&window, config.backend, config.vsync, raster_fps)?; + let target = create_target(&window, config.backend, config.vsync)?; let duration_label = crate::controls::format_time(config.duration_in_frames, config.fps); let background = config.background; let model = &config.studio; @@ -96,7 +76,6 @@ impl Presenter { let mut presenter = Self { render_cache: target.render_cache(), - snapshot_surface: None, target, window, background: Color::from_argb(background.a, background.r, background.g, background.b), @@ -129,7 +108,6 @@ impl Presenter { return Ok(()); } - self.snapshot_surface = None; self.target.resize(size.width, size.height)?; let logical = size.to_logical::(self.window.scale_factor()); self.studio_layout = StudioLayout::new( @@ -149,66 +127,10 @@ impl Presenter { Ok(()) } - /// Draws just the scene. Controls and the timeline are composited when it is shown. - pub(crate) fn snapshot( - &mut self, - tree: &usvgr::Tree, - studio_open: bool, - ) -> Option { - let view = self.studio_layout.as_ref().filter(|_| studio_open); - let mut context = self.target.gpu_context(); - // Keep window coordinates while rendering filters and shaders. Only the - // video rectangle is retained in each finished frame. - if self.snapshot_surface.is_none() { - let size = self.window.inner_size(); - let info = ImageInfo::new_n32_premul((size.width as i32, size.height as i32), None); - self.snapshot_surface = match context.as_mut() { - Some((gpu, queue)) => { - let _lock = lock_queue(*queue); - gpu::surfaces::render_target( - gpu, - gpu::Budgeted::Yes, - &info, - None, - gpu::SurfaceOrigin::TopLeft, - None, - false, - None, - ) - } - None => surfaces::raster(&info, None, None), - }; - } - let surface = self.snapshot_surface.as_mut()?; - let canvas = surface.canvas(); - canvas.clear(matte(view)); - let rect = draw_video( - canvas, - tree, - view, - self.window.scale_factor() as f32, - self.background, - &mut self.render_cache, - ); - let bounds = IRect::intersect( - &rect.round_out(), - &IRect::from_size(canvas.base_layer_size()), - )?; - let image = surface.image_snapshot_with_bounds(bounds)?; - if let Some((gpu, queue)) = context { - let _lock = lock_queue(queue); - gpu.flush_and_submit_surface(surface, gpu::SyncCpu::Yes); - } - Some(BufferedFrame { - origin: IPoint::new(bounds.left, bounds.top), - image, - }) - } - /// Draws the current frame and overlays the controls before presenting the window. pub(crate) fn present( &mut self, - frame: Option<&PlaybackFrame>, + frame: Option<&usvgr::Tree>, controls: &ControlsState, model: &Studio, scroll: &StudioScroll, @@ -225,18 +147,11 @@ impl Presenter { .. } = self; - target.prepare_frame(frame); target.present(&mut |canvas| { let view = studio_layout.as_ref().filter(|_| studio_open); canvas.clear(matte(view)); - match frame { - Some(PlaybackFrame::Image(frame)) => { - canvas.draw_image(&frame.image, Point::from(frame.origin), None); - } - Some(PlaybackFrame::Tree(tree)) => { - draw_video(canvas, tree, view, pixel_scale, *background, render_cache); - } - None => {} + if let Some(tree) = frame { + draw_video(canvas, tree, view, pixel_scale, *background, render_cache); } let show_bar = view.is_none(); @@ -276,7 +191,7 @@ fn draw_video( pixel_scale: f32, background: Color, cache: &mut RenderCache, -) -> Rect { +) { let size = canvas.base_layer_size(); let viewport = view.map_or_else( || Rect::from_iwh(size.width, size.height), @@ -312,36 +227,32 @@ fn draw_video( canvas.draw_color(background, None); fframes_skia_renderer::render::render_tree(tree, canvas, cache); canvas.restore(); - Rect::from_xywh(x, y, width, height) } fn create_target( window: &Rc, backend: PlayerBackend, vsync: bool, - raster_fps: Option, ) -> PlayerResult> { - #[cfg(not(feature = "metal"))] - let _ = raster_fps; match backend { PlayerBackend::Auto => { #[cfg(feature = "metal")] - match metal::MetalTarget::new(window, vsync, raster_fps) { + match metal::MetalTarget::new(window, vsync) { Ok(target) => return Ok(Box::new(target)), Err(err) => eprintln!("fframes player: metal is not available ({err})"), } #[cfg(feature = "vulkan")] - match create_target(window, PlayerBackend::Vulkan, vsync, None) { + match create_target(window, PlayerBackend::Vulkan, vsync) { Ok(target) => return Ok(target), Err(err) => eprintln!("fframes player: vulkan is not available ({err})"), } eprintln!("fframes player: falling back to the Skia CPU renderer"); - create_target(window, PlayerBackend::Cpu, vsync, None) + create_target(window, PlayerBackend::Cpu, vsync) } #[cfg(feature = "metal")] - PlayerBackend::Metal => Ok(Box::new(metal::MetalTarget::new(window, vsync, None)?)), + PlayerBackend::Metal => Ok(Box::new(metal::MetalTarget::new(window, vsync)?)), #[cfg(feature = "vulkan")] PlayerBackend::Vulkan => Ok(Box::new(vulkan::VulkanTarget::new(window.clone(), vsync)?)), PlayerBackend::Cpu => Ok(Box::new(cpu::CpuTarget::new(window.clone())?)), diff --git a/fframes-native-player/src/presenter/vulkan.rs b/fframes-native-player/src/presenter/vulkan.rs index 03f70e26..73740b6a 100644 --- a/fframes-native-player/src/presenter/vulkan.rs +++ b/fframes-native-player/src/presenter/vulkan.rs @@ -369,15 +369,6 @@ impl VulkanTarget { } impl WindowTarget for VulkanTarget { - fn gpu_context( - &mut self, - ) -> Option<( - &mut DirectContext, - Option<&fframes_skia_renderer::QueueLock>, - )> { - Some((&mut self.gpu, Some(&self.queue_lock))) - } - fn is_gpu(&self) -> bool { true } diff --git a/fframes-native-player/src/scheduler.rs b/fframes-native-player/src/scheduler.rs index 7e5f0e04..a295e44f 100644 --- a/fframes-native-player/src/scheduler.rs +++ b/fframes-native-player/src/scheduler.rs @@ -7,22 +7,11 @@ use std::collections::BTreeMap; use std::sync::{Condvar, Mutex, MutexGuard}; -use crate::{app::FrameReady, presenter::BufferedFrame, studio::thumbnail}; +use crate::{app::FrameReady, studio::thumbnail}; use fframes::{FFramesContext, TextCache, Video, VideoDecodersWorker, usvgr}; pub(crate) const MAX_STAGED_TREES: usize = 4; -/// A tree can go straight to the window when its frame is due. Images were already -/// drawn at the window scale and stay on the thread that owns the Skia context. -#[expect( - clippy::large_enum_variant, - reason = "The window holds one current frame. Keeping its tree inline avoids a per-frame allocation." -)] -pub(crate) enum PlaybackFrame { - Tree(usvgr::Tree), - Image(BufferedFrame), -} - pub(crate) struct Scheduler { state: Mutex, workers_wakeup: Condvar, @@ -30,7 +19,6 @@ pub(crate) struct Scheduler { lookahead: u64, thumbnails: usize, playback_decoders: VideoDecodersWorker, - pub(crate) video_sources: bool, software_decode: bool, } @@ -72,7 +60,6 @@ impl Scheduler { lookahead: usize, looping: bool, thumbnails: usize, - video_sources: bool, software_decode: bool, ) -> Self { Self { @@ -92,7 +79,6 @@ impl Scheduler { lookahead: lookahead.min(duration_in_frames) as u64, thumbnails, playback_decoders: VideoDecodersWorker::for_preview(software_decode), - video_sources, software_decode, } } @@ -126,21 +112,17 @@ impl Scheduler { } /// A forward seek within the finished buffer can show its target immediately. - pub(crate) fn seek_buffered( - &self, - position: u64, - looping: bool, - images: &mut BTreeMap, - ) -> bool { + pub(crate) fn seek_buffered(&self, position: u64, looping: bool) -> bool { let mut state = self.lock(); - if state.looping != looping || position < state.playhead || !images.contains_key(&position) + if state.looping != looping + || position < state.playhead + || !state.ready.contains_key(&position) { return false; } state.playhead = position; state.ready.retain(|&p, _| p >= position); drop(state); - images.retain(|&p, _| p >= position); self.workers_wakeup.notify_all(); true } @@ -148,48 +130,23 @@ impl Scheduler { /// Moves the playhead and returns the newest ready frame at or before it. /// Frames that are late are still returned so heavy videos degrade to a lower /// frame rate instead of freezing. - pub(crate) fn take( - &self, - playhead: u64, - images: &mut BTreeMap, - ) -> Option<(u64, PlaybackFrame)> { - let image = take_due(images, playhead).map(|(p, image)| (p, PlaybackFrame::Image(image))); + pub(crate) fn take(&self, playhead: u64) -> Option<(u64, usvgr::Tree)> { let mut state = self.lock(); let advanced = state.playhead != playhead; - let first_frame = state.displayed.is_none(); state.playhead = playhead; - let tree = - take_due(&mut state.ready, playhead).map(|(p, tree)| (p, PlaybackFrame::Tree(tree))); - let taken = tree.into_iter().chain(image).max_by_key(|(p, _)| *p); + let taken = take_due(&mut state.ready, playhead); if let Some((position, _)) = &taken { state.displayed = Some(*position); } drop(state); - if advanced || (first_frame && taken.is_some()) { + if advanced || taken.is_some() { self.workers_wakeup.notify_all(); } taken } - /// Hands one future tree to the window for offscreen drawing. Leave any due - /// frame for `take`, even if it completed between the last tick and this call. - pub(crate) fn next_to_draw(&self) -> Option<(u64, usvgr::Tree)> { - let mut state = self.lock(); - if state.displayed.is_none() || *state.ready.first_key_value()?.0 <= state.playhead { - return None; - } - let next = state.ready.pop_first(); - drop(state); - self.workers_wakeup.notify_one(); - next - } - - pub(crate) fn put_back(&self, position: u64, tree: usvgr::Tree) { - self.lock().ready.insert(position, tree); - } - pub(crate) fn shutdown(&self) { self.lock().shutdown = true; self.workers_wakeup.notify_all(); @@ -215,20 +172,14 @@ impl Scheduler { fn claim_playback(&self, state: &mut State) -> Option { let position = state.next.max(state.playhead); - // Decoding workers can each hold full-size source frames before they return a tree. - let staged = state.ready.len() - + if self.video_sources { - state.in_flight - } else { - 0 - }; + let staged = state.ready.len() + state.in_flight; if position > state.playhead.saturating_add(self.lookahead) || (!state.looping && position >= self.duration) - || (position > state.playhead && staged >= MAX_STAGED_TREES) + || (position > state.playhead && staged >= self.lookahead.max(1) as usize) { return None; } - // Keep only a few decoded trees waiting for the window. A new live position + // Bound prepared trees and in-flight work. A new live position // can bypass that limit if the clock jumped ahead of all assigned work. state.next = position + 1; state.in_flight += 1; @@ -377,116 +328,114 @@ mod tests { #[test] fn taking_out_of_order_frames_preserves_future_frames_and_rejects_stale_jobs() { - let scheduler = Scheduler::new(100, 3, false, 2, false, false); + let scheduler = Scheduler::new(100, 4, false, 2, false); for index in 0..=3 { assert_eq!(scheduler.next_job().unwrap().frame, index); } for index in [2, 1] { assert!(scheduler.complete(index, 0, Some(tree()))); } - assert_eq!(scheduler.take(1, &mut BTreeMap::new()).unwrap().0, 1); - assert!(scheduler.take(1, &mut BTreeMap::new()).is_none()); + assert_eq!(scheduler.take(1).unwrap().0, 1); + assert!(scheduler.take(1).is_none()); assert!(!scheduler.complete(0, 0, Some(tree()))); - assert_eq!(scheduler.take(2, &mut BTreeMap::new()).unwrap().0, 2); + assert_eq!(scheduler.take(2).unwrap().0, 2); scheduler.seek(50, false); assert!(!scheduler.complete(3, 0, Some(tree()))); assert_eq!(scheduler.next_job().unwrap().frame, 50); assert!(scheduler.complete(50, 1, Some(tree()))); - assert_eq!(scheduler.take(50, &mut BTreeMap::new()).unwrap().0, 50); + assert_eq!(scheduler.take(50).unwrap().0, 50); scheduler.shutdown(); assert!(scheduler.next_job().is_none()); } #[test] - fn finished_frames_fill_the_window_and_a_seek_reuses_its_target() { - let scheduler = Scheduler::new(100, 48, false, 0, false, false); - let mut images = BTreeMap::new(); - let image = fframes_skia_renderer::skia_safe::surfaces::raster_n32_premul((8, 8)) - .unwrap() - .image_snapshot(); - assert_eq!(scheduler.next_job().unwrap().frame, 0); - scheduler.complete(0, 0, Some(tree())); - assert!(scheduler.next_to_draw().is_none()); - assert!(matches!( - scheduler.take(0, &mut images), - Some((0, PlaybackFrame::Tree(_))) - )); - - for p in 1..=48 { - assert_eq!(scheduler.next_job().unwrap().frame as u64, p); - scheduler.complete(p, 0, Some(tree())); - assert_eq!(scheduler.next_to_draw().unwrap().0, p); - images.insert( - p, - BufferedFrame { - image: image.clone(), - origin: fframes_skia_renderer::skia_safe::IPoint::default(), - }, - ); + fn prepared_frames_are_bounded_and_forward_seeks_reuse_trees() { + let scheduler = Scheduler::new(100, 4, false, 0, false); + for p in 0..4 { + assert_eq!(scheduler.next_job().unwrap().frame, p); + assert!(scheduler.complete(p as u64, 0, Some(tree()))); } assert!(scheduler.claim_playback(&mut scheduler.lock()).is_none()); - assert_eq!(images.len(), 48); - assert!(scheduler.seek_buffered(40, false, &mut images)); - assert!(matches!( - scheduler.take(40, &mut images), - Some((40, PlaybackFrame::Image(_))) - )); - assert_eq!(images.len(), 8); - assert!(!scheduler.seek_buffered(41, true, &mut images)); - assert!(!scheduler.seek_buffered(0, false, &mut images)); - assert_eq!(scheduler.next_job().unwrap().frame, 49); + assert_eq!(scheduler.take(0).unwrap().0, 0); + assert_eq!(scheduler.next_job().unwrap().frame, 4); + scheduler.complete(4, 0, Some(tree())); + assert!(scheduler.seek_buffered(3, false)); + assert_eq!(scheduler.take(3).unwrap().0, 3); + assert!(!scheduler.seek_buffered(4, true)); + assert!(!scheduler.seek_buffered(0, false)); + assert_eq!(scheduler.take(4).unwrap().0, 4); } #[test] fn live_frames_take_priority_over_queued_trees() { - let scheduler = Scheduler::new(100, 48, false, 0, false, false); - let mut images = BTreeMap::new(); + let scheduler = Scheduler::new(100, 4, false, 0, false); for p in 0..4 { assert_eq!(scheduler.next_job().unwrap().frame as u64, p); scheduler.complete(p, 0, Some(tree())); } assert!(scheduler.claim_playback(&mut scheduler.lock()).is_none()); - assert!(matches!( - scheduler.take(30, &mut images), - Some((3, PlaybackFrame::Tree(_))) - )); + assert_eq!(scheduler.take(30).unwrap().0, 3); assert_eq!(scheduler.next_job().unwrap().frame, 30); scheduler.complete(30, 0, Some(tree())); - assert!(scheduler.next_to_draw().is_none()); - assert!(matches!( - scheduler.take(30, &mut images), - Some((30, PlaybackFrame::Tree(_))) - )); + assert_eq!(scheduler.take(30).unwrap().0, 30); } #[test] - fn video_workers_bound_in_flight_frames_and_release_failed_jobs() { - let scheduler = Scheduler::new(100, 48, false, 0, true, false); + fn workers_bound_in_flight_frames_and_release_failed_jobs() { + let scheduler = Scheduler::new(100, 4, false, 0, false); for p in 0..4 { assert_eq!(scheduler.next_job().unwrap().frame, p); } assert!(scheduler.claim_playback(&mut scheduler.lock()).is_none()); scheduler.complete(2, 0, None); assert_eq!(scheduler.next_job().unwrap().frame, 4); - scheduler.take(30, &mut BTreeMap::new()); + scheduler.take(30); assert_eq!(scheduler.next_job().unwrap().frame, 30); scheduler.seek(60, false); assert!(!scheduler.complete(1, 0, Some(tree()))); assert_eq!(scheduler.next_job().unwrap().frame, 60); } + #[test] + fn zero_prefetch_only_prepares_the_live_frame() { + let scheduler = Scheduler::new(3, 0, false, 0, false); + for p in 0..3 { + scheduler.take(p); + assert_eq!(scheduler.next_job().unwrap().frame as u64, p); + assert!(scheduler.claim_playback(&mut scheduler.lock()).is_none()); + assert!(scheduler.complete(p, 0, Some(tree()))); + assert_eq!(scheduler.take(p).unwrap().0, p); + } + assert!(scheduler.claim_playback(&mut scheduler.lock()).is_none()); + } + + #[test] + fn looping_buffer_uses_monotonic_positions_across_the_end() { + let scheduler = Scheduler::new(3, 2, true, 0, false); + scheduler.seek(2, true); + for p in 2..4 { + let job = scheduler.next_job().unwrap(); + assert_eq!(job.frame, p % 3); + assert!(scheduler.complete(p as u64, 1, Some(tree()))); + } + assert_eq!(scheduler.take(2).unwrap().0, 2); + assert!(scheduler.seek_buffered(3, true)); + assert_eq!(scheduler.take(3).unwrap().0, 3); + assert_eq!(scheduler.next_job().unwrap().frame, 1); + } + #[test] fn thumbnails_do_not_consume_playback_jobs_and_stop_when_finished() { - let scheduler = Scheduler::new(100, 3, false, 2, false, false); + let scheduler = Scheduler::new(100, 3, false, 2, false); assert_eq!(scheduler.next_job().unwrap().frame, 0); scheduler.complete(0, 0, Some(tree())); - scheduler.take(0, &mut BTreeMap::new()).unwrap(); + scheduler.take(0).unwrap(); assert_eq!(scheduler.next_thumbnail_job().unwrap().frame, 25); assert_eq!(scheduler.next_job().unwrap().frame, 1); assert_eq!(scheduler.next_thumbnail_job().unwrap().frame, 75); assert!(scheduler.next_thumbnail_job().is_none()); assert_eq!(scheduler.next_job().unwrap().frame, 2); - let stopped = Scheduler::new(100, 3, false, 2, false, false); + let stopped = Scheduler::new(100, 3, false, 2, false); stopped.shutdown(); assert!(stopped.next_thumbnail_job().is_none()); } From a43b6317c8ece55c65a4833b8a136d98b88352cb Mon Sep 17 00:00:00 2001 From: Dmitriy Kovalenko Date: Sat, 31 Oct 2026 10:42:48 -0700 Subject: [PATCH 15/21] perf(media): Reuse the pixel buffers of released video frames Evicted images stay in a retired list while a scene tree still holds them. The next conversion takes the buffer of the first retired image that nothing else holds, so playback does not allocate a frame-sized buffer per frame. --- fframes-media/src/video_decoder.rs | 47 ++++++++++++++++++------ fframes-media/tests/video_decoder_eof.rs | 26 +++++++++++++ 2 files changed, 62 insertions(+), 11 deletions(-) diff --git a/fframes-media/src/video_decoder.rs b/fframes-media/src/video_decoder.rs index d276b420..d0e7658d 100644 --- a/fframes-media/src/video_decoder.rs +++ b/fframes-media/src/video_decoder.rs @@ -411,10 +411,41 @@ struct ScaledFrameImage { image: Arc, } +/// Evicted images kept for buffer reuse, see `ConvertedFrames::retired`. +const MAX_RETIRED_IMAGES: usize = 8; + #[derive(Debug)] struct ConvertedFrames { scaler: SwsScaler, images: VecDeque, + /// Images evicted from the ring that a scene tree or a renderer still holds. Their + /// pixel buffers are reused once the last holder drops them, so playback does not + /// allocate a frame-sized buffer per frame. + retired: Vec>, +} + +impl ConvertedFrames { + /// A pixel buffer no other holder can see, from the ring or the retired images. + fn free_pixels(&mut self) -> Vec { + if self.images.len() == self.images.capacity() + && let Some(evicted) = self.images.pop_front() + { + if self.retired.len() == MAX_RETIRED_IMAGES { + self.retired.remove(0); + } + self.retired.push(evicted.image); + } + let Some(free) = self + .retired + .iter() + .position(|image| Arc::strong_count(image) == 1) + else { + return Vec::new(); + }; + Arc::try_unwrap(self.retired.swap_remove(free)) + .map(|image| image.data.into_owned()) + .unwrap_or_default() + } } #[derive(Debug)] @@ -465,6 +496,7 @@ impl FFmpegFrameBuf { converted: Arc::new(Mutex::new(ConvertedFrames { scaler: sws_ctx, images: VecDeque::with_capacity(buf_size.max(1)), + retired: Vec::with_capacity(MAX_RETIRED_IMAGES), })), }) } @@ -496,22 +528,15 @@ impl FFmpegFrameBuf { let pts = unsafe { (*self.latest_av_frame).pts }; let options = options.copied(); let mut converted = self.converted.lock().unwrap(); - let ConvertedFrames { scaler, images } = &mut *converted; - if let Some(last) = images.back() + if let Some(last) = converted.images.back() && last.pts == pts && last.options == options { return Ok(Arc::clone(&last.image)); } - // A queued frame may still own the evicted image. - let mut pixels = if images.len() == images.capacity() { - Arc::try_unwrap(images.pop_front().unwrap().image) - .map(|image| image.data.into_owned()) - .unwrap_or_default() - } else { - Vec::new() - }; + let mut pixels = converted.free_pixels(); + let scaler = &mut converted.scaler; unsafe { scaler.convert(options, self.latest_av_frame, &mut pixels)? }; if scaler.source_has_alpha { premultiply_alpha(&mut pixels); @@ -522,7 +547,7 @@ impl FFmpegFrameBuf { height: scaler.height as u32, data: Cow::Owned(pixels), }); - images.push_back(ScaledFrameImage { + converted.images.push_back(ScaledFrameImage { pts, options, image: Arc::clone(&image), diff --git a/fframes-media/tests/video_decoder_eof.rs b/fframes-media/tests/video_decoder_eof.rs index 9bd1a61d..cf8100a3 100644 --- a/fframes-media/tests/video_decoder_eof.rs +++ b/fframes-media/tests/video_decoder_eof.rs @@ -227,6 +227,32 @@ fn converted_images_survive_single_buffer_reuse_and_decoder_drop() { } } +#[test] +fn released_frames_lend_their_pixel_buffers_to_later_conversions() { + let video = VideoFixture::new(0); + let mut decoder = unsafe { FFmpegDecoder::new(&video.path(), FPS, 1) }.unwrap(); + let mut convert = |index| { + assert!(unsafe { decoder.decode_up_to(index) }.unwrap()); + unsafe { + decoder + .get_raw_frame() + .convert_last_decoded_frame_into_svg_image(None) + } + .unwrap() + }; + let first = convert(0); + let first_pixels = first.data.as_ptr(); + // The first image is still held, so the second one needs its own buffer. + let second = convert(30); + assert_ne!(second.data.as_ptr(), first_pixels); + drop(first); + // Once released, the buffer of the first image is reused. + let third = convert(60); + assert_eq!(third.data.as_ptr(), first_pixels); + assert_ne!(third.data.as_ref(), second.data.as_ref()); + assert_eq!(third.data.len(), 32 * 32 * 4); +} + #[test] fn a_frame_requested_again_at_another_size_is_converted_to_that_size() { let video = VideoFixture::new(0); From fa02805e0dae40d6e0a1a3745115b2a3a5c48a4e Mon Sep 17 00:00:00 2001 From: Dmitriy Kovalenko Date: Sat, 31 Oct 2026 10:42:48 -0700 Subject: [PATCH 16/21] perf(player): Prepare only the pointer frame while scrubbing A timeline drag turns lookahead off until the button is released. The filmstrip worker waits until the playback buffer is full and no drag is in progress. Seeks drop stale trees after releasing the scheduler lock. --- README.md | 5 +- fframes-native-player/src/app.rs | 37 +++++-- fframes-native-player/src/scheduler.rs | 127 ++++++++++++++++++++----- fframes/src/media_provider.rs | 3 + 4 files changed, 137 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index ff2bfb1f..54f25130 100644 --- a/README.md +++ b/README.md @@ -348,8 +348,9 @@ On macOS the studio tries VideoToolbox for supported source videos and falls bac for unsupported formats. `preview --software-decode` uses the CPU for decoding while keeping the selected drawing backend. Playback and filmstrip workers release idle source decoders. The GPU preview draws each scene through Skia directly into the window drawable or swapchain. -The default buffer holds up to four prepared scene trees. Source video frames remain attached -to their trees until drawing finishes. The CPU backend draws directly into the window buffer. +The default buffer holds up to four prepared scene trees, and only the frame under the pointer +is prepared while you drag the timeline. Decoders reuse the pixel buffers of frames that no +scene tree holds any more. The CPU backend draws directly into the window buffer. `cargo fframes new` takes `--template single-scene|multi-scene`, `--format landscape|portrait|square|uhd`, `--fps`, `--title` and `--backend`. On macOS and Linux (arm64, x86_64) the first build downloads prebuilt Skia and ffmpeg diff --git a/fframes-native-player/src/app.rs b/fframes-native-player/src/app.rs index bf0ac97d..977369fb 100644 --- a/fframes-native-player/src/app.rs +++ b/fframes-native-player/src/app.rs @@ -86,7 +86,7 @@ pub(crate) struct App<'s> { audio: Option<&'s crate::audio::AudioOutput>, window: Option>, - // Keep source frames alive until Skia finishes drawing them. + /// The tree on screen. Skia copies its video frames to the GPU during the flush. current: Option<(u64, fframes::usvgr::Tree)>, presenter: Option, redraw_pending: bool, @@ -392,10 +392,18 @@ impl<'s> App<'s> { self.needs_redraw = true; } + /// A drag along the timeline or the seek slider turns lookahead off: the workers + /// prepare only the frame under the pointer until the button is released. + fn set_drag(&mut self, drag: Option) { + self.drag = drag; + self.scheduler + .set_scrubbing(matches!(drag, Some(Control::Seek | Control::Timeline))); + } + fn toggle_studio(&mut self) { self.invalidate_view(); self.studio_open = !self.studio_open; - self.drag = None; + self.set_drag(None); self.hovered = None; self.last_pointer_activity = Instant::now(); self.needs_redraw = true; @@ -489,7 +497,7 @@ impl<'s> App<'s> { (ui.time_axis(self.pointer()) - self.scene_press_axis).abs() >= 3. }) { - self.drag = Some(Control::Timeline); + self.set_drag(Some(Control::Timeline)); } if matches!(self.drag, Some(Control::Seek | Control::Timeline)) { self.seek_to_cursor(); @@ -534,14 +542,14 @@ impl<'s> App<'s> { Some(Control::Back) => self.seek_to_frame(frame - second), Some(Control::Forward) => self.seek_to_frame(frame + second), Some(control @ (Control::Seek | Control::Timeline)) => { - self.drag = Some(control); + self.set_drag(Some(control)); self.seek_to_cursor(); } Some(control @ Control::Scene(_)) => { if let Some(ui) = self.studio_layout() { self.scene_press_axis = ui.time_axis(self.pointer()); } - self.drag = Some(control); + self.set_drag(Some(control)); } Some(Control::Focus) => self.toggle_studio(), Some(Control::Loop) => { @@ -558,7 +566,9 @@ impl<'s> App<'s> { } fn on_pointer_released(&mut self) { - if let Some(control @ Control::Scene(start)) = self.drag.take() + let drag = self.drag; + self.set_drag(None); + if let Some(control @ Control::Scene(start)) = drag && self.control_under_cursor() == Some(control) { self.step(start as i64); @@ -687,7 +697,7 @@ impl ApplicationHandler for App<'_> { self.needs_redraw = true; } } - WindowEvent::Focused(false) => self.drag = None, + WindowEvent::Focused(false) => self.set_drag(None), WindowEvent::CursorLeft { .. } => { self.cursor_inside = false; if self.hovered.take().is_some() { @@ -792,4 +802,17 @@ mod tests { assert_eq!(app.scrub_target, None); assert!(scheduler.is_current_seek(20, 3)); } + + #[test] + fn a_timeline_drag_turns_lookahead_off_until_the_button_is_released() { + let scheduler = Scheduler::new(100, 3, false, 0, false); + let mut app = paused_app(&scheduler); + app.set_drag(Some(Control::Scene(0))); + assert!(!scheduler.is_scrubbing()); + app.set_drag(Some(Control::Timeline)); + assert!(scheduler.is_scrubbing()); + app.on_pointer_released(); + assert!(!scheduler.is_scrubbing()); + assert_eq!(app.drag, None); + } } diff --git a/fframes-native-player/src/scheduler.rs b/fframes-native-player/src/scheduler.rs index a295e44f..07f7655f 100644 --- a/fframes-native-player/src/scheduler.rs +++ b/fframes-native-player/src/scheduler.rs @@ -32,6 +32,8 @@ struct State { /// Last position handed to the player, anything at or before it is stale. displayed: Option, looping: bool, + /// A timeline drag is in progress: prepare only the frame under the pointer. + scrubbing: bool, ready: BTreeMap, in_flight: usize, shutdown: bool, @@ -69,6 +71,7 @@ impl Scheduler { playhead: 0, displayed: None, looping, + scrubbing: false, ready: BTreeMap::new(), in_flight: 0, shutdown: false, @@ -106,9 +109,11 @@ impl Scheduler { state.playhead = position; state.displayed = None; state.looping = looping; - state.ready.clear(); + // Trees hold video frames. Free them after the workers can continue. + let stale = std::mem::take(&mut state.ready); drop(state); self.workers_wakeup.notify_all(); + drop(stale); } /// A forward seek within the finished buffer can show its target immediately. @@ -121,12 +126,26 @@ impl Scheduler { return false; } state.playhead = position; - state.ready.retain(|&p, _| p >= position); + let kept = state.ready.split_off(&position); + let stale = std::mem::replace(&mut state.ready, kept); drop(state); self.workers_wakeup.notify_all(); + drop(stale); true } + /// While a drag is in progress only the frame under the pointer is prepared, so a + /// scrub never waits for lookahead frames the next pointer position discards. + pub(crate) fn set_scrubbing(&self, scrubbing: bool) { + let mut state = self.lock(); + if state.scrubbing == scrubbing { + return; + } + state.scrubbing = scrubbing; + drop(state); + self.workers_wakeup.notify_all(); + } + /// Moves the playhead and returns the newest ready frame at or before it. /// Frames that are late are still returned so heavy videos degrade to a lower /// frame rate instead of freezing. @@ -135,7 +154,7 @@ impl Scheduler { let advanced = state.playhead != playhead; state.playhead = playhead; - let taken = take_due(&mut state.ready, playhead); + let (taken, skipped) = take_due(&mut state.ready, playhead); if let Some((position, _)) = &taken { state.displayed = Some(*position); } @@ -144,6 +163,7 @@ impl Scheduler { if advanced || taken.is_some() { self.workers_wakeup.notify_all(); } + drop(skipped); taken } @@ -170,17 +190,24 @@ impl Scheduler { } } - fn claim_playback(&self, state: &mut State) -> Option { + /// The next position a playback worker can prepare, if any. Prepared trees and + /// in-flight work are bounded. A new live position bypasses that bound when the + /// clock jumped ahead of all assigned work. + fn claimable_position(&self, state: &State) -> Option { + let lookahead = if state.scrubbing { 0 } else { self.lookahead }; let position = state.next.max(state.playhead); let staged = state.ready.len() + state.in_flight; - if position > state.playhead.saturating_add(self.lookahead) + if position > state.playhead.saturating_add(lookahead) || (!state.looping && position >= self.duration) - || (position > state.playhead && staged >= self.lookahead.max(1) as usize) + || (position > state.playhead && staged >= lookahead.max(1) as usize) { return None; } - // Bound prepared trees and in-flight work. A new live position - // can bypass that limit if the clock jumped ahead of all assigned work. + Some(position) + } + + fn claim_playback(&self, state: &mut State) -> Option { + let position = self.claimable_position(state)?; state.next = position + 1; state.in_flight += 1; Some(Job { @@ -192,10 +219,16 @@ impl Scheduler { }) } - // The filmstrip seeks across the video with its own decoder. + // The filmstrip seeks across the video with its own decoder. It waits until the + // first frame is shown, no drag is in progress and the playback workers have nothing + // left to prepare, so thumbnails never compete with playback or a scrub. fn next_thumbnail_job(&self) -> Option { let mut state = self.lock(); - while !state.shutdown && state.displayed.is_none() { + while !state.shutdown + && (state.displayed.is_none() + || state.scrubbing + || self.claimable_position(&state).is_some()) + { state = self .workers_wakeup .wait(state) @@ -214,6 +247,11 @@ impl Scheduler { }) } + #[cfg(test)] + pub(crate) fn is_scrubbing(&self) -> bool { + self.lock().scrubbing + } + pub(crate) fn is_current_seek(&self, position: u64, epoch: u64) -> bool { let state = self.lock(); state.playhead == position && state.epoch == epoch @@ -227,13 +265,15 @@ impl Scheduler { .displayed .is_some_and(|displayed| position <= displayed); - if !is_stale && let Some(tree) = tree { + let tree = tree.filter(|_| !is_stale); + let inserted = tree.is_some(); + if let Some(tree) = tree { state.ready.insert(position, tree); - return true; } drop(state); - self.workers_wakeup.notify_one(); - false + // A full buffer lets the filmstrip worker start, a freed slot the playback workers. + self.workers_wakeup.notify_all(); + inserted } pub(crate) fn run_worker<'a, 'media: 'a, TVideo: Video + Sync>( @@ -305,12 +345,16 @@ impl Scheduler { } } -fn take_due(ready: &mut BTreeMap, playhead: u64) -> Option<(u64, T)> { - let mut latest = None; - while ready.first_key_value().is_some_and(|(&p, _)| p <= playhead) { - latest = ready.pop_first(); - } - latest +/// Removes the frames at or before `playhead`. Returns the newest of them and the ones +/// it skipped, so the caller can drop those after releasing the lock. +fn take_due( + ready: &mut BTreeMap, + playhead: u64, +) -> (Option<(u64, T)>, BTreeMap) { + let future = ready.split_off(&playhead.saturating_add(1)); + let mut due = std::mem::replace(ready, future); + let latest = due.pop_last(); + (latest, due) } #[cfg(test)] @@ -425,16 +469,47 @@ mod tests { } #[test] - fn thumbnails_do_not_consume_playback_jobs_and_stop_when_finished() { + fn scrubbing_prepares_only_the_frame_under_the_pointer() { + let scheduler = Scheduler::new(100, 4, false, 0, false); + scheduler.set_scrubbing(true); + scheduler.seek(10, false); + assert_eq!(scheduler.next_job().unwrap().frame, 10); + assert!(scheduler.claim_playback(&mut scheduler.lock()).is_none()); + assert!(scheduler.complete(10, 1, Some(tree()))); + assert_eq!(scheduler.take(10).unwrap().0, 10); + assert!(scheduler.claim_playback(&mut scheduler.lock()).is_none()); + scheduler.set_scrubbing(false); + assert_eq!(scheduler.next_job().unwrap().frame, 11); + } + + #[test] + fn thumbnails_wait_for_a_full_playback_buffer_and_stop_when_finished() { let scheduler = Scheduler::new(100, 3, false, 2, false); assert_eq!(scheduler.next_job().unwrap().frame, 0); scheduler.complete(0, 0, Some(tree())); scheduler.take(0).unwrap(); - assert_eq!(scheduler.next_thumbnail_job().unwrap().frame, 25); - assert_eq!(scheduler.next_job().unwrap().frame, 1); - assert_eq!(scheduler.next_thumbnail_job().unwrap().frame, 75); - assert!(scheduler.next_thumbnail_job().is_none()); - assert_eq!(scheduler.next_job().unwrap().frame, 2); + let (sender, thumbnails) = std::sync::mpsc::channel(); + let scheduler = &scheduler; + std::thread::scope(|scope| { + // The channel closes when the filmstrip worker runs out of jobs. + scope.spawn(move || { + while let Some(job) = scheduler.next_thumbnail_job() { + sender.send(job.frame).unwrap(); + } + }); + // Playback jobs come first while the buffer has room. + for p in 1..=3 { + assert_eq!(scheduler.next_job().unwrap().frame, p); + if p < 3 { + assert!(thumbnails.try_recv().is_err()); + } + assert!(scheduler.complete(p as u64, 0, Some(tree()))); + } + assert_eq!(thumbnails.recv().unwrap(), 25); + assert_eq!(thumbnails.recv().unwrap(), 75); + assert!(thumbnails.recv().is_err()); + }); + assert!(scheduler.claim_playback(&mut scheduler.lock()).is_none()); let stopped = Scheduler::new(100, 3, false, 2, false); stopped.shutdown(); assert!(stopped.next_thumbnail_job().is_none()); diff --git a/fframes/src/media_provider.rs b/fframes/src/media_provider.rs index d60d0e2a..0216a440 100644 --- a/fframes/src/media_provider.rs +++ b/fframes/src/media_provider.rs @@ -53,6 +53,9 @@ pub trait MediaProvider<'a>: Send + Sync + Debug { } /// Whether the provider lists video files that need a decoder during playback. + /// A provider that can return `Some` from `resolve_video` must override this method + /// or `get_all_video_data`. Otherwise the native preview uses its no-video defaults: + /// more workers, and decoders that are not shared between them. fn has_video_sources(&self) -> bool { !self.get_all_video_data().is_empty() } From 2faf74d930a21961f3db59492789cd82544e056e Mon Sep 17 00:00:00 2001 From: Dmitriy Kovalenko Date: Sat, 31 Oct 2026 10:42:48 -0700 Subject: [PATCH 17/21] perf(player): Feed the audio callback from a lock-free ring The callback reads a single-producer single-consumer ring and two atomics. Samples carry the epoch they were mixed for, so a seek discards them without a lock. --- Cargo.lock | 7 ++ fframes-native-player/Cargo.toml | 3 +- fframes-native-player/src/audio.rs | 138 +++++++++++++++++++---------- 3 files changed, 99 insertions(+), 49 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 98da069a..e9a46c06 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1294,6 +1294,7 @@ dependencies = [ "fframes_skia_renderer", "hello-world-example", "raw-window-metal", + "rtrb", "softbuffer", "winit", ] @@ -3419,6 +3420,12 @@ version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c20b6793b5c2fa6553b250154b78d6d0db37e72700ae35fad9387a46f487c97" +[[package]] +name = "rtrb" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5974e397833d890f2f74963408e0d4a6cd8eda2f6a445ae06750291b5d7a997e" + [[package]] name = "rust-fuzzy-search" version = "0.1.1" diff --git a/fframes-native-player/Cargo.toml b/fframes-native-player/Cargo.toml index b8016315..b691bc13 100644 --- a/fframes-native-player/Cargo.toml +++ b/fframes-native-player/Cargo.toml @@ -17,6 +17,7 @@ fframes_skia_renderer.workspace = true winit = "0.30" softbuffer = "0.4" cpal = { version = "0.18", optional = true } +rtrb = { version = "0.4.1", optional = true } # The CAMetalLayer of the window, presented to by Metal and by Vulkan (MoltenVK). [target.'cfg(target_os = "macos")'.dependencies] @@ -30,7 +31,7 @@ hello-world-example = { path = "../examples/hello-world" } [features] default = ["vulkan", "audio"] -audio = ["cpal"] +audio = ["cpal", "dep:rtrb"] # At least one GPU backend is required to build Skia with Ganesh. The player falls back # to the Skia CPU raster backend at runtime if the GPU backend can not be initialized. vulkan = ["fframes_skia_renderer/vulkan", "dep:raw-window-metal"] diff --git a/fframes-native-player/src/audio.rs b/fframes-native-player/src/audio.rs index 1013bccd..979a5198 100644 --- a/fframes-native-player/src/audio.rs +++ b/fframes-native-player/src/audio.rs @@ -4,7 +4,8 @@ use fframes::{ AudioMixOptions, AudioMixer, AudioTimelineSamples, AudioTimelineUnit, FFramesContext, ResolvedAudioMap, }; -use std::collections::VecDeque; +use rtrb::{Consumer, Producer, RingBuffer}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, Condvar, Mutex, MutexGuard}; /// Samples mixed per step, same order of magnitude as an encoder audio frame. @@ -14,30 +15,40 @@ pub(crate) struct AudioOutput { // Not `Send` on every platform, lives on the main thread. _stream: cpal::Stream, shared: Arc, + /// The write end of the sample ring, used by the feeder thread only. + producer: Mutex>, pub(crate) sample_rate: usize, } +/// A mixed stereo sample, tagged with the playback it was mixed for. +#[derive(Clone, Copy)] +struct Sample { + left: f32, + right: f32, + epoch: u64, +} + struct Shared { - queue: Mutex, + control: Mutex, feeder_wakeup: Condvar, + /// Read by the audio callback without a lock. Both change under `control`. + playing: AtomicBool, + /// Bumped on every seek. The callback discards samples of earlier epochs. + epoch: AtomicU64, /// How many mixed samples to keep queued (~100ms). target_len: usize, } -struct Queue { - /// Stereo frames `(left, right)`. - samples: VecDeque<(f32, f32)>, - playing: bool, +struct Control { looping: bool, /// Next position to mix, in samples since the start of the (possibly looped) playback. feed_position: u64, - epoch: u64, shutdown: bool, } impl Shared { - fn lock(&self) -> MutexGuard<'_, Queue> { - self.queue + fn lock(&self) -> MutexGuard<'_, Control> { + self.control .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) } @@ -52,26 +63,37 @@ impl AudioOutput { let sample_rate = config.sample_rate() as usize; let channels = config.channels() as usize; let sample_format = config.sample_format(); + let target_len = sample_rate / 10; let shared = Arc::new(Shared { - queue: Mutex::new(Queue { - samples: VecDeque::new(), - playing: false, + control: Mutex::new(Control { looping: false, feed_position: 0, - epoch: 0, shutdown: false, }), feeder_wakeup: Condvar::new(), - target_len: sample_rate / 10, + playing: AtomicBool::new(false), + epoch: AtomicU64::new(0), + target_len, }); + // Room for the samples of the current playback plus the ones of the playback + // before a seek, which the callback discards. + let (producer, consumer) = RingBuffer::new(2 * target_len + CHUNK); let config = config.config(); let stream = match sample_format { - cpal::SampleFormat::F32 => build_stream::(&device, config, channels, &shared), - cpal::SampleFormat::I16 => build_stream::(&device, config, channels, &shared), - cpal::SampleFormat::U16 => build_stream::(&device, config, channels, &shared), - cpal::SampleFormat::I32 => build_stream::(&device, config, channels, &shared), + cpal::SampleFormat::F32 => { + build_stream::(&device, config, channels, &shared, consumer) + } + cpal::SampleFormat::I16 => { + build_stream::(&device, config, channels, &shared, consumer) + } + cpal::SampleFormat::U16 => { + build_stream::(&device, config, channels, &shared, consumer) + } + cpal::SampleFormat::I32 => { + build_stream::(&device, config, channels, &shared, consumer) + } format => return Err(format!("unsupported sample format {format}")), }?; stream.play().map_err(|e| e.to_string())?; @@ -79,19 +101,19 @@ impl AudioOutput { Ok(Self { _stream: stream, shared, + producer: Mutex::new(producer), sample_rate, }) } /// Restarts the audio at `position` (in frames of the player position space). pub(crate) fn reset(&self, position: u64, fps: usize, playing: bool, looping: bool) { - let mut queue = self.shared.lock(); - queue.epoch += 1; - queue.samples.clear(); - queue.feed_position = position * self.sample_rate as u64 / fps as u64; - queue.playing = playing; - queue.looping = looping; - drop(queue); + let mut control = self.shared.lock(); + control.feed_position = position * self.sample_rate as u64 / fps as u64; + control.looping = looping; + self.shared.epoch.fetch_add(1, Ordering::AcqRel); + self.shared.playing.store(playing, Ordering::Release); + drop(control); self.shared.feeder_wakeup.notify_all(); } @@ -100,7 +122,7 @@ impl AudioOutput { self.shared.feeder_wakeup.notify_all(); } - /// Mixes audio into the queue until `shutdown`. Runs on a scoped thread. + /// Mixes audio into the ring until `shutdown`. Runs on a scoped thread. pub(crate) fn run_feeder( &self, ctx: &FFramesContext<'_, '_>, @@ -125,25 +147,38 @@ impl AudioOutput { } let mut left = vec![0.; CHUNK]; let mut right = vec![0.; CHUNK]; + let mut producer = self + .producer + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let capacity = producer.buffer().capacity(); loop { let (position, looping, epoch) = { - let mut queue = self.shared.lock(); + let mut control = self.shared.lock(); loop { - if queue.shutdown { + if control.shutdown { return; } - let has_more = queue.looping || queue.feed_position < total_samples; - if queue.playing && has_more && queue.samples.len() < self.shared.target_len { + let has_more = control.looping || control.feed_position < total_samples; + let queued = capacity - producer.slots(); + if self.shared.playing.load(Ordering::Acquire) + && has_more + && queued < self.shared.target_len + { break; } - queue = self + control = self .shared .feeder_wakeup - .wait(queue) + .wait(control) .unwrap_or_else(std::sync::PoisonError::into_inner); } - (queue.feed_position, queue.looping, queue.epoch) + ( + control.feed_position, + control.looping, + self.shared.epoch.load(Ordering::Acquire), + ) }; let timeline_sample = if looping { @@ -161,15 +196,16 @@ impl AudioOutput { &mut right[..len], ); - let mut queue = self.shared.lock(); - if queue.epoch == epoch { - queue.samples.extend( - left[..len] - .iter() - .copied() - .zip(right[..len].iter().copied()), - ); - queue.feed_position += len as u64; + let mut control = self.shared.lock(); + // The wait above leaves at least a chunk of room in the ring. + if self.shared.epoch.load(Ordering::Acquire) == epoch + && let Ok(chunk) = producer.write_chunk_uninit(len) + { + let samples = left[..len] + .iter() + .zip(&right[..len]) + .map(|(&left, &right)| Sample { left, right, epoch }); + control.feed_position += chunk.fill_from_iter(samples) as u64; } } } @@ -180,19 +216,27 @@ fn build_stream>( config: cpal::StreamConfig, channels: usize, shared: &Arc, + mut consumer: Consumer, ) -> Result { let shared = Arc::clone(shared); device .build_output_stream( config, + // The callback never blocks: it reads the ring and two atomics. move |data: &mut [T], _: &cpal::OutputCallbackInfo| { - let mut queue = shared.lock(); - let playing = queue.playing; + let epoch = shared.epoch.load(Ordering::Acquire); + let playing = shared.playing.load(Ordering::Acquire); for frame in data.chunks_mut(channels) { + // Samples mixed before the last seek are discarded, also while paused. + while consumer.peek().is_ok_and(|sample| sample.epoch != epoch) { + let _ = consumer.pop(); + } let (left, right) = if playing { - queue.samples.pop_front().unwrap_or((0.0, 0.0)) + consumer + .pop() + .map_or((0.0, 0.0), |sample| (sample.left, sample.right)) } else { (0.0, 0.0) }; @@ -208,9 +252,7 @@ fn build_stream>( } } - let needs_more = playing && queue.samples.len() < shared.target_len; - drop(queue); - if needs_more { + if playing && consumer.slots() < shared.target_len { shared.feeder_wakeup.notify_one(); } }, From dfda5234e8c5ab41c7bee698b23b569106bcb093 Mon Sep 17 00:00:00 2001 From: Dmitriy Kovalenko Date: Sat, 31 Oct 2026 10:48:41 -0700 Subject: [PATCH 18/21] perf(player): Present without waiting for the GPU Skia copies image pixels into staging buffers during the flush, so the trees can be dropped before the GPU is done. Metal submits without a CPU sync. Vulkan signals a semaphore per swapchain image and lets the present wait for it. The Vulkan target also releases Skia resources unused for a second. --- fframes-native-player/src/presenter/metal.rs | 5 +- fframes-native-player/src/presenter/mod.rs | 6 +- fframes-native-player/src/presenter/vulkan.rs | 86 ++++++++++++++----- 3 files changed, 70 insertions(+), 27 deletions(-) diff --git a/fframes-native-player/src/presenter/metal.rs b/fframes-native-player/src/presenter/metal.rs index fedc8141..c97a1589 100644 --- a/fframes-native-player/src/presenter/metal.rs +++ b/fframes-native-player/src/presenter/metal.rs @@ -104,8 +104,9 @@ impl WindowTarget for MetalTarget { .ok_or_else(|| PlayerError::Skia("can not draw into the drawable".to_owned()))?; draw(surface.canvas()); - self.gpu - .flush_and_submit_surface(&mut surface, SyncCpu::Yes); + // The flush copies image pixels into transfer buffers, so the CPU does not + // wait for the GPU. The layer limits the drawables in flight. + self.gpu.flush_and_submit_surface(&mut surface, SyncCpu::No); drop(surface); // Skia submitted to the same queue, the drawable is presented after its work. diff --git a/fframes-native-player/src/presenter/mod.rs b/fframes-native-player/src/presenter/mod.rs index a215e4b7..6cf45092 100644 --- a/fframes-native-player/src/presenter/mod.rs +++ b/fframes-native-player/src/presenter/mod.rs @@ -33,9 +33,9 @@ trait WindowTarget { /// The window has a new size in physical pixels, neither is zero. fn resize(&mut self, width: u32, height: u32) -> PlayerResult<()>; - /// Draws the next image of the window with `draw` and shows it. Returns once the GPU is - /// done: video frame images point into the pixel buffers of the tree, which is dropped - /// as soon as the next frame replaces it. + /// Draws the next image of the window with `draw` and shows it. The GPU backends + /// return before the GPU is done: Skia copies video frame pixels into its staging + /// memory during the flush, so the tree can be dropped once this returns. fn present(&mut self, draw: &mut dyn FnMut(&Canvas)) -> PlayerResult<()>; } diff --git a/fframes-native-player/src/presenter/vulkan.rs b/fframes-native-player/src/presenter/vulkan.rs index 73740b6a..182078c8 100644 --- a/fframes-native-player/src/presenter/vulkan.rs +++ b/fframes-native-player/src/presenter/vulkan.rs @@ -1,11 +1,12 @@ use std::ffi::CStr; use std::mem::ManuallyDrop; use std::rc::Rc; +use std::time::{Duration, Instant}; use fframes_skia_renderer::render::RenderCache; use fframes_skia_renderer::skia_safe::gpu::ganesh::vk::backend_render_targets; use fframes_skia_renderer::skia_safe::gpu::{ - self, DirectContext, FlushInfo, SurfaceOrigin, SyncCpu, + self, DirectContext, FlushInfo, SemaphoresSubmitted, SurfaceOrigin, SyncCpu, backend_semaphores, }; use fframes_skia_renderer::skia_safe::{Canvas, ColorType, Surface}; use fframes_skia_renderer::vulkan::ash::{ @@ -27,11 +28,14 @@ type SurfaceLayer = raw_window_metal::Layer; #[cfg(not(target_os = "macos"))] type SurfaceLayer = (); +const GPU_CACHE_IDLE: Duration = Duration::from_secs(1); + /// Skia draws into the images of a swapchain of the window. pub(super) struct VulkanTarget { swapchain: Option, // Dropped in `drop`, after the swapchain images it draws into. gpu: ManuallyDrop, + last_cache_cleanup: Instant, queue: vk::Queue, queue_lock: QueueLock, surface: vk::SurfaceKHR, @@ -51,15 +55,23 @@ struct Swapchain { handle: vk::SwapchainKHR, /// A Skia surface on every image of the swapchain. images: Vec, + /// One per image, signaled once Skia's work on the image is done. The present of + /// the image waits for it, so the CPU does not have to. + rendered: Vec, /// The swapchain no longer matches the window and is recreated before the next frame. outdated: bool, } impl Swapchain { - fn destroy(self, swapchain_fn: &ash::khr::swapchain::Device) { + fn destroy(self, device: &ash::Device, swapchain_fn: &ash::khr::swapchain::Device) { // Skia's views of the images go first. drop(self.images); - unsafe { swapchain_fn.destroy_swapchain(self.handle, None) }; + unsafe { + for semaphore in self.rendered { + device.destroy_semaphore(semaphore, None); + } + swapchain_fn.destroy_swapchain(self.handle, None); + } } } @@ -97,6 +109,7 @@ impl VulkanTarget { let mut target = Self { swapchain: None, gpu: ManuallyDrop::new(gpu), + last_cache_cleanup: Instant::now(), queue, queue_lock, surface, @@ -200,7 +213,7 @@ impl VulkanTarget { }; if extent.width == 0 || extent.height == 0 { if let Some(old) = old { - old.destroy(&self.swapchain_fn); + old.destroy(device, &self.swapchain_fn); } return Ok(()); } @@ -264,7 +277,7 @@ impl VulkanTarget { // Retired by the new swapchain, or of no use if it could not be created. if let Some(old) = old { - old.destroy(&self.swapchain_fn); + old.destroy(device, &self.swapchain_fn); } let handle = handle.map_err(vulkan_error("can not create the swapchain"))?; @@ -311,20 +324,29 @@ impl VulkanTarget { .collect::>>() }); - match images { - Ok(images) => { - self.swapchain = Some(Swapchain { - handle, - images, - outdated: false, - }); - Ok(()) - } + let mut swapchain = match images { + Ok(images) => Swapchain { + handle, + images, + rendered: Vec::new(), + outdated: false, + }, Err(err) => { unsafe { self.swapchain_fn.destroy_swapchain(handle, None) }; - Err(err) + return Err(err); + } + }; + for _ in 0..swapchain.images.len() { + match unsafe { device.create_semaphore(&vk::SemaphoreCreateInfo::default(), None) } { + Ok(semaphore) => swapchain.rendered.push(semaphore), + Err(err) => { + swapchain.destroy(device, &self.swapchain_fn); + return Err(vulkan_error("can not create a semaphore")(err)); + } } } + self.swapchain = Some(swapchain); + Ok(()) } /// The index of the next swapchain image, once it can be drawn into. `None` while the @@ -401,10 +423,17 @@ impl WindowTarget for VulkanTarget { return Ok(()); }; let surface = &mut swapchain.images[index as usize]; + let rendered = swapchain.rendered[index as usize]; draw(surface.canvas()); let _queue = lock_queue(Some(&self.queue_lock)); + // Release unlocked Skia resources from earlier scenes while retaining active frames. + if self.last_cache_cleanup.elapsed() >= GPU_CACHE_IDLE { + self.gpu.perform_deferred_cleanup(GPU_CACHE_IDLE, None); + self.last_cache_cleanup = Instant::now(); + } + // Ask Skia for the layout the presentation engine reads. `BackendSurfaceAccess::Present` // changes the layout only when Skia's context lists `VK_KHR_swapchain`, and the // contexts of `SkiaVulkanCtx` do not. @@ -412,17 +441,30 @@ impl WindowTarget for VulkanTarget { gpu::vk::ImageLayout::PRESENT_SRC_KHR, self.backend.queue_family_index(), ); - self.gpu.flush_surface_with_texture_state( - surface, - &FlushInfo::default(), - Some(&present_layout), - ); - self.gpu.submit(SyncCpu::Yes); + // SAFETY: the semaphore lives as long as the swapchain, which is destroyed after + // the device is idle. + let mut signal = [unsafe { backend_semaphores::make_vk(rendered.as_raw() as _) }]; + let mut flush_info = FlushInfo::default(); + // SAFETY: `signal` outlives the flush. + unsafe { flush_info.set_signal_semaphores(&mut signal) }; + let signaled = + self.gpu + .flush_surface_with_texture_state(surface, &flush_info, Some(&present_layout)); + // The flush copied image pixels into staging buffers, so the CPU does not wait for + // the GPU: the present does. Without a signal (nothing was drawn) it waits here. + let wait_semaphores: &[vk::Semaphore] = if signaled == SemaphoresSubmitted::Yes { + self.gpu.submit(SyncCpu::No); + &[rendered] + } else { + self.gpu.submit(SyncCpu::Yes); + &[] + }; let presented = unsafe { self.swapchain_fn.queue_present( self.queue, &vk::PresentInfoKHR::default() + .wait_semaphores(wait_semaphores) .swapchains(&[swapchain.handle]) .image_indices(&[index]), ) @@ -448,7 +490,7 @@ impl Drop for VulkanTarget { } if let Some(swapchain) = self.swapchain.take() { - swapchain.destroy(&self.swapchain_fn); + swapchain.destroy(device, &self.swapchain_fn); } unsafe { From de143f8fa25bc8a3e5de89dad3394ecbf0ef29da Mon Sep 17 00:00:00 2001 From: Dmitriy Kovalenko Date: Sat, 31 Oct 2026 11:49:25 -0700 Subject: [PATCH 19/21] perf(player): Share source decoders between a few playback workers A preview decoder keeps the last eight frames it received. A worker that asks for an earlier frame takes one of them instead of seeking back, so workers can prepare trees out of order. Projects with source videos now default to up to four workers instead of one. --- fframes-media/src/video_decoder.rs | 98 ++++++++++++++++++++++-- fframes-media/tests/video_decoder_eof.rs | 20 +++++ fframes-native-player/src/options.rs | 46 ++++++----- 3 files changed, 137 insertions(+), 27 deletions(-) diff --git a/fframes-media/src/video_decoder.rs b/fframes-media/src/video_decoder.rs index d0e7658d..009decd9 100644 --- a/fframes-media/src/video_decoder.rs +++ b/fframes-media/src/video_decoder.rs @@ -399,8 +399,16 @@ pub struct FFmpegDecoder { draining: bool, /// The target frame holds a frame decoded since the last seek. has_decoded_frame: bool, + /// Frames received since the last seek, oldest first, at most `recent_capacity`. + /// Workers that request frames out of order take one of these instead of seeking back. + recent: VecDeque<*mut AVFrame>, + recent_capacity: usize, + seeks: u64, } +/// Frames the preview keeps for out-of-order requests, see `FFmpegDecoder::recent`. +const RECENT_FRAMES: usize = 8; + unsafe impl Send for FFmpegDecoder {} unsafe impl Sync for FFmpegDecoder {} @@ -717,6 +725,14 @@ impl FFmpegDecoder { last_offset: None, draining: false, has_decoded_frame: false, + recent: VecDeque::new(), + // Export workers own their decoders and read in order. + recent_capacity: if mode == DecodeMode::Configured { + 0 + } else { + RECENT_FRAMES + }, + seeks: 0, }; decoder.pkt = av_packet_alloc(); @@ -869,12 +885,58 @@ impl FFmpegDecoder { self.last_offset = Some(offset); self.draining = false; self.has_decoded_frame = false; + self.seeks += 1; + self.forget_recent(); av_packet_unref(self.pkt); Ok(()) } } + /// How many times the decoder repositioned the stream. + pub fn seeks(&self) -> u64 { + self.seeks + } + + /// Keeps a reference to the frame that was just received. + unsafe fn remember(&mut self, frame: *mut AVFrame) { + if self.recent_capacity == 0 { + return; + } + let clone = unsafe { av_frame_clone(frame) }; + if clone.is_null() { + return; + } + if self.recent.len() == self.recent_capacity + && let Some(mut oldest) = self.recent.pop_front() + { + unsafe { av_frame_free(&raw mut oldest) }; + } + self.recent.push_back(clone); + } + + fn forget_recent(&mut self) { + for mut frame in self.recent.drain(..) { + unsafe { av_frame_free(&raw mut frame) }; + } + } + + /// The frame the stream shows at `target_pts`, if the recent frames cover it. They + /// are consecutive, so the first one at or after the target is the right one. + fn recent_frame(&self, target_pts: i64) -> Option<*mut AVFrame> { + let covered = self + .recent + .front() + .is_some_and(|&oldest| unsafe { (*oldest).pts } <= target_pts); + if !covered { + return None; + } + self.recent + .iter() + .copied() + .find(|&frame| unsafe { (*frame).pts } >= target_pts) + } + /// Moves offset to the start of the video if need to loop /// # Safety /// Generally safe but uses libav functions @@ -979,6 +1041,28 @@ impl FFmpegDecoder { // would be returned. Far jumps forward seek to the closest keyframe instead of // decoding every frame in between (frames can be requested out of order by // parallel renderers). + let target_pts = av_rescale_q( + offset, + self.custom_time_base, + self.video_stream_info.time_base, + ); + if (*self.frame_buf.latest_av_frame).pts == target_pts { + return Ok(true); + } + // A recent frame leaves the stream position alone, so the worker that + // reads in order continues without a seek. + if let Some(frame) = self.recent_frame(target_pts) { + av_frame_unref(self.frame_buf.latest_av_frame); + let ret = av_frame_ref(self.frame_buf.latest_av_frame, frame); + if ret < 0 { + return Err(FFramesMediaError::LibAVAudioDecodingError(( + ret, + "Error reusing a decoded frame".to_string(), + ))); + } + return Ok(true); + } + let seek_ahead_frames = 2 * i64::from(self.custom_time_base.den.max(1)); let needs_seek = self .last_offset @@ -990,12 +1074,6 @@ impl FFmpegDecoder { self.seek_to_offset(offset)?; } - let target_pts = av_rescale_q( - offset, - self.custom_time_base, - self.video_stream_info.time_base, - ); - if (*self.frame_buf.latest_av_frame).pts >= target_pts { return Ok(true); } @@ -1015,8 +1093,13 @@ impl FFmpegDecoder { av_frame_unref(target_frame); av_frame_move_ref(target_frame, self.recv_frame); self.has_decoded_frame = true; - if (*target_frame).pts >= target_pts { + let due = (*target_frame).pts >= target_pts; + // Skipped frames are kept too, so the recent frames stay consecutive. + if due || self.recent_capacity > 0 { self.transfer_hardware_surface_data(target_frame)?; + self.remember(self.frame_buf.latest_av_frame); + } + if due { return Ok(true); } continue; @@ -1086,6 +1169,7 @@ impl Drop for FFmpegDecoder { avformat_close_input(&raw mut self.fmt_ctx); av_packet_free(&raw mut self.pkt); } + self.forget_recent(); } } diff --git a/fframes-media/tests/video_decoder_eof.rs b/fframes-media/tests/video_decoder_eof.rs index cf8100a3..a34eaab2 100644 --- a/fframes-media/tests/video_decoder_eof.rs +++ b/fframes-media/tests/video_decoder_eof.rs @@ -253,6 +253,26 @@ fn released_frames_lend_their_pixel_buffers_to_later_conversions() { assert_eq!(third.data.len(), 32 * 32 * 4); } +#[test] +fn preview_decoders_serve_recent_frames_out_of_order_without_seeking() { + use fframes_media::DecodeMode; + + let video = VideoFixture::new(2); + let mut decoder = + unsafe { FFmpegDecoder::new_with_mode(&video.path(), FPS, 1, DecodeMode::Preview) } + .unwrap(); + // Parallel workers reach the decoder out of order within a few frames. + for index in [100, 102, 101, 104, 103, 100, 105, 107, 106] { + assert_frame(&mut decoder, index); + } + assert_eq!(decoder.seeks(), 1, "only the jump from the start seeks"); + // Beyond the recent frames the decoder seeks back as before. + assert_frame(&mut decoder, 90); + assert_eq!(decoder.seeks(), 2); + assert_frame(&mut decoder, 91); + assert_eq!(decoder.seeks(), 2); +} + #[test] fn a_frame_requested_again_at_another_size_is_converted_to_that_size() { let video = VideoFixture::new(0); diff --git a/fframes-native-player/src/options.rs b/fframes-native-player/src/options.rs index 83656080..28094d36 100644 --- a/fframes-native-player/src/options.rs +++ b/fframes-native-player/src/options.rs @@ -44,9 +44,9 @@ pub struct PlayerOptions<'a, 'media> { /// Play the video's audio map on the default output device. pub audio: bool, /// Maximum number of threads building frame trees (`Video::render_frame`) in parallel. - /// Defaults to one worker for video sources so the decoder reads frames in order. - /// Source video workers are limited to the default buffer size plus one live frame. - /// Other projects use half the available cores, up to eight workers. + /// Defaults to half the available cores: up to four workers for video sources, which + /// share one decoder per file, and up to eight otherwise. Source video workers are + /// limited to the default buffer size plus one live frame. pub render_threads: Option, /// Maximum number of future frame trees to prepare. The default (`usize::MAX`) /// buffers up to four trees. Zero prepares only the currently due frame. @@ -88,31 +88,34 @@ impl PlayerOptions<'_, '_> { } pub(crate) fn render_threads(&self) -> usize { + // Source decoders are shared and keep recent frames, so a few workers can + // prepare trees in parallel. More workers than staged trees would idle. if self.media.is_some_and(MediaProvider::has_video_sources) { return self .render_threads - .unwrap_or(1) + .unwrap_or_else(|| (cores() / 2).min(crate::scheduler::MAX_STAGED_TREES)) .clamp(1, crate::scheduler::MAX_STAGED_TREES + 1); } self.render_threads - .unwrap_or_else(|| { - // The preview owns its workers. Querying Rayon here starts a second idle pool. - let cores = std::env::var("FFRAMES_NUM_THREADS") - .ok() - .and_then(|value| value.parse::().ok()) - .or_else(|| { - std::env::var("RAYON_NUM_THREADS") - .ok() - .and_then(|value| value.parse::().ok()) - .filter(|&threads| threads > 0) - }) - .unwrap_or_else(|| std::thread::available_parallelism().map_or(1, usize::from)); - (cores / 2).clamp(1, 8) - }) + .unwrap_or_else(|| (cores() / 2).clamp(1, 8)) .max(1) } } +/// The preview owns its workers. Querying Rayon here starts a second idle pool. +fn cores() -> usize { + std::env::var("FFRAMES_NUM_THREADS") + .ok() + .and_then(|value| value.parse::().ok()) + .or_else(|| { + std::env::var("RAYON_NUM_THREADS") + .ok() + .and_then(|value| value.parse::().ok()) + .filter(|&threads| threads > 0) + }) + .unwrap_or_else(|| std::thread::available_parallelism().map_or(1, usize::from)) +} + #[cfg(test)] mod tests { use super::*; @@ -128,7 +131,7 @@ mod tests { } #[test] - fn video_sources_default_to_one_worker() { + fn video_sources_default_to_a_few_workers() { let media = fframes::DynamicMediaProvider::default().with_video( "clip.mp4", fframes::media::VideoMedia { @@ -141,7 +144,10 @@ mod tests { media: Some(&media), ..Default::default() }; - assert_eq!(options.render_threads(), 1); + assert_eq!( + options.render_threads(), + (fframes::get_thread_count() / 2).clamp(1, 4) + ); options.render_threads = Some(4); assert_eq!(options.render_threads(), 4); options.render_threads = Some(16); From e1bafc069c204ed4a9074754bd7b366fa8f28b9d Mon Sep 17 00:00:00 2001 From: Dmitriy Kovalenko Date: Sat, 31 Oct 2026 12:07:05 -0700 Subject: [PATCH 20/21] fix(player): Keep 100 text cache entries per worker Eight entries were fewer than the text nodes of one teej-podcast frame, so every frame laid out all its text again and the worker used eight times the CPU. --- fframes-native-player/src/scheduler.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/fframes-native-player/src/scheduler.rs b/fframes-native-player/src/scheduler.rs index 07f7655f..e27a2c2e 100644 --- a/fframes-native-player/src/scheduler.rs +++ b/fframes-native-player/src/scheduler.rs @@ -285,9 +285,10 @@ impl Scheduler { kind: WorkerKind, on_frame_ready: &(dyn Fn(FrameReady) + Sync), ) { - // Text caches are `Rc` based and therefore owned by the worker. - // A changing frame counter would otherwise retain 100 sets of glyph outlines per worker. - const TEXT_CACHE_ENTRIES: usize = 8; + // Text caches are `Rc` based and therefore owned by the worker. The entries are + // small. Fewer than the text nodes of one frame makes every frame lay out all + // its text again: 8 entries cost teej-podcast eight times the worker CPU. + const TEXT_CACHE_ENTRIES: usize = 100; let text_cache = TextCache::new(TEXT_CACHE_ENTRIES); let mut converter_cache = usvgr::Cache::new_with_text_cache(TEXT_CACHE_ENTRIES); let video_decoders = match kind { From 161feae2f284e3f4966e885287951c2d9c7caf75 Mon Sep 17 00:00:00 2001 From: Dmitriy Kovalenko Date: Sat, 31 Oct 2026 14:14:35 -0700 Subject: [PATCH 21/21] feat(media): Keep one shared decoder per playback speed The decoder pool keys each file by playback speed and keeps the four speeds used last, as the per-worker decoders did before. The preview decoder counts its position in stream time, so the recent frames and the loop position follow the speed. --- fframes-media/src/video_decoder.rs | 141 +++++++++------ fframes-media/tests/video_decoder_eof.rs | 78 +++++++- fframes/src/video_data.rs | 221 ++++++++++++++++------- 3 files changed, 314 insertions(+), 126 deletions(-) diff --git a/fframes-media/src/video_decoder.rs b/fframes-media/src/video_decoder.rs index 009decd9..a6d5aadd 100644 --- a/fframes-media/src/video_decoder.rs +++ b/fframes-media/src/video_decoder.rs @@ -1,6 +1,6 @@ use crate::FFramesMediaError; use crate::error::Result; -use crate::video_types::FrameConvertOptions; +use crate::video_types::{FrameConvertOptions, PlaybackSpeed}; use ffmpeg_sys_fframes::SwsFlags::{SWS_BICUBIC, SWS_BILINEAR}; use ffmpeg_sys_fframes::*; use std::borrow::Cow; @@ -391,10 +391,11 @@ pub struct FFmpegDecoder { video_stream_info: VideoStreamInfo, codec_ctx: *mut AVCodecContext, pkt: *mut AVPacket, + /// Offsets are counted in this time base: `speed / target_fps` seconds per offset. custom_time_base: AVRational, - duration_in_frames: i64, - /// The offset of the previous `decode_up_to` call, in `custom_time_base` units. - last_offset: Option, + target_fps: i32, + /// The stream position of the previous decode or seek, in the stream's time base. + last_pts: Option, /// A null packet has been sent; receive delayed frames until decoder EOF. draining: bool, /// The target frame holds a frame decoded since the last seek. @@ -636,6 +637,35 @@ impl FFmpegDecoder { .map(|slot| Arc::clone(&slot.image)) } + /// Plays the stream `speed` times faster: offset `n` shows the frame at + /// `n * speed / target_fps` seconds of the stream. + pub fn with_playback_speed(mut self, speed: PlaybackSpeed) -> Self { + self.custom_time_base = AVRational { + num: speed.thousandths(), + den: self.target_fps * PlaybackSpeed::SCALE, + }; + self.last_pts = None; + self.current_loop = 0; + self + } + + /// The stream position of the offset, in the stream's time base. + fn offset_pts(&self, offset: i64) -> i64 { + unsafe { + av_rescale_q( + offset, + self.custom_time_base, + self.video_stream_info.time_base, + ) + } + } + + /// The exact stream duration, in the stream's time base. `None` if it is unknown. + fn stream_duration(&self) -> Option { + let duration = self.video_stream_info.duration; + (duration > 0).then_some(duration) + } + /// Creates a new decoder for the video file at the specified path /// # Safety /// Generally safe but uses libav functions @@ -704,12 +734,6 @@ impl FFmpegDecoder { den: target_fps as i32, }; - let duration_in_frames = av_rescale_q( - video_stream_info.duration, - video_stream_info.time_base, - custom_time_base, - ); - let mut decoder = Self { codec_ctx, fmt_ctx, @@ -720,9 +744,9 @@ impl FFmpegDecoder { frame_buf: Arc::new(frame_buf), video_stream_info, custom_time_base, - duration_in_frames, + target_fps: target_fps as i32, current_loop: 0, - last_offset: None, + last_pts: None, draining: false, has_decoded_frame: false, recent: VecDeque::new(), @@ -859,13 +883,11 @@ impl FFmpegDecoder { /// # Safety /// Generally safe but uses libav functions pub unsafe fn seek_to_offset(&mut self, offset: i64) -> Result<()> { - unsafe { - let timestamp = av_rescale_q( - offset, - self.custom_time_base, - self.video_stream_info.time_base, - ); + unsafe { self.seek_to_pts(self.offset_pts(offset)) } + } + unsafe fn seek_to_pts(&mut self, timestamp: i64) -> Result<()> { + unsafe { let ret = av_seek_frame( self.fmt_ctx, self.video_stream_info.stream_index, @@ -882,7 +904,7 @@ impl FFmpegDecoder { (*self.frame_buf.latest_av_frame).pts = -1; avcodec_flush_buffers(self.codec_ctx); - self.last_offset = Some(offset); + self.last_pts = Some(timestamp); self.draining = false; self.has_decoded_frame = false; self.seeks += 1; @@ -937,27 +959,35 @@ impl FFmpegDecoder { .find(|&frame| unsafe { (*frame).pts } >= target_pts) } - /// Moves offset to the start of the video if need to loop + /// Decodes the frame at the offset of the video played in a loop. The position in + /// the loop comes from the exact stream duration, so loops at a fractional speed + /// do not drift and do not end early. /// # Safety /// Generally safe but uses libav functions - pub unsafe fn adjust_offset_for_looping(&mut self, offset: i64) -> Result { - // An unknown or empty duration can't be looped over (and would divide by zero below). - if self.duration_in_frames <= 0 || offset < self.duration_in_frames { - return Ok(offset); - } + pub unsafe fn decode_looping(&mut self, offset: i64) -> Result { + unsafe { + let pts = self.offset_pts(offset); + // An unknown duration can't be looped over (and would divide by zero below). + let Some(duration) = self.stream_duration() else { + return self.decode_up_to_pts(pts); + }; - let new_loop_index = offset / self.duration_in_frames; - let new_offset = offset % self.duration_in_frames; - if new_loop_index == self.current_loop { - return Ok(new_offset); - } + let loop_index = pts.div_euclid(duration); + let pts = pts.rem_euclid(duration); + if loop_index != self.current_loop { + self.current_loop = loop_index; + self.seek_to_pts(pts)?; + } - self.current_loop = new_loop_index; - unsafe { - self.seek_to_offset(new_offset)?; - } + if self.decode_up_to_pts(pts)? { + return Ok(true); + } - Ok(new_offset) + // The frames end before the stream's duration: start the next loop. + self.current_loop += 1; + self.seek_to_pts(0)?; + self.decode_up_to_pts(0) + } } /// If we are actually using hardware accelerated frames it is required to correctly @@ -1036,16 +1066,11 @@ impl FFmpegDecoder { /// # Safety /// Generally safe but uses libav functions pub unsafe fn decode_up_to(&mut self, offset: i64) -> Result { + unsafe { self.decode_up_to_pts(self.offset_pts(offset)) } + } + + unsafe fn decode_up_to_pts(&mut self, target_pts: i64) -> Result { unsafe { - // Going back needs a seek, otherwise the newer frame that was already decoded - // would be returned. Far jumps forward seek to the closest keyframe instead of - // decoding every frame in between (frames can be requested out of order by - // parallel renderers). - let target_pts = av_rescale_q( - offset, - self.custom_time_base, - self.video_stream_info.time_base, - ); if (*self.frame_buf.latest_av_frame).pts == target_pts { return Ok(true); } @@ -1063,16 +1088,20 @@ impl FFmpegDecoder { return Ok(true); } - let seek_ahead_frames = 2 * i64::from(self.custom_time_base.den.max(1)); - let needs_seek = self - .last_offset - .map_or(offset > seek_ahead_frames, |last_offset| { - offset < last_offset || offset - last_offset > seek_ahead_frames - }); - self.last_offset = Some(offset); - if needs_seek { - self.seek_to_offset(offset)?; + // Going back needs a seek, otherwise the newer frame that was already decoded + // would be returned. Far jumps forward (more than 2 seconds of the stream) seek + // to the closest keyframe instead of decoding every frame in between (frames can + // be requested out of order by parallel renderers). A new decoder is at the start. + let seek_ahead = av_rescale_q( + 2, + AVRational { num: 1, den: 1 }, + self.video_stream_info.time_base, + ); + let last_pts = self.last_pts.unwrap_or(0); + if target_pts < last_pts || target_pts - last_pts > seek_ahead { + self.seek_to_pts(target_pts)?; } + self.last_pts = Some(target_pts); if (*self.frame_buf.latest_av_frame).pts >= target_pts { return Ok(true); @@ -1108,8 +1137,10 @@ impl FFmpegDecoder { // The target lies after the last frame's timestamp but before the end of // the stream (e.g. a 24 fps clip sampled at 30 fps): the last frame is // still the one on screen, so return it instead of reporting the end. - let shows_last_frame = - self.has_decoded_frame && offset < self.duration_in_frames; + let shows_last_frame = self.has_decoded_frame + && self + .stream_duration() + .is_some_and(|duration| target_pts < duration); if shows_last_frame { self.transfer_hardware_surface_data(target_frame)?; } diff --git a/fframes-media/tests/video_decoder_eof.rs b/fframes-media/tests/video_decoder_eof.rs index a34eaab2..4e028faf 100644 --- a/fframes-media/tests/video_decoder_eof.rs +++ b/fframes-media/tests/video_decoder_eof.rs @@ -6,7 +6,7 @@ use std::process::Command; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; -use fframes_media::{FFmpegDecoder, FrameConvertOptions, ResizeVideoFrame}; +use fframes_media::{FFmpegDecoder, FrameConvertOptions, PlaybackSpeed, ResizeVideoFrame}; const FPS: usize = 30; const FRAME_COUNT: i64 = 300; @@ -149,9 +149,8 @@ fn b_frames_are_drained_and_decoding_can_restart_after_eof() { assert_frame(&mut decoder, FRAME_COUNT - 1); // Looping also seeks after decoding the delayed tail. - let offset = unsafe { decoder.adjust_offset_for_looping(FRAME_COUNT) }.unwrap(); - assert_eq!(offset, 0); - assert_frame(&mut decoder, offset); + assert!(unsafe { decoder.decode_looping(FRAME_COUNT) }.unwrap()); + assert_frame(&mut decoder, 0); assert_frame(&mut decoder, FRAME_COUNT - 1); } @@ -364,8 +363,7 @@ fn last_frame_is_shown_until_the_stream_ends_at_a_higher_fps() { for _ in 0..2 { assert!(!unsafe { decoder.decode_up_to(60) }.unwrap()); } - let offset = unsafe { decoder.adjust_offset_for_looping(60) }.unwrap(); - assert_eq!(offset, 0); + assert!(unsafe { decoder.decode_looping(60) }.unwrap()); assert_frame(&mut decoder, 0); // A seek straight to the tail must not return a frame left over from before the seek. @@ -514,3 +512,71 @@ fn preview_falls_back_for_lossless_alpha_video() { assert!((127..=128).contains(&pixels[3])); } } + +/// The source frame that offset `n` shows at `num / den` speed: the first frame at or +/// after `n * num / den` (the last frame until the stream ends), in a loop if `looping`. +fn expected_frame(offset: i64, num: i64, den: i64, looping: bool) -> i64 { + let mut position = offset * num; + if looping { + position %= FRAME_COUNT * den; + } + ((position + den - 1) / den).min(FRAME_COUNT - 1) +} + +fn assert_shows(decoder: &FFmpegDecoder, speed: f32, offset: i64, frame: i64) { + let seconds = unsafe { decoder.get_raw_frame().timestamp_seconds() }; + assert!( + (seconds - frame as f32 / FPS as f32).abs() < 0.0001, + "{speed}x: offset {offset} shows {seconds}s, expected frame {frame}" + ); +} + +fn assert_speed(speed: f32, num: i64, den: i64) { + let video = VideoFixture::new(3); + let speed_of = |speed| { + unsafe { FFmpegDecoder::new(&video.path(), FPS, 1) } + .unwrap() + .with_playback_speed(PlaybackSpeed::new(speed)) + }; + + // Every offset before the end of the stream has a frame, also when the stream's + // duration is not a whole number of offsets. + let mut decoder = speed_of(speed); + let offsets = (FRAME_COUNT * den + num - 1) / num; + for offset in 0..offsets { + assert!( + unsafe { decoder.decode_up_to(offset) }.unwrap(), + "{speed}x: offset {offset}" + ); + assert_shows( + &decoder, + speed, + offset, + expected_frame(offset, num, den, false), + ); + } + assert!(!unsafe { decoder.decode_up_to(offsets) }.unwrap()); + + // Loops follow the exact duration and do not drift. + let mut decoder = speed_of(speed); + for offset in 0..3 * offsets { + assert!( + unsafe { decoder.decode_looping(offset) }.unwrap(), + "{speed}x looping: offset {offset}" + ); + assert_shows( + &decoder, + speed, + offset, + expected_frame(offset, num, den, true), + ); + } +} + +#[test] +fn playback_speed_scales_the_source_time() { + assert_speed(2.0, 2, 1); + assert_speed(1.5, 3, 2); + assert_speed(1.4, 7, 5); + assert_speed(0.5, 1, 2); +} diff --git a/fframes/src/video_data.rs b/fframes/src/video_data.rs index 86dd0d3f..b6112ee3 100644 --- a/fframes/src/video_data.rs +++ b/fframes/src/video_data.rs @@ -6,7 +6,7 @@ use std::sync::Arc; use crate::error::Result; #[cfg(not(target_arch = "wasm32"))] use fframes_media::ImageMetadata; -pub use fframes_media::{FrameConvertOptions, ResizeVideoFrame}; +pub use fframes_media::{FrameConvertOptions, PlaybackSpeed, ResizeVideoFrame}; #[cfg(not(target_arch = "wasm32"))] use std::{ collections::HashMap, @@ -68,7 +68,7 @@ pub trait FFramesSyncedVideoFrame<'media> { fn stream_duration_in_seconds(&self) -> f32; } -#[derive(Debug, Clone, Copy, Default)] +#[derive(Debug, Clone, Copy)] pub struct SyncVideoFrameInput<'media> { /// The seconds timestamp in seconds to start playing the video from /// If not provided starts playing from the beginning of the video or scene. @@ -77,11 +77,30 @@ pub struct SyncVideoFrameInput<'media> { /// Also if `true` the `get_synced_video_frame` API should never return None, only in case of /// decoding errors, so in some cases it might be okay to panic if the frame is not available. pub looping: bool, + /// Playback speed of the video: `2.0` plays it twice as fast, `0.5` at half speed. + /// Every frame shows the source frame at `seconds since start_from * speed`, so the + /// motion stays even at any speed and frame rate. With `looping` a loop takes + /// `duration / speed` seconds. The speed is kept to 1/1000 (`PlaybackSpeed`); a value + /// that is not positive and finite plays at normal speed. Default `1.0`. + /// The speed is constant: each speed of a file has its own decoder, and the + /// renderer keeps the 4 used last. + pub speed: f32, /// The wasm preview editor is not supporting decoding video frames in runtime /// so this allows to provide a fallback image for the debugging pub editor_fallback_image: Option<&'media ImageData<'media>>, } +impl Default for SyncVideoFrameInput<'_> { + fn default() -> Self { + Self { + start_from: 0.0, + looping: false, + speed: 1.0, + editor_fallback_image: None, + } + } +} + /// This is used by the editor as a fallback for the static image #[derive(Debug, Clone)] pub struct WasmEditorVideoFrameFallback<'media> { @@ -187,40 +206,97 @@ type DecoderSlot = Arc>>; #[cfg(not(target_arch = "wasm32"))] #[derive(Default)] struct DecoderEntry { + /// The preview job that used the decoder last, see `DecoderPool::finish_frame`. last_used: u64, + /// The order of the last `DecoderPool::slot` call, for `MAX_DECODERS_PER_VIDEO`. + last_access: u64, slot: DecoderSlot, } +/// Each decoder keeps the file open and its frames in memory. A file played at more +/// speeds closes the idle decoder that was used least recently. +#[cfg(not(target_arch = "wasm32"))] +const MAX_DECODERS_PER_VIDEO: usize = 4; + #[cfg(not(target_arch = "wasm32"))] #[derive(Default)] struct DecoderPool { - sources: HashMap, + /// The decoders of each file, one per playback speed: offsets of different speeds + /// point to different parts of the file. + sources: HashMap>, generation: u64, + accesses: u64, } #[cfg(not(target_arch = "wasm32"))] impl DecoderPool { - fn slot(&mut self, path: &Path) -> DecoderSlot { - let entry = if let Some(entry) = self.sources.get_mut(path) { - entry + fn slot(&mut self, path: &Path, speed: PlaybackSpeed) -> DecoderSlot { + self.accesses += 1; + let decoders = if let Some(decoders) = self.sources.get_mut(path) { + decoders } else { self.sources.entry(path.to_owned()).or_default() }; + let index = if let Some(index) = decoders.iter().position(|(s, _)| *s == speed) { + index + } else { + if decoders.len() >= MAX_DECODERS_PER_VIDEO + && let Some(idle) = decoders + .iter() + .enumerate() + .filter(|(_, (_, entry))| Arc::strong_count(&entry.slot) == 1) + .min_by_key(|(_, (_, entry))| entry.last_access) + .map(|(index, _)| index) + { + decoders.remove(idle); + } + decoders.push((speed, DecoderEntry::default())); + decoders.len() - 1 + }; + let entry = &mut decoders[index].1; entry.last_used = self.generation; + entry.last_access = self.accesses; Arc::clone(&entry.slot) } + /// The decoders of `path` at every speed. A bare file name finds an unambiguous source. + fn slots_of(&self, path: &Path) -> Vec { + let decoders = self.sources.get(path).or_else(|| { + let mut matches = self + .sources + .iter() + .filter(|(source, _)| source.file_name() == Some(path.as_os_str())); + let (_, decoders) = matches.next()?; + matches.next().is_none().then_some(decoders) + }); + decoders + .map(|decoders| { + decoders + .iter() + .map(|(_, entry)| Arc::clone(&entry.slot)) + .collect() + }) + .unwrap_or_default() + } + fn finish_frame(&mut self, concurrent_jobs: usize) -> Vec { // Count completed jobs, not timeline positions. Scrubs, loops and out-of-order // workers share the same grace period. Keep all layers of an active composition. self.generation += 1; - self.sources - .extract_if(|_, entry| { - self.generation - entry.last_used > concurrent_jobs.max(1) as u64 + 1 - && Arc::strong_count(&entry.slot) == 1 - }) - .map(|(_, entry)| entry.slot) - .collect() + let generation = self.generation; + let grace = concurrent_jobs.max(1) as u64 + 1; + let mut retired = Vec::new(); + self.sources.retain(|_, decoders| { + retired.extend( + decoders + .extract_if(.., |(_, entry)| { + generation - entry.last_used > grace && Arc::strong_count(&entry.slot) == 1 + }) + .map(|(_, entry)| entry.slot), + ); + !decoders.is_empty() + }); + retired } } @@ -297,6 +373,11 @@ impl VideoDecodersWorker { return Ok(None); }; + // The editor decodes on the grid of the video's frames. + let speed = i64::from(PlaybackSpeed::new(options.speed).thousandths()); + let scale = i64::from(PlaybackSpeed::SCALE); + let offset = (offset * speed + scale - 1) / scale; + let last_frame_to_display_image = metadata.duration * ctx.time_base.fps as f32; if offset > last_frame_to_display_image as i64 && !options.looping { return Ok(None); @@ -350,69 +431,51 @@ impl VideoDecodersWorker { pts: i64, filename: &str, ) -> Result>> { - let decoder = { - let pool = self.pool.lock()?; - let path = Path::new(filename); - pool.sources - .get(path) - .or_else(|| { - // Older callers use a bare filename. Only resolve an unambiguous source. - let mut matches = pool - .sources - .iter() - .filter(|(source, _)| source.file_name() == Some(path.as_os_str())); - let (_, entry) = matches.next()?; - matches.next().is_none().then_some(entry) - }) - .map(|entry| Arc::clone(&entry.slot)) - }; - let Some(slot) = decoder else { return Ok(None) }; - Ok(slot - .lock()? - .as_ref() - .and_then(|decoder| decoder.get_decoded_image_in_buf(pts))) + let slots = self.pool.lock()?.slots_of(Path::new(filename)); + for slot in slots { + let image = slot + .lock()? + .as_ref() + .and_then(|decoder| decoder.get_decoded_image_in_buf(pts)); + if image.is_some() { + return Ok(image); + } + } + Ok(None) } #[cfg(not(target_arch = "wasm32"))] pub(crate) fn get_synced_frame<'media>( &self, VideoMedia { path, .. }: &VideoMedia, - mut offset: i64, + offset: i64, ctx: &FFramesContext<'_, 'media>, options: &SyncVideoFrameInput, ) -> Result + 'media>>> { - // A slot serializes this source's cursor. Opening a file, decoding a frame - // or tearing down a hardware session never holds the shared pool lock. - let slot = self.pool.lock()?.slot(path); + // A slot serializes the cursor of this source at this speed. Opening a file, + // decoding a frame or tearing down a hardware session never holds the pool lock. + let speed = PlaybackSpeed::new(options.speed); + let slot = self.pool.lock()?.slot(path, speed); let mut guard = slot.lock()?; unsafe { let decoder = match guard.as_mut() { Some(decoder) => decoder, - None => guard.insert(fframes_media::FFmpegDecoder::new_with_mode( - path, - ctx.time_base.fps, - self.buffer_size, - self.mode, - )?), + None => guard.insert( + fframes_media::FFmpegDecoder::new_with_mode( + path, + ctx.time_base.fps, + self.buffer_size, + self.mode, + )? + .with_playback_speed(speed), + ), }; - if options.looping { - offset = decoder.adjust_offset_for_looping(offset)?; - } - - let has_frame = decoder.decode_up_to(offset)?; - if has_frame { - Ok(Some(decoder.retained_frame()?)) - } else if options.looping { - decoder.seek_to_offset(0)?; - let has_first_frame = decoder.decode_up_to(0)?; - - decoder.current_loop += 1; - Ok(has_first_frame - .then(|| decoder.retained_frame()) - .transpose()?) + let has_frame = if options.looping { + decoder.decode_looping(offset)? } else { - Ok(None) - } + decoder.decode_up_to(offset)? + }; + Ok(has_frame.then(|| decoder.retained_frame()).transpose()?) } } } @@ -426,17 +489,18 @@ mod decoder_pool_tests { let mut pool = DecoderPool::default(); let first = Path::new("first/clip.mp4"); let second = Path::new("second/clip.mp4"); - let held = pool.slot(first); - assert!(!Arc::ptr_eq(&held, &pool.slot(second))); + let normal = PlaybackSpeed::NORMAL; + let held = pool.slot(first, normal); + assert!(!Arc::ptr_eq(&held, &pool.slot(second, normal))); for _ in 0..24 { // Overlapping sources in a composite must all stay warm. for i in 0..12 { - pool.slot(Path::new(&format!("layer-{i}.mp4"))); + pool.slot(Path::new(&format!("layer-{i}.mp4")), normal); } drop(pool.finish_frame(5)); } assert!(!pool.sources.contains_key(second)); - assert!(Arc::ptr_eq(&held, &pool.slot(first))); + assert!(Arc::ptr_eq(&held, &pool.slot(first, normal))); assert_eq!(pool.sources.len(), 13); drop(held); for _ in 0..9 { @@ -444,4 +508,31 @@ mod decoder_pool_tests { } assert!(pool.sources.is_empty()); } + + #[test] + fn a_file_keeps_one_decoder_per_speed_and_closes_the_idle_one_used_least_recently() { + let mut pool = DecoderPool::default(); + let clip = Path::new("clip.mp4"); + let normal = pool.slot(clip, PlaybackSpeed::new(1.0)); + // Speeds that round to the same thousandth share a decoder. + assert!(Arc::ptr_eq( + &normal, + &pool.slot(clip, PlaybackSpeed::new(1.000_1)) + )); + let held = pool.slot(clip, PlaybackSpeed::new(2.0)); + for speed in [0.5, 1.5] { + pool.slot(clip, PlaybackSpeed::new(speed)); + } + assert_eq!(pool.sources[clip].len(), MAX_DECODERS_PER_VIDEO); + drop(normal); + // A fifth speed closes the idle decoder used least recently, not the one in use. + pool.slot(clip, PlaybackSpeed::new(3.0)); + let speeds: Vec<_> = pool.sources[clip] + .iter() + .map(|(speed, _)| speed.thousandths()) + .collect(); + assert_eq!(speeds, [2000, 500, 1500, 3000]); + assert_eq!(pool.slots_of(Path::new("clip.mp4")).len(), 4); + drop(held); + } }