diff --git a/engine/page/host/src/main.rs b/engine/page/host/src/main.rs index 02c15dc6..303963ba 100644 --- a/engine/page/host/src/main.rs +++ b/engine/page/host/src/main.rs @@ -97,7 +97,10 @@ fn main() { ); // on screen, bevy_winit's event loop is the runner (a ScheduleRunner here replaced it: no window, 11-04) if !a.window && !app.is_plugin_added::() { - app.add_plugins(ScheduleRunnerPlugin::run_loop(Duration::from_secs_f64(1.0 / 60.0))); + // 60 frames a second, as most screens; RAEA_PAGE_FPS stands in for another screen's rate (a 120 Hz one, to + // measure how a 60 fps video plays where the window draws twice a frame) + let fps = std::env::var("RAEA_PAGE_FPS").ok().and_then(|v| v.parse::().ok()).filter(|f| (1.0..=480.0).contains(f)).unwrap_or(60.0); + app.add_plugins(ScheduleRunnerPlugin::run_loop(Duration::from_secs_f64(1.0 / fps))); } app.add_plugins(HostPlugin { size: a.size, double_click: Duration::from_millis(500), picking }) .add_plugins(brp::plugins(brp::port(), |r| r)) diff --git a/io/apple/src/convert.rs b/io/apple/src/convert.rs new file mode 100644 index 00000000..c1fa41e4 --- /dev/null +++ b/io/apple/src/convert.rs @@ -0,0 +1,40 @@ +//! VideoToolbox's BGRA as the RGBA a Frame holds. + +/// A row of BGRA as RGBA, its alpha kept or made 255. Four bytes at a time as one word, which the compiler turns into +/// vector code: a 4K frame took 5.5 ms a byte at a time, most of what playing one cost (RAEA_MEDIA_STATS, 11-13). +pub(crate) fn bgra_to_rgba(src: &[u8], dst: &mut [u8], alpha: bool) { + let opaque = if alpha { 0 } else { 0xff00_0000 }; + for (d, s) in dst.chunks_exact_mut(4).zip(src.chunks_exact(4)) { + let v = u32::from_le_bytes([s[0], s[1], s[2], s[3]]); + let out = (v & 0xff00_ff00) | ((v & 0xff) << 16) | ((v >> 16) & 0xff) | opaque; + d.copy_from_slice(&out.to_le_bytes()); + } +} + +#[cfg(test)] +mod tests { + use super::bgra_to_rgba; + + #[test] + fn bgra_becomes_rgba_with_its_alpha_kept_or_made_opaque() { + let src = [10, 20, 30, 40, 1, 2, 3, 0]; + let mut out = [0u8; 8]; + bgra_to_rgba(&src, &mut out, true); + assert_eq!(out, [30, 20, 10, 40, 3, 2, 1, 0]); + bgra_to_rgba(&src, &mut out, false); + assert_eq!(out, [30, 20, 10, 255, 3, 2, 1, 255]); + } + + /// A 4K frame's worth, timed (cargo test -p raea-io-apple --release convert -- --nocapture). + #[test] + fn a_4k_frame_is_converted_in_a_frame_s_small_part() { + let src: Vec = (0..3840 * 2160 * 4).map(|i| (i % 251) as u8).collect(); + let mut out = vec![0u8; src.len()]; + let t = std::time::Instant::now(); + for _ in 0..10 { + bgra_to_rgba(&src, &mut out, false); + } + eprintln!("4K BGRA to RGBA: {:.2} ms a frame", t.elapsed().as_secs_f64() * 100.0); + assert_eq!(&out[..4], &[src[2], src[1], src[0], 255]); + } +} diff --git a/io/apple/src/decode.rs b/io/apple/src/decode.rs index d3c7e580..504fd786 100644 --- a/io/apple/src/decode.rs +++ b/io/apple/src/decode.rs @@ -213,13 +213,7 @@ impl Decoder { let mut rgba = vec![0u8; w * h * 4]; for y in 0..h { let row = std::slice::from_raw_parts(base.add(y * bpr), w * 4); - let dst = &mut rgba[y * w * 4..(y + 1) * w * 4]; - for x in 0..w { - dst[x * 4] = row[x * 4 + 2]; - dst[x * 4 + 1] = row[x * 4 + 1]; - dst[x * 4 + 2] = row[x * 4]; - dst[x * 4 + 3] = if self.alpha { row[x * 4 + 3] } else { 255 }; - } + crate::convert::bgra_to_rgba(row, &mut rgba[y * w * 4..(y + 1) * w * 4], self.alpha); } rgba }); diff --git a/io/apple/src/lib.rs b/io/apple/src/lib.rs index 25e83af4..71b077bc 100644 --- a/io/apple/src/lib.rs +++ b/io/apple/src/lib.rs @@ -4,6 +4,7 @@ #![cfg(target_vendor = "apple")] mod audio; +mod convert; mod decode; mod encode; mod vt; diff --git a/runtime/dom/media/src/lib.rs b/runtime/dom/media/src/lib.rs index 83ed4767..5fd0df5d 100644 --- a/runtime/dom/media/src/lib.rs +++ b/runtime/dom/media/src/lib.rs @@ -5,15 +5,18 @@ //! A seek is the prop `__seek` ([time, n], set by the dom layer's `currentTime`); playing is `__play` ([on, time, n, //! loop], set by `play()` and `pause()`), `__muted` the element's muted and `__volume` its volume. Playing, the decoder's frames come on the //! source's clock (its sound's: raea-runtime-js media_play.rs), each a `time` news for the element's currentTime, with -//! `timeupdate` four times a second as a browser's, `playing` once it's going, and `ended` at its end. raea-runtime-dom's +//! `timeupdate` four times a second as a browser's, `playing` once it's going, and `ended` at its end. A playing +//! video's frames come ahead of their time, each with the moment it is due: at each sync (the page's own frame) the +//! one due then is shown, so the picture keeps the page's rhythm whatever the decoder's. raea-runtime-dom's //! page host keeps one `Medias` and syncs it after each commit. -use std::collections::HashMap; +use std::collections::{HashMap, VecDeque}; use std::sync::Arc; +use std::time::{Duration, Instant}; use blitz_dom::BaseDocument; use blitz_dom::node::{ImageData, RasterImageData, SpecialElementData}; -use raea_runtime_js::media_html::stream::{Cmd, Msg, Stream}; +use raea_runtime_js::media_html::stream::{Cmd, Halvings, Msg, Stream}; use serde_json::{Value, json}; use raea_sys_page::ops::Nodes; @@ -35,6 +38,14 @@ pub struct Media { duration: f64, /// the latest frame: width, height, pixels frame: Option<(u32, u32, Arc>)>, + /// what it is drawn from when it came as its size alone (its halving), until that goes to the painter with it + halves: Halvings, + /// how many halvings its decoder is asked to send with each frame + halvings: u8, + /// a playing video's frames not due yet: each its time, its picture and when to show it + ahead: VecDeque<(f64, (u32, u32, Vec), Instant, Halvings)>, + /// when this was last synced, and how far apart syncs have been lately (the page's frame, s) + synced: (Option, f64), /// `__play` as last seen, and whether it asks to play play: Option, playing: bool, @@ -47,6 +58,8 @@ pub struct Media { stats: Option<(std::time::Instant, u32, u32, f64, f64)>, /// RAEA_MEDIA_STATS=1: when a frame was last shown, and the longest wait between two this second (ms) shown_gap: (Option, f64), + /// RAEA_MEDIA_STATS=1, this second: the page's frames, the longest (ms), and how late frames were shown (sum, max; ms) + paced: (u32, f64, f64, f64), } impl Media { @@ -76,7 +89,7 @@ pub fn sync(doc: &mut BaseDocument, nodes: &Nodes, medias: &mut Medias, out: &mu Some(m) if m.src == src && m.load == load => m, _ => { let stream = (!src.is_empty()).then(|| raea_runtime_js::media_html::stream::open(&raea_sys_page::net::page_path(&src), video)).flatten(); - medias.insert(id, Media { src, load, poster: None, seek: None, stream, video, loaded: false, seeking: None, seeked: false, duration: f64::NAN, frame: None, play: None, playing: false, muted: None, volume: None, told: None, shown_gap: (None, 0.0), stats: (std::env::var("RAEA_MEDIA_STATS").as_deref() == Ok("1")).then(|| (std::time::Instant::now(), 0, 0, 0.0, 0.0)) }); + medias.insert(id, Media { src, load, poster: None, seek: None, stream, video, loaded: false, seeking: None, seeked: false, duration: f64::NAN, frame: None, halves: Vec::new(), halvings: 0, ahead: VecDeque::new(), synced: (None, 1.0 / 60.0), paced: (0, 0.0, 0.0, 0.0), play: None, playing: false, muted: None, volume: None, told: None, shown_gap: (None, 0.0), stats: (std::env::var("RAEA_MEDIA_STATS").as_deref() == Ok("1")).then(|| (std::time::Instant::now(), 0, 0, 0.0, 0.0)) }); medias.get_mut(&id).unwrap() } }; @@ -117,6 +130,7 @@ pub fn sync(doc: &mut BaseDocument, nodes: &Nodes, medias: &mut Medias, out: &mu st.ask(Cmd::Play { at: p.get(1).and_then(Value::as_f64).unwrap_or(0.0), looping: p.get(3).and_then(Value::as_bool).unwrap_or(false) }); } else { st.ask(Cmd::Pause); + m.ahead.clear(); } } m.play = play; @@ -131,9 +145,12 @@ pub fn sync(doc: &mut BaseDocument, nodes: &Nodes, medias: &mut Medias, out: &mu bevy::log::warn!("raea-page: media: {err}"); out.push(json!({"id": id, "type": "error", "message": err})); } - Msg::Frame(t, f) => { + Msg::Frame(t, f, halves) => { + // the first frame or a seek's: shown now, and what was on its way for before the seek is not + m.ahead.clear(); let (w, h) = (f.width, f.height); m.frame = Some((w, h, Arc::new(f.rgba))); + m.halves = halves; redraw = true; came += 1; if !m.loaded { @@ -149,6 +166,12 @@ pub fn sync(doc: &mut BaseDocument, nodes: &Nodes, medias: &mut Medias, out: &mu told(m, id, t, out); } } + // a playing video's frame, for later; one that comes after a pause is for nothing + Msg::Due(t, f, due, halves) => { + if m.playing { + m.ahead.push_back((t, (f.width, f.height, f.rgba), due, halves)); + } + } Msg::Time(t) => { if m.landed(t) { out.push(json!({"id": id, "type": "seeked", "time": t})); @@ -159,13 +182,58 @@ pub fn sync(doc: &mut BaseDocument, nodes: &Nodes, medias: &mut Medias, out: &mu out.push(json!({"id": id, "type": "time", "time": t})); } } - Msg::Playing(t) => out.push(json!({"id": id, "type": "playing", "time": t})), + Msg::Playing(t) => { + m.ahead.clear(); + out.push(json!({"id": id, "type": "playing", "time": t})); + } Msg::Ended(t) => { m.playing = false; out.push(json!({"id": id, "type": "ended", "time": t})); } } } + // the frame due at this sync: the latest whose moment has come, or comes within half a page frame (shown a + // touch early rather than a whole page frame late); the ones before it were never seen + let now = Instant::now(); + if let Some(last) = m.synced.0 { + let dt = now.duration_since(last).as_secs_f64(); + m.synced.1 += 0.1 * (dt.min(0.1) - m.synced.1); + if m.stats.is_some() && !m.ahead.is_empty() { + (m.paced.0, m.paced.1) = (m.paced.0 + 1, m.paced.1.max(dt * 1e3)); + } + } + m.synced.0 = Some(now); + let reach = now + Duration::from_secs_f64((0.5 * m.synced.1).min(0.008)); + let (due, passed) = due_by(&mut m.ahead, reach); + came += passed; + if let Some((t, (w, h, rgba), at, halves)) = due { + m.halves = halves; + let late = now.saturating_duration_since(at).as_secs_f64() * 1e3; + (m.paced.2, m.paced.3) = (m.paced.2 + late, m.paced.3.max(late)); + m.frame = Some((w, h, Arc::new(rgba))); + (redraw, m.seeked) = (true, true); + came += 1; + told(m, id, t, out); + } + // how many halvings its frames should come with: as many as bring one within twice the size it is drawn at + // (the painter's rule, blitz-paint mip_level), counted from its box as last laid out + if m.video && draw && let (Some((w, h, _)), Some(n), Some(st)) = (m.frame.as_ref(), nodes.by_id.get(&id), m.stream.as_mut()) { + let want = doc.get_node(*n).map_or(0, |node| { + let (l, scale) = (node.final_layout(), doc.viewport().scale_f64()); + let drawn = (l.content_box_width() as f64 * scale / *w as f64).min(l.content_box_height() as f64 * scale / *h as f64); + let mut n = 0u8; + while drawn > 0.0 && drawn * f64::from(1u32 << (n + 1)) <= 1.0 && n < 6 { + n += 1; + } + n + }); + // RAEA_MEDIA_HALVINGS=0: frames drawn as they come, as before (to compare with) + let want = if std::env::var("RAEA_MEDIA_HALVINGS").as_deref() == Ok("0") { 0 } else { want }; + if want != m.halvings { + m.halvings = want; + st.ask(Cmd::Halvings(want)); + } + } if redraw && m.video && draw { let pic = match (&m.poster, m.seeked) { (Some(p), false) => poster_image(p).map(|(w, h, px)| (w, h, Arc::new(px))), @@ -175,7 +243,7 @@ pub fn sync(doc: &mut BaseDocument, nodes: &Nodes, medias: &mut Medias, out: &mu .or_else(|| m.frame.as_ref().map(|(w, h, px)| (*w, *h, px.clone()))); if let (Some((w, h, px)), Some(n)) = (pic, nodes.by_id.get(&id)) { let began = std::time::Instant::now(); - set_image(doc, *n, w, h, px); + set_image(doc, *n, w, h, px, std::mem::take(&mut m.halves)); if let Some(s) = m.stats.as_mut().filter(|_| came > 0) { let ms = began.elapsed().as_secs_f64() * 1e3; (s.1, s.2, s.3, s.4) = (s.1 + 1, s.2 + came - 1, s.3 + ms, s.4.max(ms)); @@ -185,9 +253,10 @@ pub fn sync(doc: &mut BaseDocument, nodes: &Nodes, medias: &mut Medias, out: &mu m.shown_gap.0 = Some(began); let took = s.0.elapsed().as_secs_f64(); if took >= 1.0 { - eprintln!("raea-page: video {took:.2} s: {} frames shown ({w}x{h}), {} passed over before the page drew, picture set avg {:.2} ms max {:.2} ms, longest gap {:.0} ms", s.1, s.2, s.3 / s.1.max(1) as f64, s.4, m.shown_gap.1); + eprintln!("raea-page: video {took:.2} s: {} frames shown ({w}x{h}), {} passed over before the page drew, picture set avg {:.2} ms max {:.2} ms, longest gap {:.0} ms; the page synced {} times, longest apart {:.0} ms; shown late avg {:.1} ms max {:.1} ms", s.1, s.2, s.3 / s.1.max(1) as f64, s.4, m.shown_gap.1, m.paced.0, m.paced.1, m.paced.2 / s.1.max(1) as f64, m.paced.3); *s = (std::time::Instant::now(), 0, 0, 0.0, 0.0); m.shown_gap.1 = 0.0; + m.paced = (0, 0.0, 0.0, 0.0); } } } @@ -195,6 +264,16 @@ pub fn sync(doc: &mut BaseDocument, nodes: &Nodes, medias: &mut Medias, out: &mu } } +/// Of the frames waiting, the one to show by `reach`: the latest whose moment has come, taken out with the ones before +/// it, and how many those were (frames nobody saw: the page drew less often than they came). +fn due_by(ahead: &mut VecDeque<(f64, T, Instant, Halvings)>, reach: Instant) -> (Option<(f64, T, Instant, Halvings)>, u32) { + let (mut due, mut passed) = (None, 0); + while ahead.front().is_some_and(|f| f.2 <= reach) { + passed += due.replace(ahead.pop_front().unwrap()).is_some() as u32; + } + (due, passed) +} + /// A playing element's time: each as `time` (its currentTime), and a `timeupdate` four times a second. fn told(m: &mut Media, id: u64, t: f64, out: &mut Vec) { out.push(json!({"id": id, "type": "time", "time": t})); @@ -219,12 +298,44 @@ fn poster_image(src: &str) -> Option<(u32, u32, Vec)> { Some((size.x, size.y, img.data?)) } -/// Node `n` drawn as an image: `rgba` as its raster, its layout made again for the picture's size. -fn set_image(doc: &mut BaseDocument, n: blitz_dom::NodeId, w: u32, h: u32, rgba: Arc>) { +/// Node `n` drawn as an image: `rgba` as its raster. Its layout is made again only when the picture's size isn't the +/// last one's: a playing video's next frame is the same size, and making its box and its ancestors' layout again for +/// each was a part of every page frame it played in. +fn set_image(doc: &mut BaseDocument, n: blitz_dom::NodeId, w: u32, h: u32, rgba: Arc>, halves: Halvings) { let Some(node) = doc.get_node_mut(n) else { return }; let Some(el) = node.element_data_mut() else { return }; - el.special_data = SpecialElementData::Image(Box::new(ImageData::Raster(RasterImageData { width: w, height: h, data: vello::peniko::Blob::new(rgba) }))); + let same = matches!(&el.special_data, SpecialElementData::Image(i) if matches!(&**i, ImageData::Raster(r) if (r.width, r.height) == (w, h))); + let image = RasterImageData { width: w, height: h, data: vello::peniko::Blob::new(rgba) }; + if !halves.is_empty() { + raea_sys_page::preload_mips(&image, halves); + } + el.special_data = SpecialElementData::Image(Box::new(ImageData::Raster(image))); + if same { + return node.insert_damage(raea_sys_page::REPAINT_DAMAGE); + } node.clear_layout_cache(); node.insert_damage(raea_sys_page::ALL_DAMAGE); node.mark_ancestors_dirty(); } + +#[cfg(test)] +mod tests { + use super::*; + + /// At a sync the frame shown is the latest one due by then; earlier ones due are passed over, later ones wait. + #[test] + fn the_frame_due_is_the_latest_whose_moment_has_come() { + let t0 = Instant::now(); + let at = |ms: u64| t0 + Duration::from_millis(ms); + let mut ahead: VecDeque<(f64, u8, Instant, Halvings)> = (0..4).map(|i| (i as f64 / 60.0, i as u8, at(17 * i as u64), Vec::new())).collect(); + // nothing is due before the first + assert!(due_by(&mut ahead, t0 - Duration::from_millis(1)).0.is_none()); + // a page drawing as often as the frames come shows each + assert_eq!(due_by(&mut ahead, at(0)).0.map(|f| f.1), Some(0)); + assert_eq!(due_by(&mut ahead, at(17)).0.map(|f| f.1), Some(1)); + // a page a frame late shows the latest and counts the one it passed + let (due, passed) = due_by(&mut ahead, at(51)); + assert_eq!((due.map(|f| f.1), passed), (Some(3), 1)); + assert!(ahead.is_empty()); + } +} diff --git a/runtime/js/src/media_html.rs b/runtime/js/src/media_html.rs index 55b93ba0..fd7c42ee 100644 --- a/runtime/js/src/media_html.rs +++ b/runtime/js/src/media_html.rs @@ -12,7 +12,11 @@ pub mod stream { /// where a playing source has got to, that it played to its end, or its failure. pub enum Msg { Meta { w: u32, h: u32, duration: f64 }, - Frame(f64, raea_io_media::Frame), + /// a frame to show now (the first, a seek's). Asked for halvings (`Cmd::Halvings`), it comes as its size and + /// the halving it is drawn from, its own pixels left behind + Frame(f64, raea_io_media::Frame, Halvings), + /// a playing video's frame, decoded ahead of its time: the frame at this time, to show at this moment + Due(f64, raea_io_media::Frame, std::time::Instant, Halvings), /// a playing source's time (an audio one's, which has no frames; a video's when paused) Time(f64), /// playing from this time: the sound is on its way @@ -22,6 +26,48 @@ pub mod stream { Fail(String), } + /// Halvings of a frame, largest first: each a 2x2 mean of the one before (width, height, RGBA). A frame drawn at + /// under half its size is drawn from one, so a 4K frame in a small player is neither sampled across four of + /// its pixels at a time (it shimmers) nor sent whole to the GPU for every frame; and since the one it is drawn + /// from is all that is sent, a quarter second of 4K frames waiting is 30 to 120 MB where whole it would be 500. + pub type Halvings = Vec<(u32, u32, Vec)>; + + /// `n` halvings of an RGBA picture (fewer once a side is a pixel). + pub fn halvings(w: u32, h: u32, rgba: &[u8], n: u8) -> Halvings { + let mut out: Halvings = Vec::new(); + for _ in 0..n { + let (sw, sh, src) = match out.last() { + Some(last) => (last.0, last.1, last.2.as_slice()), + None => (w, h, rgba), + }; + if sw < 2 || sh < 2 { + break; + } + let (dw, dh) = ((sw / 2) as usize, (sh / 2) as usize); + let stride = sw as usize * 4; + let mut px = vec![0u8; dw * dh * 4]; + for (y, row) in px.chunks_exact_mut(dw * 4).enumerate() { + let (top, bottom) = (&src[2 * y * stride..][..dw * 8], &src[(2 * y + 1) * stride..][..dw * 8]); + for ((d, a), b) in row.chunks_exact_mut(4).zip(top.chunks_exact(8)).zip(bottom.chunks_exact(8)) { + for c in 0..4 { + d[c] = ((a[c] as u16 + a[c + 4] as u16 + b[c] as u16 + b[c + 4] as u16 + 2) / 4) as u8; + } + } + } + out.push((dw as u32, dh as u32, px)); + } + out + } + + /// A frame as it is sent when `n` halvings are asked for: whole with none; else its size, and the nth halving + /// alone (the one it is drawn from). + pub fn sent(f: raea_io_media::Frame, n: u8) -> (raea_io_media::Frame, Halvings) { + match halvings(f.width, f.height, &f.rgba, n).pop() { + Some(last) => (raea_io_media::Frame { rgba: Vec::new(), ..f }, vec![last]), + None => (f, Vec::new()), + } + } + /// What the element asks of its decoder. Play plays from `at` on the source's clock (its sound's, when it has /// sound and a device): frames as they fall due, each `Frame` news, and `Ended` at its end unless `looping`. pub enum Cmd { @@ -31,6 +77,9 @@ pub mod stream { Muted(bool), /// the element's volume, 0..1: the sound's gain while it isn't muted Volume(f32), + /// how many times each frame should be halved from here on (the page counts them from the size the video is + /// drawn at; 0 while it is drawn at over half its size) + Halvings(u8), } pub struct Stream { @@ -133,7 +182,7 @@ pub mod stream { let i = dec.info().clone(); let _ = tx.send(Msg::Meta { w: i.width, h: i.height, duration: i.duration }); let _ = tx.send(match dec.frame_at(0.0) { - Ok(f) => Msg::Frame(0.0, f), + Ok(f) => Msg::Frame(0.0, f, Vec::new()), Err(e) => Msg::Fail(e), }); Some(crate::media_play::Source::Video(dec)) diff --git a/runtime/js/src/media_play.rs b/runtime/js/src/media_play.rs index cc2d797e..2e2bf99b 100644 --- a/runtime/js/src/media_play.rs +++ b/runtime/js/src/media_play.rs @@ -1,13 +1,15 @@ //! A `video` or `audio` playing (media_html.rs's stream, on its decoder's thread): seeks answered with the frame at the //! time asked; played, frames as they fall due on the source's clock, which is its sound's (media_sound.rs) when it has -//! sound and the Mac a device, else the wall's. Frames are asked for at the clock's time, so a decoder that falls -//! behind skips frames rather than letting the picture lag its sound. +//! sound and the Mac a device, else the wall's. A playing video's frames are decoded a little ahead of the clock and +//! sent with the moment each is due (`Msg::Due`), so how long a frame takes to decode is no part of when it is +//! shown, and a loop's first frame is ready before its last has gone. A decoder that falls behind the clock skips +//! frames rather than letting the picture lag its sound. use std::path::Path; use std::sync::mpsc::{Receiver, RecvTimeoutError, Sender}; use std::time::{Duration, Instant}; -use crate::media_html::stream::{Cmd, Msg}; +use crate::media_html::stream::{Cmd, Msg, sent}; use crate::media_sound::Sound; /// What was opened: a video's decoder, or an audio source's samples. @@ -33,10 +35,19 @@ pub struct Player<'a> { volume: f32, clock: Option, looping: bool, - /// the frame (index at fps) last sent while playing, so a frame isn't decoded twice - shown: Option, + /// the frame (index at fps) to decode next while playing + cursor: i64, + /// what the cursor's frames are ahead of the clock by, in laps: `duration` once the cursor has gone round a + /// loop the clock hasn't yet, else 0 + lap: f64, + /// frames the clock passed before they were decoded, since the last frame sent (for the stats) + skipped: i64, /// when the last Time news went (an audio source's, which has no frames) told: Option, + /// how many halvings each frame is sent with (`Cmd::Halvings`) + halvings: u8, + /// where the picture last sent was in time, while paused: sent again when the halvings asked for change + still: Option, /// RAEA_MEDIA_STATS=1: how play is going, said a line a second stats: Option, } @@ -49,7 +60,7 @@ struct Stats { skipped: u32, sum: f64, max: f64, - /// the longest wait between two frames sent, ms + /// the longest wait between two frames decoded, ms gap: f64, last: Option, } @@ -81,7 +92,7 @@ impl<'a> Player<'a> { Source::Audio(a) => (a.duration(), 0.0), }; let fps = if fps.is_finite() && fps > 1.0 { fps } else { 30.0 }; - Player { path, source, duration, fps, sound: None, muted: false, volume: 1.0, clock: None, looping: false, shown: None, told: None, stats: (std::env::var("RAEA_MEDIA_STATS").as_deref() == Ok("1")).then(Stats::new) } + Player { path, source, duration, fps, sound: None, muted: false, volume: 1.0, clock: None, looping: false, cursor: 0, lap: 0.0, skipped: 0, told: None, halvings: 0, still: Some(0.0), stats: (std::env::var("RAEA_MEDIA_STATS").as_deref() == Ok("1")).then(Stats::new) } } /// Answers the element's asks until it's gone (its stream dropped). @@ -119,6 +130,13 @@ impl<'a> Player<'a> { s.set_volume(v); } } + Cmd::Halvings(n) => { + // paused, the picture showing is sent again with what it is now drawn from + if n != self.halvings && self.clock.is_none() && seek.is_none() { + seek = self.still; + } + self.halvings = n; + } } } if pause && self.clock.is_some() { @@ -152,20 +170,47 @@ impl<'a> Player<'a> { } } - /// How long until the next frame falls due on the clock. Waking every half frame instead (as this did) sent - /// frames 17 to 50 ms apart at 30 fps, whichever half the wake fell in: the picture juddered, and two frames - /// landing inside one of the page's frames lost the page one of them (RAEA_MEDIA_STATS, 11-13: longest gap 43 ms - /// every second, 1 to 3 frames a second passed over). A sound's clock moves in its device's steps, so a wake a - /// moment early waits again, briefly. + /// How long until there is something to do: the cursor's frame comes within the lead, or the clock reaches the + /// end. (Frames were once sent when they fell due, each after its own decode: 22 to 29 ms apart at 60 fps where + /// 16.7 is even, a 4K frame 6.6 ms late, and a loop's first frame behind a seek; RAEA_MEDIA_STATS, 11-13.) fn until_next(&self) -> Duration { let frame = 1.0 / self.fps; - let wait = match (&self.source, self.shown) { - (Source::Video(_), Some(i)) => ((i + 1) as f64 * frame - self.now()).clamp(0.0005, frame), - _ => 0.5 * frame, + let wait = match &self.source { + Source::Video(_) => { + let now = self.now(); + let decode = if self.at_end() { f64::INFINITY } else { self.cursor_time() - self.lead() - now }; + decode.min(self.duration - now).clamp(0.0005, frame) + } + Source::Audio(_) => 0.5 * frame, }; Duration::from_secs_f64(wait) } + /// The cursor's frame's time on the clock. + fn cursor_time(&self) -> f64 { + self.cursor as f64 / self.fps + self.lap + } + + /// The cursor is past the source's last frame (a looping source's cursor goes round instead). + fn at_end(&self) -> bool { + !self.looping && self.duration.is_finite() && self.cursor as f64 / self.fps >= self.duration - 0.5 / self.fps + } + + /// How far ahead of the clock frames are decoded: a quarter second, which rides out a decode that stalls (a + /// reader starting again, a busy machine: 40 to 85 ms seen) and the seek back to a loop's start (70 ms). No more + /// than 128 MB of frames wait at once, 200 nearing a loop's end, and never fewer than two frames: a 4K frame + /// sent whole is 33 MB (three wait), sent as the halving it is drawn from 8 or 2 (the quarter second). + fn lead(&self) -> f64 { + let frame = 1.0 / self.fps; + let bytes = match &self.source { + Source::Video(d) => (d.info().width as f64 * d.info().height as f64 * 4.0 / 4f64.powi(self.halvings as i32)).max(1.0), + Source::Audio(_) => 1.0, + }; + let near_wrap = self.looping && self.duration.is_finite() && (self.lap > 0.0 || self.duration - self.cursor as f64 * frame < 0.25); + let most = ((if near_wrap { 200e6 } else { 128e6 }) / bytes).floor().max(2.0) * frame; + most.min(0.25) + } + /// The clock's time now. fn now(&self) -> f64 { match (&self.clock, &self.sound) { @@ -186,8 +231,13 @@ impl<'a> Player<'a> { let volume = self.volume; self.sound = Some(audio.and_then(|a| Sound::open(a, self.muted).inspect(|s| s.set_volume(volume)).inspect_err(|e| bevy::log::debug!("raea-runtime-js: media plays on the wall's clock: {e}")).ok())); } - self.shown = None; + (self.cursor, self.lap, self.skipped) = ((t * self.fps).floor() as i64, 0.0, 0); self.told = None; + self.restart(t); + } + + /// The clock started at `t` (the sound from there, else the wall). + fn restart(&mut self, t: f64) { self.clock = Some(match &self.sound { Some(Some(s)) => { s.start(t); @@ -211,36 +261,67 @@ impl<'a> Player<'a> { /// The frame at `t`, sent as the frame at exactly `t`; false when the element's gone. fn frame(&mut self, t: f64, tx: &Sender) -> bool { match &mut self.source { - Source::Video(dec) => tx.send(dec.frame_at(t).map_or_else(Msg::Fail, |f| Msg::Frame(t, f))).is_ok(), + Source::Video(dec) => { + self.still = Some(t); + let n = self.halvings; + tx.send(dec.frame_at(t).map_or_else(Msg::Fail, |f| { + let (f, halves) = sent(f, n); + Msg::Frame(t, f, halves) + })) + .is_ok() + } Source::Audio(_) => tx.send(Msg::Time(t)).is_ok(), } } - /// One step of playing: the end (again from the start when looping), or the frame falling due now. + /// One step of playing: the end (the clock again from the start when looping), or the cursor's frame decoded + /// and sent with the moment it is due, once it is within the lead. One frame a step, so an ask is never kept + /// waiting behind more than one decode. fn tick(&mut self, tx: &Sender) -> bool { - let t = self.now(); - if self.duration.is_finite() && t >= self.duration - 0.5 / self.fps { - if self.looping { - self.go(0.0); - return self.frame(0.0, tx); + let mut t = self.now(); + let frame = 1.0 / self.fps; + if self.duration.is_finite() && self.looping && t >= self.duration { + // round again: the frames already sent for this lap were timed on the old clock's end, which is now + if self.lap > 0.0 { + self.lap = 0.0; + } else { + self.cursor = 0; } + self.restart(t - self.duration); + t = self.now(); + } else if self.duration.is_finite() && !self.looping && t >= self.duration - 0.5 * frame { self.halt(); return tx.send(Msg::Ended(self.duration)).is_ok(); } match &self.source { Source::Video(_) => { - let i = (t * self.fps).floor() as i64; - if self.shown == Some(i) { + // a looping source's cursor goes round ahead of the clock + if self.looping && self.duration.is_finite() && self.lap == 0.0 && self.cursor as f64 * frame >= self.duration - 0.5 * frame { + (self.cursor, self.lap) = (0, self.duration); + } + if self.at_end() || self.cursor_time() - self.lead() > t { return true; } - let skipped = self.shown.map_or(0, |s| i - s - 1); - self.shown = Some(i); + // behind the clock by more than a frame: the frames it passed are skipped + if self.cursor_time() + frame < t { + let i = ((t - self.lap) * self.fps).floor() as i64; + self.skipped += i - self.cursor; + self.cursor = i; + } + let (at, due_in) = (self.cursor as f64 * frame, self.cursor_time() - t); let began = Instant::now(); - let ok = self.frame(t, tx); + let Source::Video(dec) = &mut self.source else { return true }; + // asked for mid-frame: a source whose frames aren't exactly on fps's grid gives the frame showing then + let n = self.halvings; + let got = dec.frame_at(at + 0.5 * frame).map(|f| sent(f, n)); + let took = began.elapsed(); + let due = began + Duration::from_secs_f64(due_in.max(0.0)); + self.cursor += 1; if let Some(s) = self.stats.as_mut() { - s.frame(began.elapsed(), skipped, self.fps); + s.frame(took, std::mem::take(&mut self.skipped), self.fps); } - ok + self.still = Some(at); + tx.send(got.map_or_else(Msg::Fail, |(f, halves)| Msg::Due(at, f, due, halves))).is_ok() } Source::Audio(_) => { if self.told.is_some_and(|w| w.elapsed() < Duration::from_millis(100)) { @@ -281,6 +362,23 @@ mod tests { ok.then_some(p) } + /// A halving's pixel is the mean of the four it stands for; an odd last row or column is left out; a picture a + /// pixel wide has none; and a frame is sent whole when none are asked for. + #[test] + fn halvings_are_means_of_four() { + use crate::media_html::stream::{halvings, sent}; + let px: Vec = (0..5 * 3 * 4).map(|i| (i * 3) as u8).collect(); + let h = halvings(5, 3, &px, 3); + assert_eq!(h.iter().map(|(w, h, p)| (*w, *h, p.len())).collect::>(), vec![(2, 1, 8)]); + let mean = |x: usize, c: usize| ((px[x * 4 + c] as u16 + px[(x + 1) * 4 + c] as u16 + px[(5 + x) * 4 + c] as u16 + px[(5 + x + 1) * 4 + c] as u16 + 2) / 4) as u8; + assert_eq!(h[0].2, vec![mean(0, 0), mean(0, 1), mean(0, 2), mean(0, 3), mean(2, 0), mean(2, 1), mean(2, 2), mean(2, 3)]); + let f = raea_io_media::Frame { pts: 0.0, width: 5, height: 3, rgba: px.clone() }; + let (whole, none) = sent(f.clone(), 0); + assert!(whole.rgba == px && none.is_empty()); + let (size, one) = sent(f, 1); + assert!(size.rgba.is_empty() && (size.width, size.height) == (5, 3) && one == vec![h[0].clone()]); + } + /// Played on the wall's clock (RAEA_AUDIO_MUTE=1: no device), a video's frames come as they fall due and it ends /// at its end; a seek answers with the frame at exactly its time; looping, it goes round; paused, it stops. #[test] @@ -290,7 +388,7 @@ mod tests { let Some(path) = clip() else { return eprintln!("no ffmpeg: skipped") }; let mut st = stream::open(path.to_str().unwrap(), true).expect("a stream"); assert!(matches!(next(&st, Duration::from_secs(10)), Some(Msg::Meta { w: 160, h: 90, .. }))); - assert!(matches!(next(&st, Duration::from_secs(5)), Some(Msg::Frame(t, _)) if t == 0.0)); + assert!(matches!(next(&st, Duration::from_secs(5)), Some(Msg::Frame(t, ..)) if t == 0.0)); let began = Instant::now(); st.ask(Cmd::Play { at: 0.0, looping: false }); @@ -298,9 +396,12 @@ mod tests { let (mut frames, mut last, mut ended) = (0, -1.0, None); while let Some(m) = next(&st, Duration::from_secs(3)) { match m { - Msg::Frame(t, f) => { + Msg::Due(t, f, due, _) => { assert!(t > last, "frames in order: {t} after {last}"); assert_eq!((f.width, f.height), (160, 90)); + // each is due when its time comes on the clock (the wall's, here), sent before then + let on_clock = due.duration_since(began).as_secs_f64(); + assert!((on_clock - t).abs() < 0.06, "the frame at {t} is due at {on_clock}"); (frames, last) = (frames + 1, t); } Msg::Ended(t) => { @@ -313,17 +414,27 @@ mod tests { let (end, took) = ended.expect("it ended"); assert!((end - 1.0).abs() < 0.05, "ended at its duration: {end}"); assert!((0.85..1.4).contains(&took), "a second's clip took {took} s"); - assert!(frames >= 20, "{frames} frames of 30"); + assert!(frames >= 25, "{frames} frames of 30"); st.ask(Cmd::Seek(0.5)); - assert!(matches!(next(&st, Duration::from_secs(2)), Some(Msg::Frame(t, _)) if t == 0.5)); + assert!(matches!(next(&st, Duration::from_secs(2)), Some(Msg::Frame(t, ..)) if t == 0.5)); + let looped = Instant::now(); st.ask(Cmd::Play { at: 0.5, looping: true }); - let (mut wrapped, mut last, until) = (false, 0.5, Instant::now() + Duration::from_millis(1200)); + let (mut wrapped, mut last, mut laps, until) = (false, 0.5, 0.0, Instant::now() + Duration::from_millis(2300)); while Instant::now() < until { match next(&st, Duration::from_millis(200)) { - Some(Msg::Frame(t, _)) => { - wrapped |= t < last; + Some(Msg::Due(t, _, due, _)) => { + // the first frame of the next lap is sent before the last frame of this one has gone + if t < last { + wrapped = true; + laps += 1.0; + assert!(due > Instant::now(), "the loop's first frame came after it was due"); + } + // round the loop a frame is still due when the clock is at its time: half a second of play + // to the end, then a second a lap + let on_clock = due.duration_since(looped).as_secs_f64(); + assert!((on_clock - (t - 0.5 + laps)).abs() < 0.06, "lap {laps}: the frame at {t} is due {on_clock} s after play"); last = t; } Some(Msg::Ended(_)) => panic!("a looping source doesn't end"), @@ -333,13 +444,20 @@ mod tests { assert!(wrapped, "looping went round"); st.ask(Cmd::Pause); - let mut after = 0; - while let Some(m) = next(&st, Duration::from_millis(300)) { - if matches!(m, Msg::Frame(..)) { - after += 1; + // paused, asked for two halvings: the picture showing comes again as its size and its second halving + while next(&st, Duration::from_millis(300)).is_some() {} + st.ask(Cmd::Halvings(2)); + match next(&st, Duration::from_secs(2)) { + Some(Msg::Frame(_, f, halves)) => { + assert_eq!((f.width, f.height), (160, 90)); + assert!(f.rgba.is_empty(), "the frame's own pixels stay behind"); + assert_eq!(halves.iter().map(|(w, h, px)| (*w, *h, px.len())).collect::>(), vec![(40, 22, 40 * 22 * 4)]); } + _ => panic!("no frame with its halvings"), } - assert!(after <= 1, "{after} frames after pausing"); + // what was on its way comes; then nothing more + while next(&st, Duration::from_millis(300)).is_some() {} + assert!(next(&st, Duration::from_millis(300)).is_none(), "frames after pausing"); let _ = std::fs::remove_file(path); } } diff --git a/systems/page-ops/src/lib.rs b/systems/page-ops/src/lib.rs index d65d4deb..0985ce8d 100644 --- a/systems/page-ops/src/lib.rs +++ b/systems/page-ops/src/lib.rs @@ -434,3 +434,5 @@ pub fn menu_items(d: &BaseDocument, menu: NodeId) -> Vec { /// Everything about an element made again (blitz-dom's ALL_DAMAGE, crate-private there): what loading an image does. pub const ALL_DAMAGE: style::selector_parser::RestyleDamage = style::selector_parser::RestyleDamage::from_bits_retain(0b_0000_0000_1110_1111); +/// Only what an element is painted with changed, its box as it was: a video's next frame, the same size as its last. +pub const REPAINT_DAMAGE: style::selector_parser::RestyleDamage = style::selector_parser::RestyleDamage::REPAINT; diff --git a/systems/page/src/lib.rs b/systems/page/src/lib.rs index 5b2c19a5..27b97add 100644 --- a/systems/page/src/lib.rs +++ b/systems/page/src/lib.rs @@ -314,7 +314,13 @@ pub fn iframes() -> bool { std::env::var_os("RAEA_PAGE_IFRAMES").is_none_or(|v| v != "0") } -pub use ops::ALL_DAMAGE; +pub use ops::{ALL_DAMAGE, REPAINT_DAMAGE}; + +/// A picture's halvings handed to the painter ready made (blitz-paint preload_mips): a video's frame is then drawn +/// from the one nearest its drawn size. On the page's thread, where it is painted. +pub fn preload_mips(image: &blitz_dom::node::RasterImageData, levels: Vec<(u32, u32, Vec)>) { + blitz_paint::preload_mips(image, levels.into_iter().map(|(w, h, px)| blitz_dom::node::RasterImageData::new(w, h, std::sync::Arc::new(px))).collect()); +} /// A document for a page `w` x `h` on its own entity: html, head and a body with no margin and no background (the /// page is drawn over 3D and video), which is the root the composition's elements go in. (host.rs opens a root's so.) diff --git a/vendor/blitz-paint/src/lib.rs b/vendor/blitz-paint/src/lib.rs index 952b58cf..099cf575 100644 --- a/vendor/blitz-paint/src/lib.rs +++ b/vendor/blitz-paint/src/lib.rs @@ -36,6 +36,8 @@ pub struct TilePattern { pub tile: kurbo::Size, } +pub use render::preload_mips; + /// raea: the node whose box or text the paint calls going out now draw (a glyph run, the element its text is in), for /// a painter that tells nodes apart: the layer pass's ID scene (raea-sys-layers, docs/review.feature R2b). None outside /// paint_scene. diff --git a/vendor/blitz-paint/src/render.rs b/vendor/blitz-paint/src/render.rs index 1d37ee1a..92329ca8 100644 --- a/vendor/blitz-paint/src/render.rs +++ b/vendor/blitz-paint/src/render.rs @@ -1516,12 +1516,49 @@ fn halve(src: &RasterImageData) -> RasterImageData { RasterImageData::new(w, h, Arc::new(out)) } -/// The level of `image` to sample when it is drawn at `scale` of its size: the image itself, or a halving of it -fn mip_level(image: &RasterImageData, scale: f64) -> RasterImageData { +/// raea: halvings of an image handed over ready made, largest first: a video's frame, halved on its decoder's thread. +/// Not every halving need be there, and the image itself may come without its pixels (only its size): it is then +/// drawn from these alone (video_level). Kept while the image lives, as the ones made here are. +pub fn preload_mips(image: &RasterImageData, levels: Vec) { + MIPS.with_borrow_mut(|mips| { + mips.retain(|_, (weak, _)| weak.upgrade().is_some()); + mips.insert(image.data.id(), (image.data.downgrade(), Some(levels))); + }); +} + +/// How many times `image` is halved to be sampled at `scale` of its size: the most that leaves it at least the +/// size it is drawn. +fn halvings_at(image: &RasterImageData, scale: f64) -> usize { let mut want = 0; while scale * f64::from(1u32 << (want + 1)) <= 1.0 && image.width >> (want + 1) >= 1 && image.height >> (want + 1) >= 1 { want += 1; } + want +} + +/// A video's frame as it is drawn at `scale` of its size: nothing is halved for it here (it is a new picture each +/// time), so of the halvings it came with (preload_mips) the smallest still at least the size mip_level would +/// sample, else the largest. The frame itself when it came with none, or is drawn at over half its size and has +/// its pixels. +fn video_level(image: &RasterImageData, scale: f64) -> RasterImageData { + let whole = image.data.data().len() >= image.width as usize * image.height as usize * 4; + let want = halvings_at(image, scale); + if want == 0 && whole { + return image.clone(); + } + MIPS.with_borrow(|mips| match mips.get(&image.data.id()) { + Some((_, Some(levels))) if !levels.is_empty() => { + let least = (image.width >> want).max(1); + levels.iter().rev().find(|l| l.width >= least).unwrap_or(&levels[0]).clone() + } + _ => image.clone(), + }) +} + +/// The level of `image` to sample when it is drawn at `scale` of its size: the image itself, or a halving of it, +/// made now if it isn't there yet (an img's, once). +fn mip_level(image: &RasterImageData, scale: f64) -> RasterImageData { + let want = halvings_at(image, scale); if want == 0 { return image.clone(); } @@ -2385,10 +2422,20 @@ impl ElementCx<'_, '_> { } } // raea: a minified is drawn from its halving nearest its drawn size (mip_level); a