Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion engine/page/host/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,10 @@ fn main() {
);
// on screen, bevy_winit's event loop is the runner (a ScheduleRunner here replaced it: no window, 11-04)
if !a.window && !app.is_plugin_added::<ScheduleRunnerPlugin>() {
app.add_plugins(ScheduleRunnerPlugin::run_loop(Duration::from_secs_f64(1.0 / 60.0)));
// 60 frames a second, as most screens; RAEA_PAGE_FPS stands in for another screen's rate (a 120 Hz one, to
// measure how a 60 fps video plays where the window draws twice a frame)
let fps = std::env::var("RAEA_PAGE_FPS").ok().and_then(|v| v.parse::<f64>().ok()).filter(|f| (1.0..=480.0).contains(f)).unwrap_or(60.0);
app.add_plugins(ScheduleRunnerPlugin::run_loop(Duration::from_secs_f64(1.0 / fps)));
}
app.add_plugins(HostPlugin { size: a.size, double_click: Duration::from_millis(500), picking })
.add_plugins(brp::plugins(brp::port(), |r| r))
Expand Down
40 changes: 40 additions & 0 deletions io/apple/src/convert.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
//! VideoToolbox's BGRA as the RGBA a Frame holds.

/// A row of BGRA as RGBA, its alpha kept or made 255. Four bytes at a time as one word, which the compiler turns into
/// vector code: a 4K frame took 5.5 ms a byte at a time, most of what playing one cost (RAEA_MEDIA_STATS, 11-13).
pub(crate) fn bgra_to_rgba(src: &[u8], dst: &mut [u8], alpha: bool) {
let opaque = if alpha { 0 } else { 0xff00_0000 };
for (d, s) in dst.chunks_exact_mut(4).zip(src.chunks_exact(4)) {
let v = u32::from_le_bytes([s[0], s[1], s[2], s[3]]);
let out = (v & 0xff00_ff00) | ((v & 0xff) << 16) | ((v >> 16) & 0xff) | opaque;
d.copy_from_slice(&out.to_le_bytes());
}
}

#[cfg(test)]
mod tests {
use super::bgra_to_rgba;

#[test]
fn bgra_becomes_rgba_with_its_alpha_kept_or_made_opaque() {
let src = [10, 20, 30, 40, 1, 2, 3, 0];
let mut out = [0u8; 8];
bgra_to_rgba(&src, &mut out, true);
assert_eq!(out, [30, 20, 10, 40, 3, 2, 1, 0]);
bgra_to_rgba(&src, &mut out, false);
assert_eq!(out, [30, 20, 10, 255, 3, 2, 1, 255]);
}

/// A 4K frame's worth, timed (cargo test -p raea-io-apple --release convert -- --nocapture).
#[test]
fn a_4k_frame_is_converted_in_a_frame_s_small_part() {
let src: Vec<u8> = (0..3840 * 2160 * 4).map(|i| (i % 251) as u8).collect();
let mut out = vec![0u8; src.len()];
let t = std::time::Instant::now();
for _ in 0..10 {
bgra_to_rgba(&src, &mut out, false);
}
eprintln!("4K BGRA to RGBA: {:.2} ms a frame", t.elapsed().as_secs_f64() * 100.0);
assert_eq!(&out[..4], &[src[2], src[1], src[0], 255]);
}
}
8 changes: 1 addition & 7 deletions io/apple/src/decode.rs
Original file line number Diff line number Diff line change
Expand Up @@ -213,13 +213,7 @@ impl Decoder {
let mut rgba = vec![0u8; w * h * 4];
for y in 0..h {
let row = std::slice::from_raw_parts(base.add(y * bpr), w * 4);
let dst = &mut rgba[y * w * 4..(y + 1) * w * 4];
for x in 0..w {
dst[x * 4] = row[x * 4 + 2];
dst[x * 4 + 1] = row[x * 4 + 1];
dst[x * 4 + 2] = row[x * 4];
dst[x * 4 + 3] = if self.alpha { row[x * 4 + 3] } else { 255 };
}
crate::convert::bgra_to_rgba(row, &mut rgba[y * w * 4..(y + 1) * w * 4], self.alpha);
}
rgba
});
Expand Down
1 change: 1 addition & 0 deletions io/apple/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
#![cfg(target_vendor = "apple")]

mod audio;
mod convert;
mod decode;
mod encode;
mod vt;
Expand Down
133 changes: 122 additions & 11 deletions runtime/dom/media/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,18 @@
//! A seek is the prop `__seek` ([time, n], set by the dom layer's `currentTime`); playing is `__play` ([on, time, n,
//! loop], set by `play()` and `pause()`), `__muted` the element's muted and `__volume` its volume. Playing, the decoder's frames come on the
//! source's clock (its sound's: raea-runtime-js media_play.rs), each a `time` news for the element's currentTime, with
//! `timeupdate` four times a second as a browser's, `playing` once it's going, and `ended` at its end. raea-runtime-dom's
//! `timeupdate` four times a second as a browser's, `playing` once it's going, and `ended` at its end. A playing
//! video's frames come ahead of their time, each with the moment it is due: at each sync (the page's own frame) the
//! one due then is shown, so the picture keeps the page's rhythm whatever the decoder's. raea-runtime-dom's
//! page host keeps one `Medias` and syncs it after each commit.

use std::collections::HashMap;
use std::collections::{HashMap, VecDeque};
use std::sync::Arc;
use std::time::{Duration, Instant};

use blitz_dom::BaseDocument;
use blitz_dom::node::{ImageData, RasterImageData, SpecialElementData};
use raea_runtime_js::media_html::stream::{Cmd, Msg, Stream};
use raea_runtime_js::media_html::stream::{Cmd, Halvings, Msg, Stream};
use serde_json::{Value, json};

use raea_sys_page::ops::Nodes;
Expand All @@ -35,6 +38,14 @@ pub struct Media {
duration: f64,
/// the latest frame: width, height, pixels
frame: Option<(u32, u32, Arc<Vec<u8>>)>,
/// what it is drawn from when it came as its size alone (its halving), until that goes to the painter with it
halves: Halvings,
/// how many halvings its decoder is asked to send with each frame
halvings: u8,
/// a playing video's frames not due yet: each its time, its picture and when to show it
ahead: VecDeque<(f64, (u32, u32, Vec<u8>), Instant, Halvings)>,
/// when this was last synced, and how far apart syncs have been lately (the page's frame, s)
synced: (Option<Instant>, f64),
/// `__play` as last seen, and whether it asks to play
play: Option<Value>,
playing: bool,
Expand All @@ -47,6 +58,8 @@ pub struct Media {
stats: Option<(std::time::Instant, u32, u32, f64, f64)>,
/// RAEA_MEDIA_STATS=1: when a frame was last shown, and the longest wait between two this second (ms)
shown_gap: (Option<std::time::Instant>, f64),
/// RAEA_MEDIA_STATS=1, this second: the page's frames, the longest (ms), and how late frames were shown (sum, max; ms)
paced: (u32, f64, f64, f64),
}

impl Media {
Expand Down Expand Up @@ -76,7 +89,7 @@ pub fn sync(doc: &mut BaseDocument, nodes: &Nodes, medias: &mut Medias, out: &mu
Some(m) if m.src == src && m.load == load => m,
_ => {
let stream = (!src.is_empty()).then(|| raea_runtime_js::media_html::stream::open(&raea_sys_page::net::page_path(&src), video)).flatten();
medias.insert(id, Media { src, load, poster: None, seek: None, stream, video, loaded: false, seeking: None, seeked: false, duration: f64::NAN, frame: None, play: None, playing: false, muted: None, volume: None, told: None, shown_gap: (None, 0.0), stats: (std::env::var("RAEA_MEDIA_STATS").as_deref() == Ok("1")).then(|| (std::time::Instant::now(), 0, 0, 0.0, 0.0)) });
medias.insert(id, Media { src, load, poster: None, seek: None, stream, video, loaded: false, seeking: None, seeked: false, duration: f64::NAN, frame: None, halves: Vec::new(), halvings: 0, ahead: VecDeque::new(), synced: (None, 1.0 / 60.0), paced: (0, 0.0, 0.0, 0.0), play: None, playing: false, muted: None, volume: None, told: None, shown_gap: (None, 0.0), stats: (std::env::var("RAEA_MEDIA_STATS").as_deref() == Ok("1")).then(|| (std::time::Instant::now(), 0, 0, 0.0, 0.0)) });
medias.get_mut(&id).unwrap()
}
};
Expand Down Expand Up @@ -117,6 +130,7 @@ pub fn sync(doc: &mut BaseDocument, nodes: &Nodes, medias: &mut Medias, out: &mu
st.ask(Cmd::Play { at: p.get(1).and_then(Value::as_f64).unwrap_or(0.0), looping: p.get(3).and_then(Value::as_bool).unwrap_or(false) });
} else {
st.ask(Cmd::Pause);
m.ahead.clear();
}
}
m.play = play;
Expand All @@ -131,9 +145,12 @@ pub fn sync(doc: &mut BaseDocument, nodes: &Nodes, medias: &mut Medias, out: &mu
bevy::log::warn!("raea-page: media: {err}");
out.push(json!({"id": id, "type": "error", "message": err}));
}
Msg::Frame(t, f) => {
Msg::Frame(t, f, halves) => {
// the first frame or a seek's: shown now, and what was on its way for before the seek is not
m.ahead.clear();
let (w, h) = (f.width, f.height);
m.frame = Some((w, h, Arc::new(f.rgba)));
m.halves = halves;
redraw = true;
came += 1;
if !m.loaded {
Expand All @@ -149,6 +166,12 @@ pub fn sync(doc: &mut BaseDocument, nodes: &Nodes, medias: &mut Medias, out: &mu
told(m, id, t, out);
}
}
// a playing video's frame, for later; one that comes after a pause is for nothing
Msg::Due(t, f, due, halves) => {
if m.playing {
m.ahead.push_back((t, (f.width, f.height, f.rgba), due, halves));
}
}
Msg::Time(t) => {
if m.landed(t) {
out.push(json!({"id": id, "type": "seeked", "time": t}));
Expand All @@ -159,13 +182,58 @@ pub fn sync(doc: &mut BaseDocument, nodes: &Nodes, medias: &mut Medias, out: &mu
out.push(json!({"id": id, "type": "time", "time": t}));
}
}
Msg::Playing(t) => out.push(json!({"id": id, "type": "playing", "time": t})),
Msg::Playing(t) => {
m.ahead.clear();
out.push(json!({"id": id, "type": "playing", "time": t}));
}
Msg::Ended(t) => {
m.playing = false;
out.push(json!({"id": id, "type": "ended", "time": t}));
}
}
}
// the frame due at this sync: the latest whose moment has come, or comes within half a page frame (shown a
// touch early rather than a whole page frame late); the ones before it were never seen
let now = Instant::now();
if let Some(last) = m.synced.0 {
let dt = now.duration_since(last).as_secs_f64();
m.synced.1 += 0.1 * (dt.min(0.1) - m.synced.1);
if m.stats.is_some() && !m.ahead.is_empty() {
(m.paced.0, m.paced.1) = (m.paced.0 + 1, m.paced.1.max(dt * 1e3));
}
}
m.synced.0 = Some(now);
let reach = now + Duration::from_secs_f64((0.5 * m.synced.1).min(0.008));
let (due, passed) = due_by(&mut m.ahead, reach);
came += passed;
if let Some((t, (w, h, rgba), at, halves)) = due {
m.halves = halves;
let late = now.saturating_duration_since(at).as_secs_f64() * 1e3;
(m.paced.2, m.paced.3) = (m.paced.2 + late, m.paced.3.max(late));
m.frame = Some((w, h, Arc::new(rgba)));
(redraw, m.seeked) = (true, true);
came += 1;
told(m, id, t, out);
}
// how many halvings its frames should come with: as many as bring one within twice the size it is drawn at
// (the painter's rule, blitz-paint mip_level), counted from its box as last laid out
if m.video && draw && let (Some((w, h, _)), Some(n), Some(st)) = (m.frame.as_ref(), nodes.by_id.get(&id), m.stream.as_mut()) {
let want = doc.get_node(*n).map_or(0, |node| {
let (l, scale) = (node.final_layout(), doc.viewport().scale_f64());
let drawn = (l.content_box_width() as f64 * scale / *w as f64).min(l.content_box_height() as f64 * scale / *h as f64);
let mut n = 0u8;
while drawn > 0.0 && drawn * f64::from(1u32 << (n + 1)) <= 1.0 && n < 6 {
n += 1;
}
n
});
// RAEA_MEDIA_HALVINGS=0: frames drawn as they come, as before (to compare with)
let want = if std::env::var("RAEA_MEDIA_HALVINGS").as_deref() == Ok("0") { 0 } else { want };
if want != m.halvings {
m.halvings = want;
st.ask(Cmd::Halvings(want));
}
}
if redraw && m.video && draw {
let pic = match (&m.poster, m.seeked) {
(Some(p), false) => poster_image(p).map(|(w, h, px)| (w, h, Arc::new(px))),
Expand All @@ -175,7 +243,7 @@ pub fn sync(doc: &mut BaseDocument, nodes: &Nodes, medias: &mut Medias, out: &mu
.or_else(|| m.frame.as_ref().map(|(w, h, px)| (*w, *h, px.clone())));
if let (Some((w, h, px)), Some(n)) = (pic, nodes.by_id.get(&id)) {
let began = std::time::Instant::now();
set_image(doc, *n, w, h, px);
set_image(doc, *n, w, h, px, std::mem::take(&mut m.halves));
if let Some(s) = m.stats.as_mut().filter(|_| came > 0) {
let ms = began.elapsed().as_secs_f64() * 1e3;
(s.1, s.2, s.3, s.4) = (s.1 + 1, s.2 + came - 1, s.3 + ms, s.4.max(ms));
Expand All @@ -185,16 +253,27 @@ pub fn sync(doc: &mut BaseDocument, nodes: &Nodes, medias: &mut Medias, out: &mu
m.shown_gap.0 = Some(began);
let took = s.0.elapsed().as_secs_f64();
if took >= 1.0 {
eprintln!("raea-page: video {took:.2} s: {} frames shown ({w}x{h}), {} passed over before the page drew, picture set avg {:.2} ms max {:.2} ms, longest gap {:.0} ms", s.1, s.2, s.3 / s.1.max(1) as f64, s.4, m.shown_gap.1);
eprintln!("raea-page: video {took:.2} s: {} frames shown ({w}x{h}), {} passed over before the page drew, picture set avg {:.2} ms max {:.2} ms, longest gap {:.0} ms; the page synced {} times, longest apart {:.0} ms; shown late avg {:.1} ms max {:.1} ms", s.1, s.2, s.3 / s.1.max(1) as f64, s.4, m.shown_gap.1, m.paced.0, m.paced.1, m.paced.2 / s.1.max(1) as f64, m.paced.3);
*s = (std::time::Instant::now(), 0, 0, 0.0, 0.0);
m.shown_gap.1 = 0.0;
m.paced = (0, 0.0, 0.0, 0.0);
}
}
}
}
}
}

/// Of the frames waiting, the one to show by `reach`: the latest whose moment has come, taken out with the ones before
/// it, and how many those were (frames nobody saw: the page drew less often than they came).
fn due_by<T>(ahead: &mut VecDeque<(f64, T, Instant, Halvings)>, reach: Instant) -> (Option<(f64, T, Instant, Halvings)>, u32) {
let (mut due, mut passed) = (None, 0);
while ahead.front().is_some_and(|f| f.2 <= reach) {
passed += due.replace(ahead.pop_front().unwrap()).is_some() as u32;
}
(due, passed)
}

/// A playing element's time: each as `time` (its currentTime), and a `timeupdate` four times a second.
fn told(m: &mut Media, id: u64, t: f64, out: &mut Vec<Value>) {
out.push(json!({"id": id, "type": "time", "time": t}));
Expand All @@ -219,12 +298,44 @@ fn poster_image(src: &str) -> Option<(u32, u32, Vec<u8>)> {
Some((size.x, size.y, img.data?))
}

/// Node `n` drawn as an image: `rgba` as its raster, its layout made again for the picture's size.
fn set_image(doc: &mut BaseDocument, n: blitz_dom::NodeId, w: u32, h: u32, rgba: Arc<Vec<u8>>) {
/// Node `n` drawn as an image: `rgba` as its raster. Its layout is made again only when the picture's size isn't the
/// last one's: a playing video's next frame is the same size, and making its box and its ancestors' layout again for
/// each was a part of every page frame it played in.
fn set_image(doc: &mut BaseDocument, n: blitz_dom::NodeId, w: u32, h: u32, rgba: Arc<Vec<u8>>, halves: Halvings) {
let Some(node) = doc.get_node_mut(n) else { return };
let Some(el) = node.element_data_mut() else { return };
el.special_data = SpecialElementData::Image(Box::new(ImageData::Raster(RasterImageData { width: w, height: h, data: vello::peniko::Blob::new(rgba) })));
let same = matches!(&el.special_data, SpecialElementData::Image(i) if matches!(&**i, ImageData::Raster(r) if (r.width, r.height) == (w, h)));
let image = RasterImageData { width: w, height: h, data: vello::peniko::Blob::new(rgba) };
if !halves.is_empty() {
raea_sys_page::preload_mips(&image, halves);
}
el.special_data = SpecialElementData::Image(Box::new(ImageData::Raster(image)));
if same {
return node.insert_damage(raea_sys_page::REPAINT_DAMAGE);
}
node.clear_layout_cache();
node.insert_damage(raea_sys_page::ALL_DAMAGE);
node.mark_ancestors_dirty();
}

#[cfg(test)]
mod tests {
use super::*;

/// At a sync the frame shown is the latest one due by then; earlier ones due are passed over, later ones wait.
#[test]
fn the_frame_due_is_the_latest_whose_moment_has_come() {
let t0 = Instant::now();
let at = |ms: u64| t0 + Duration::from_millis(ms);
let mut ahead: VecDeque<(f64, u8, Instant, Halvings)> = (0..4).map(|i| (i as f64 / 60.0, i as u8, at(17 * i as u64), Vec::new())).collect();
// nothing is due before the first
assert!(due_by(&mut ahead, t0 - Duration::from_millis(1)).0.is_none());
// a page drawing as often as the frames come shows each
assert_eq!(due_by(&mut ahead, at(0)).0.map(|f| f.1), Some(0));
assert_eq!(due_by(&mut ahead, at(17)).0.map(|f| f.1), Some(1));
// a page a frame late shows the latest and counts the one it passed
let (due, passed) = due_by(&mut ahead, at(51));
assert_eq!((due.map(|f| f.1), passed), (Some(3), 1));
assert!(ahead.is_empty());
}
}
Loading
Loading