Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion io/home/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,9 @@ pub fn src(p: &std::path::Path) -> String {
src_on(&p.display().to_string(), cfg!(windows))
}

fn src_on(s: &str, windows: bool) -> String {
/// `src` of the path `s` as Windows writes it (`windows`) or as Unix does, whichever system asks (a test of Windows'
/// form runs on the Mac too)
pub fn src_on(s: &str, windows: bool) -> String {
if !windows {
return s.to_string();
}
Expand Down
9 changes: 8 additions & 1 deletion io/media/src/mezz.rs
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,13 @@ const INTRA_QP: u8 = 16;
/// all-intra H.264 at `part`: the copy's frame k is the source's frame at the range's start + k / fps.
#[cfg(all(feature = "h264", not(target_vendor = "apple")))]
fn encode(src: &Path, part: &Path, range: Option<(usize, usize)>, p: &Probe) -> R<()> {
// a frame too detailed for openh264's buffer at INTRA_QP (Native:3: noise, some 1.6 MB a 1080p frame) is made
// again at QP 26, which every copy had until openh264 took the quantiser asked for (#28)
encode_at(src, part, range, p, INTRA_QP).or_else(|e| if e.contains("Native:3") { encode_at(src, part, range, p, 26) } else { Err(e) })
}

#[cfg(all(feature = "h264", not(target_vendor = "apple")))]
fn encode_at(src: &Path, part: &Path, range: Option<(usize, usize)>, p: &Probe, qp: u8) -> R<()> {
let (t0, t1) = match range {
Some((a, b)) => (a as f64 * p.frame_secs(), (b + 1) as f64 * p.frame_secs()),
None => (0.0, p.secs),
Expand All @@ -94,7 +101,7 @@ fn encode(src: &Path, part: &Path, range: Option<(usize, usize)>, p: &Probe) ->
}
let fps = if info.fps > 0.0 { info.fps } else { p.fps };
let opts = crate::EncodeOpts { width: info.width, height: info.height, fps, kbps: 0, quality: None, nv12: false, realtime: false };
let mut enc = crate::soft::h264_encoder(part, &opts, Some(INTRA_QP))?;
let mut enc = crate::soft::h264_encoder(part, &opts, Some(qp))?;
let n = ((t1 - t0) * fps).round().max(1.0) as usize;
for k in 0..n {
enc.push(&dec.frame_at(t0 + k as f64 / fps)?.rgba)?;
Expand Down
10 changes: 4 additions & 6 deletions io/soft/src/h264.rs
Original file line number Diff line number Diff line change
Expand Up @@ -224,9 +224,9 @@ pub fn mux_h264(
})
}

/// An encoder for one GOP that depends on nothing before it: fixed quantiser (no rate control state carried
/// between frames), the first frame an IDR with its SPS/PPS, no other keyframes. Two GOPs made from the same frames
/// are the same bytes, so an export can be stitched from GOPs made at different times.
/// An encoder for one GOP that depends on nothing before it: fixed quantiser (fixed_qp: the rate control can't move
/// it, so no state is carried between frames), the first frame an IDR with its SPS/PPS, no other keyframes. Two GOPs
/// made from the same frames are the same bytes, so an export can be stitched from GOPs made at different times.
#[cfg(feature = "h264")]
pub struct GopEncoder {
pub(crate) enc: H264Encoder,
Expand All @@ -239,8 +239,6 @@ impl GopEncoder {
pub fn new(w: u32, h: u32, fps: f64, qp: u8) -> R<GopEncoder> {
let config = EncoderConfig::new()
.max_frame_rate(FrameRate::from_hz(fps as f32))
.rate_control_mode(RateControlMode::Off)
.qp(openh264::encoder::QpRange::new(qp, qp))
.skip_frames(false)
.scene_change_detect(false)
.adaptive_quantization(false)
Expand All @@ -251,7 +249,7 @@ impl GopEncoder {
.num_threads(1)
// BT.709 limited range, as h264_encoder tags its stream
.vui(openh264::encoder::VuiConfig::bt709());
let enc = H264Encoder::with_api_config(OpenH264API::from_source(), config)
let enc = H264Encoder::with_api_config(OpenH264API::from_source(), crate::fixed_qp(config, qp))
.map_err(|e| format!("openh264: {e}"))?;
Ok(GopEncoder {
enc,
Expand Down
12 changes: 11 additions & 1 deletion io/soft/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -259,6 +259,16 @@ pub fn quality_kbps(o: &EncodeOpts) -> u32 {
}
}

/// openh264 at a fixed quantiser `qp`: its Quality mode with the QP range pinned (#28, tests/quality.rs). With rate control
/// off it encodes at its default QP 26 whatever the range says (the crate never sets the layer's iDLayerQp). With it
/// on, it refuses a bitrate of 0 and a pinned QP below 14 (Native:3; 13 passes all-intra but not with P-frames), and
/// the bitrate moves nothing once the QP is pinned. A frame too big for its buffer at that QP is Native:3 too.
#[cfg(feature = "h264")]
pub(crate) fn fixed_qp(config: EncoderConfig, qp: u8) -> EncoderConfig {
let qp = qp.clamp(14, 51);
config.rate_control_mode(RateControlMode::Quality).bitrate(BitRate::from_bps(50_000_000)).qp(openh264::encoder::QpRange::new(qp, qp))
}

/// openh264's encoder at the options' bitrate (quality_kbps), or all-intra at a fixed quantiser when `intra_qp` is set (every frame an IDR: an optimized copy, raea_io_media::mezz).
#[cfg(feature = "h264")]
pub fn h264_encoder(out: &Path, o: &EncodeOpts, intra_qp: Option<u8>) -> R<Box<dyn VideoEncoder>> {
Expand All @@ -272,7 +282,7 @@ pub fn h264_encoder(out: &Path, o: &EncodeOpts, intra_qp: Option<u8>) -> R<Box<d
// untagged, ffmpeg and other players read it as BT.601 and shift reds and greens
.vui(openh264::encoder::VuiConfig::bt709());
let config = match intra_qp {
Some(qp) => config.rate_control_mode(RateControlMode::Off).qp(openh264::encoder::QpRange::new(qp, qp)).intra_frame_period(openh264::encoder::IntraFramePeriod::from_num_frames(1)),
Some(qp) => fixed_qp(config, qp).intra_frame_period(openh264::encoder::IntraFramePeriod::from_num_frames(1)),
None => config,
};
let enc = H264Encoder::with_api_config(OpenH264API::from_source(), config).map_err(|e| format!("openh264: {e}"))?;
Expand Down
84 changes: 61 additions & 23 deletions io/soft/tests/quality.rs
Original file line number Diff line number Diff line change
@@ -1,34 +1,40 @@
//! #28 (3): `--quality` reaches openh264. It ignored EncodeOpts.quality, so draft and master gave byte-identical files
//! (Windows' and Linux's encoder); now quality scales its bitrate (quality_kbps).
//! What openh264 is asked for reaches what it writes (#28; Windows' and Linux's encoder):
//! - (3) `--quality`: it ignored EncodeOpts.quality, so draft and master gave byte-identical files; now quality scales
//! its bitrate (quality_kbps).
//! - (2) a fixed quantiser: the mezz copy's all-intra QP (h264_encoder's `intra_qp`) and a GOP's (GopEncoder). With
//! rate control off, as both were, openh264 wrote QP 26 whatever was asked: QP 12 and QP 40 gave the same bytes.
#![cfg(feature = "h264")]

use raea_io_codec::EncodeOpts;
use raea_io_soft::h264_encoder;
use raea_io_soft::{GopEncoder, h264_encoder};

const W: u32 = 640;
const H: u32 = 360;
/// soft rings moving across, `w`x`h`, frame `k`: detailed enough that the bitrate or the quantiser decides the size,
/// not noise that every rate leaves at openh264's coarsest quantiser
fn rings(w: u32, h: u32, ring: i64, k: u32) -> Vec<u8> {
(0..w * h).flat_map(|i| {
let (x, y) = (i % w, i / w);
let (dx, dy) = (x as i64 - w as i64 / 2 + k as i64 * 4, y as i64 - h as i64 / 2);
let v = ((dx * dx + dy * dy) / ring % 256) as u8;
[v, v.wrapping_mul(3), 255 - v, 255]
}).collect()
}

/// 90 frames (three seconds, so the rate outweighs the first frame) of a moving pattern of soft rings: detailed enough that the bitrate decides the size, not noise that
/// every rate leaves at openh264's coarsest quantiser
fn encode(quality: Option<f32>) -> u64 {
let dir = std::env::temp_dir().join(format!("raea-soft-quality-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let out = dir.join(format!("{quality:?}.mp4"));
let opts = EncodeOpts { width: W, height: H, fps: 30.0, kbps: 2000, quality, nv12: false, realtime: false };
let mut enc = h264_encoder(&out, &opts, None).unwrap();
for k in 0..90u32 {
let px: Vec<u8> = (0..W * H).flat_map(|i| {
let (x, y) = (i % W, i / W);
let (dx, dy) = (x as i64 - 320 + k as i64 * 4, y as i64 - 180);
let v = ((dx * dx + dy * dy) / 23 % 256) as u8;
[v, v.wrapping_mul(3), 255 - v, 255]
}).collect();
enc.push(&px).unwrap();
/// The size of `n` frames of rings encoded to an MP4 by h264_encoder
fn mp4(o: &EncodeOpts, intra_qp: Option<u8>, ring: i64, n: u32) -> u64 {
let out = std::env::temp_dir().join(format!("raea-soft-quality-{}-{:?}-{intra_qp:?}.mp4", std::process::id(), o.quality));
let mut enc = h264_encoder(&out, o, intra_qp).unwrap();
for k in 0..n {
enc.push(&rings(o.width, o.height, ring, k)).unwrap();
}
enc.finish(None).unwrap();
let n = std::fs::metadata(&out).unwrap().len();
let size = std::fs::metadata(&out).unwrap().len();
let _ = std::fs::remove_file(&out);
n
size
}

/// 90 frames of 640x360: three seconds, so the rate outweighs the first frame
fn encode(quality: Option<f32>) -> u64 {
mp4(&EncodeOpts { width: 640, height: 360, fps: 30.0, kbps: 2000, quality, nv12: false, realtime: false }, None, 23, 90)
}

#[test]
Expand All @@ -37,3 +43,35 @@ fn quality_changes_what_openh264_writes() {
// before, all three were the same file
assert!(draft * 5 / 4 < rate && rate * 5 / 4 < master, "draft {draft}, high {rate} and master {master} bytes: a higher quality spends more");
}

const W: u32 = 320;
const H: u32 = 192;

fn intra(qp: u8) -> u64 {
mp4(&EncodeOpts { width: W, height: H, fps: 30.0, kbps: 0, quality: None, nv12: false, realtime: false }, Some(qp), 13, 6)
}

fn gop(qp: u8) -> Vec<u8> {
let mut enc = GopEncoder::new(W, H, 30.0, qp).unwrap();
(0..6).flat_map(|k| enc.push(&rings(W, H, 13, k)).unwrap().0).collect()
}

#[test]
fn the_mezz_copys_quantiser_changes_what_openh264_writes() {
let (fine, coarse) = (intra(16), intra(40));
assert!(coarse * 2 < fine, "QP 16 {fine} bytes, QP 40 {coarse}: a finer quantiser spends more");
}

#[test]
fn a_gops_quantiser_changes_what_openh264_writes() {
let (fine, coarse) = (gop(16).len(), gop(40).len());
assert!(coarse * 2 < fine, "QP 16 {fine} bytes, QP 40 {coarse}: a finer quantiser spends more");
// and a GOP is still the same bytes each time it's made (an export stitched from GOPs made at different times)
assert!(gop(16) == gop(16));
}

#[test]
fn a_quantiser_below_openh264s_floor_is_its_floor() {
// openh264 refuses a pinned QP under 14 with rate control on (Native:3); asked for less, it gets 14
assert!(gop(4) == gop(14));
}
9 changes: 7 additions & 2 deletions runtime/bundle/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,13 @@ mod outside;
mod pid;
mod rewrite;
pub mod styles;
mod urls;
pub use cache::*;
pub use entry::*;
pub use fresh::*;
pub use outside::*;
pub use rewrite::*;
pub use urls::*;
pub use raea_core_scene::media_path;
#[cfg(test)]
mod tests;
Expand All @@ -39,7 +41,9 @@ pub fn brand_css(dir: &Path, w: u32) -> Option<String> {
let f = here.join("brand").join("brand.css");
if let Ok(css) = std::fs::read_to_string(&f) {
let kept: Vec<&str> = css.lines().filter(|l| !l.trim_start().starts_with("@import")).collect();
return Some(format!("{}\n:root {{ --brand-scale: {}; }}\n", kept.join("\n"), 2.5 * w as f64 / 1080.0));
// its fonts and images as files: /media/ from the edit, the rest from brand/ (urls.rs)
let kept = css_file_urls(&kept.join("\n"), dir, &here.join("brand"), |_| ());
return Some(format!("{kept}\n:root {{ --brand-scale: {}; }}\n", 2.5 * w as f64 / 1080.0));
}
d = here.parent();
}
Expand All @@ -55,7 +59,8 @@ pub fn bundle_css(js: &str) -> Option<String> {

/// A page's own CSS: the space's brand kit (brand_css), then the CSS the bundle `js` imported (bundle_css)
pub fn page_css(dir: &Path, w: u32, js: &Path) -> Option<String> {
let bundled = std::fs::read_to_string(js).ok().and_then(|t| bundle_css(&t));
// its url()s as files from the edit's folder (urls.rs), where the CSS files it imports are found
let bundled = std::fs::read_to_string(js).ok().and_then(|t| bundle_css(&t)).map(|c| css_file_urls(&c, dir, dir, |_| ()));
match (brand_css(dir, w), bundled) {
(Some(b), Some(c)) => Some(format!("{b}\n{c}")),
(b, c) => b.or(c),
Expand Down
85 changes: 85 additions & 0 deletions runtime/bundle/src/urls.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
//! A stylesheet's `url()`s that name the composition's files, written as what the page reads them by: a path on the Mac
//! and Linux, a file URL on Windows (raea_io_home::src). Written raw, a Windows path breaks twice in CSS: its
//! backslashes are escapes (`\A` a newline, `\c` and the hex digits after it a character) and `C:` a URL scheme (#28).

use std::path::Path;

/// `css` with each `url(..)` naming a file (not a URL with a scheme, a data: URI or a `#fragment`) as that file, quoted:
/// `/media/...` (staticFile's) found from the edit's folder `media` (media_path), anything else from `rel`, the folder
/// the stylesheet's own relative URLs start from. `missing` hears of each file that isn't there.
pub fn css_file_urls(css: &str, media: &Path, rel: &Path, missing: impl FnMut(&Path)) -> String {
urls_as(css, media, rel, missing, raea_io_home::src)
}

/// css_file_urls with each file written by `src` (a test writes Windows' form on any system)
fn urls_as(css: &str, media: &Path, rel: &Path, mut missing: impl FnMut(&Path), src: impl Fn(&Path) -> String) -> String {
if !css.contains("url(") {
return css.to_string();
}
let mut out = String::with_capacity(css.len());
let mut rest = css;
while let Some(at) = rest.find("url(") {
out.push_str(&rest[..at + 4]);
rest = &rest[at + 4..];
let Some(end) = rest.find(')') else { break };
let inner = rest[..end].trim();
let quote = inner.chars().next().filter(|c| *c == '"' || *c == '\'');
let url = quote.map_or(inner, |q| inner.trim_matches(q));
if url.is_empty() || url.contains("://") || url.starts_with("data:") || url.starts_with('#') {
out.push_str(&rest[..end]);
} else {
let file = raea_core_scene::media_path(if url.starts_with("/media/") { media } else { rel }, url);
if !file.exists() {
missing(&file);
}
out.push_str(&format!("\"{}\"", src(&file)));
}
rest = &rest[end..];
}
out.push_str(rest);
out
}

#[cfg(test)]
mod tests {
use std::path::Path;

/// css_file_urls as each path's look says it's written: Windows' form for a C:\ path, so it's checked on any system
fn css_file_urls(css: &str, media: &Path, rel: &Path, missing: impl FnMut(&Path)) -> String {
let src = |p: &Path| raea_io_home::src_on(&p.display().to_string(), p.display().to_string().as_bytes().get(1) == Some(&b':'));
super::urls_as(css, media, rel, missing, src)
}

#[test]
fn a_windows_path_is_a_file_url_with_no_backslash() {
let dir = Path::new(r"C:\Users\a\AppData\Local\Temp\claude\edit");
let css = r#"@font-face { font-family: X; src: url("/media/fonts/x.woff2") format("woff2"); } .a { background: url(bg.png) }"#;
let mut gone = Vec::new();
let out = css_file_urls(css, dir, dir, |p| gone.push(p.to_path_buf()));
assert!(!out.contains('\\'), "{out}");
assert!(out.contains(r#"url("file:///C:/Users/a/AppData/Local/Temp/claude/edit/"#), "{out}");
assert!(out.contains(r#"fonts/x.woff2") format("woff2")"#) && out.contains(r#"bg.png") }"#), "{out}");
assert_eq!(gone.len(), 2);
}

#[test]
fn urls_naming_no_file_stay() {
let dir = std::env::temp_dir();
let css = r#"a { b: url(https://x.io/f.woff2), url('data:font/woff2;base64,AA'), url(#clip), url() }"#;
assert_eq!(css_file_urls(css, &dir, &dir, |_| panic!("no file named")), css);
assert_eq!(css_file_urls("a { color: red }", &dir, &dir, |_| ()), "a { color: red }");
}

#[test]
fn a_relative_url_starts_from_the_stylesheets_folder_and_media_from_the_edits() {
let edit = std::env::temp_dir().join(format!("raea-urls-{}", std::process::id()));
let brand = edit.join("brand");
std::fs::create_dir_all(edit.join("media")).unwrap();
std::fs::create_dir_all(&brand).unwrap();
std::fs::write(edit.join("media/m.woff2"), b"").unwrap();
std::fs::write(brand.join("b.woff2"), b"").unwrap();
let out = css_file_urls("url(/media/m.woff2) url(b.woff2)", &edit, &brand, |p| panic!("{p:?} is there"));
let _ = std::fs::remove_dir_all(&edit);
assert_eq!(out, format!("url(\"{}\") url(\"{}\")", edit.join("media/m.woff2").display(), brand.join("b.woff2").display()));
}
}
53 changes: 6 additions & 47 deletions runtime/page/src/elements.rs
Original file line number Diff line number Diff line change
Expand Up @@ -151,30 +151,10 @@ pub(crate) fn walk(react: &World, e: Entity, parent: u64, at: At, dir: &Path, ho
out.order.insert(parent, list);
}

/// A stylesheet's `url()`s of the composition's media (`/media/...`, staticFile's) as the files' paths, which
/// raea-runtime-js's @font-face and backgrounds read; other text as it is.
/// A stylesheet's `url()`s that name the composition's files as what raea-runtime-js's @font-face and backgrounds read
/// (raea-runtime-bundle's css_file_urls: a file URL on Windows, #28), relative ones from the edit's folder.
pub(crate) fn media_urls(text: &str, dir: &Path) -> String {
if !text.contains("url(") || !text.contains("/media/") {
return text.to_string();
}
let mut out = String::with_capacity(text.len());
let mut rest = text;
while let Some(i) = rest.find("url(") {
out.push_str(&rest[..i + 4]);
rest = &rest[i + 4..];
let Some(j) = rest.find(')') else { break };
let inner = rest[..j].trim();
let q = inner.chars().next().filter(|c| *c == '"' || *c == '\'');
let url = q.map_or(inner, |c| inner.trim_matches(c));
if url.starts_with("/media/") {
out.push_str(&format!("\"{}\"", raea_runtime_react::scene::media_path(dir, url).display()));
} else {
out.push_str(&rest[..j]);
}
rest = &rest[j..];
}
out.push_str(rest);
out
raea_runtime_react::scene::css_file_urls(text, dir, dir, |_| ())
}

/// A <Video>'s crop (raea-page's `fit_rect`) at source time `at`, from its framing="face" and punchIn (motion.feature,
Expand Down Expand Up @@ -256,31 +236,10 @@ pub(crate) fn page_props(kind: &str, props: Option<&Map<String, Value>>, dir: &P
out
}

/// A CSS value with each `url(..)` that names a media file (not a URL with a scheme, nor a data: URI) turned into
/// that file's path, as an img's src is (a background or a border-image of staticFile("x.png")).
/// A CSS value with each `url(..)` that names a media file as that file, as an img's src is (a background or a
/// border-image of staticFile("x.png")); one that isn't there refuses the render.
fn css_urls(dir: &Path, s: &str) -> String {
let mut out = String::with_capacity(s.len());
let mut rest = s;
while let Some(at) = rest.find("url(") {
out.push_str(&rest[..at + 4]);
rest = &rest[at + 4..];
let Some(end) = rest.find(')') else { break };
let inner = rest[..end].trim();
let quote = inner.chars().next().filter(|c| *c == '"' || *c == '\'');
let path = quote.map_or(inner, |q| inner.trim_matches(q));
if path.contains("://") || path.starts_with("data:") || path.is_empty() {
out.push_str(&rest[..end]);
} else {
let file = raea_runtime_react::scene::media_path(dir, path);
if !file.exists() {
REFUSED.with(|c| c.borrow_mut().push(format!("missing image: {}", file.display())));
}
out.push_str(&format!("\"{}\"", raea_io_home::src(&file)));
}
rest = &rest[end..];
}
out.push_str(rest);
out
raea_runtime_react::scene::css_file_urls(s, dir, dir, |f| REFUSED.with(|c| c.borrow_mut().push(format!("missing image: {}", f.display()))))
}

#[cfg(test)]
Expand Down
Loading
Loading