From 1899d29dff9e2189d6bfb7a94183fb2c57bc70bc Mon Sep 17 00:00:00 2001 From: Zachary Guglin Date: Thu, 19 Nov 2026 13:58:40 -0800 Subject: [PATCH 1/6] raea check says who can fix what it finds, so a round goes only to what another render can change MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two Telluride walkaround projects (11-14, 11-16) took nine renders; five chased findings no render could move. The checks measured right every time; what they didn't say was whose each finding was. Nothing a rule measures, and no line it draws, changes here: what changes is how a finding is told. - The picture's own or the encode's (raea-app-check-cause). A rule an encode can cause is measured again on the composition drawn at the same frames without encoding (raea render --at, out of the record). Blotching is the first: the walkaround's phone footage scored blotchy in 9 of 28 samples before any encoding, so every export failed and no setting cleared it. Now the picture's own is a note, and what the encode added is the error with a fix raea has: --quality high added to 14 of 28 samples (x264 crf 12 moved the score at most 0.005; master added none), so the check says render with --quality master, and the master export then passes with the note. - Carried. A finding an earlier export of the same composition had at the same moment ends "[also in …]": what changed between them didn't move it. Its rank stays as measured. - A source isn't a delivery: a take under assets/ or media/ checked on its own reports its loudness as a note. The text rules (clipped, UI zones, reading time, face, cues, brand kit) move to raea-app-check-text, so raea render can hold its own text to them, and raea-app-check stays under its size limit. Probe gives the video's frame rate. The rendering guide tells agents to fix every error in one pass, not to repeat a change that didn't move a carried finding, and that a note is for the person. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 21 ++- Cargo.toml | 2 + apps/check/Cargo.toml | 5 +- apps/check/cause/Cargo.toml | 15 ++ apps/check/cause/src/lib.rs | 174 ++++++++++++++++++++ apps/check/cause/src/tests.rs | 107 ++++++++++++ apps/check/read/src/lib.rs | 10 +- apps/check/src/lib.rs | 27 +-- apps/check/src/picture.rs | 74 +++++++-- apps/check/text/Cargo.toml | 13 ++ apps/check/{src/text.rs => text/src/lib.rs} | 87 +++++----- docs/manual/concepts/rendering.md | 7 +- docs/manual/reference/cli/check.md | 7 + sdk/guides/craft/motion/rendering.md | 12 +- tools/layers.toml | 2 + 15 files changed, 480 insertions(+), 83 deletions(-) create mode 100644 apps/check/cause/Cargo.toml create mode 100644 apps/check/cause/src/lib.rs create mode 100644 apps/check/cause/src/tests.rs create mode 100644 apps/check/text/Cargo.toml rename apps/check/{src/text.rs => text/src/lib.rs} (82%) diff --git a/Cargo.lock b/Cargo.lock index 97b0f7cc8..be3d26e68 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13654,10 +13654,11 @@ dependencies = [ name = "raea-app-check" version = "0.1.0" dependencies = [ - "dunce", + "raea-app-check-cause", "raea-app-check-flash", "raea-app-check-inbox", "raea-app-check-read", + "raea-app-check-text", "raea-core-brief", "raea-core-facts", "raea-io-ffmpeg", @@ -13667,6 +13668,15 @@ dependencies = [ "serde_json", ] +[[package]] +name = "raea-app-check-cause" +version = "0.1.0" +dependencies = [ + "image", + "raea-core-facts", + "serde_json", +] + [[package]] name = "raea-app-check-flash" version = "0.1.0" @@ -13690,6 +13700,15 @@ dependencies = [ "serde_json", ] +[[package]] +name = "raea-app-check-text" +version = "0.1.0" +dependencies = [ + "dunce", + "raea-core-facts", + "serde_json", +] + [[package]] name = "raea-app-ci" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index a9c85e6e7..fb321c6ef 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -187,9 +187,11 @@ members = [ "runtime/scene2d", "runtime/shared", "apps/check", + "apps/check/cause", "apps/check/flash", "apps/check/inbox", "apps/check/read", + "apps/check/text", "apps/direction", "apps/notes", "apps/fingerprint", diff --git a/apps/check/Cargo.toml b/apps/check/Cargo.toml index 4b914bd8e..45c005241 100644 --- a/apps/check/Cargo.toml +++ b/apps/check/Cargo.toml @@ -7,7 +7,6 @@ publish = false description = "`raea check`: the floor's measurable lines on an export, ranked P0-P3" [dependencies] -dunce = "1" # taste.ts rules, as the raea CLI evaluates them (docs/projects.feature PJ0) raea-core-brief = { path = "../../core/brief" } # a transcript's words, read as `raea world` reads them (docs/direction.feature DR1) @@ -21,6 +20,10 @@ raea-io-ffmpeg = { path = "../../io/ffmpeg" } raea-sys-measure = { path = "../../systems/measure" } # what it reads of a video, through FFmpeg or raea's own decoders where it isn't installed raea-app-check-read = { path = "read" } +# who can fix what it finds: the encode's or the picture's own, carried, a source (split out to keep this crate small) +raea-app-check-cause = { path = "cause" } +# the text rules on a render's manifest (split out: raea render holds its own text to them) +raea-app-check-text = { path = "text" } # the flashing screen (WCAG 2.3.1), split out to keep this crate small raea-app-check-flash = { path = "flash" } # what the person sent from the raea app, handed to the agent (split out to keep this crate small) diff --git a/apps/check/cause/Cargo.toml b/apps/check/cause/Cargo.toml new file mode 100644 index 000000000..0ec6c8c4e --- /dev/null +++ b/apps/check/cause/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "raea-app-check-cause" +version = "0.1.0" +edition = "2024" +license = "FSL-1.1-ALv2" +publish = false +description = "Who can fix what raea check finds: the encode's or the picture's own, carried from an earlier export, a source rather than a delivery" + +[dependencies] +# a project's record of renders and checks (DR3) +raea-core-facts = { path = "../../../core/facts" } +image = { version = "0.25", default-features = false, features = ["png"] } + +[dev-dependencies] +serde_json = "1" diff --git a/apps/check/cause/src/lib.rs b/apps/check/cause/src/lib.rs new file mode 100644 index 000000000..9f5ec8e32 --- /dev/null +++ b/apps/check/cause/src/lib.rs @@ -0,0 +1,174 @@ +//! Who can fix what `raea check` finds, so a round is spent only on what another render can change (11-19: two +//! Telluride projects took nine renders, five of them for findings no render could move). Three things, for any rule: +//! +//! - **Introduced or there already.** A rule an encode can cause is measured again on the composition's own frames, +//! rendered without encoding (`unencoded`), and the export is set against them (`against`): what the picture already +//! held is the footage's (a note), what the export adds is the encode's (the error, with a fix raea has). +//! - **Carried.** A finding an earlier export of the same composition had at the same moment (`earlier`, `carried`) +//! says since when: the change between them didn't move it, so doing that again won't. +//! - **A source isn't a delivery.** A take or a download checked on its own (`is_source`) isn't held to a delivery's +//! loudness target. +//! +//! None of it changes what a rule measures or where it draws the line. +use std::path::{Path, PathBuf}; +use std::process::Command; + +use raea_core_facts::{Finding, record}; + +pub type R = Result; + +/// Whether a record entry is the render that wrote `out` (a path as the record names it) +fn rendered(e: &record::Entry, out: &str) -> bool { + e.by == "raea render" && e.out.as_deref() == Some(out) +} + +/// The composition a video was rendered from, and its project: the newest `raea render` writing it in the project's +/// record (DR3). None outside a project, or for a video no render there wrote (a take, a download). +pub fn composition_of(video: &Path) -> Option<(PathBuf, PathBuf)> { + let project = record::up(video, ".raea/project")?; + let want = record::rel(&project, video); + let made = record::read(&project).into_iter().rev().find(|e| e.kind == "video" && rendered(e, &want))?; + let of = project.join(&made.of); + of.is_file().then_some((project, of)) +} + +/// Whether a file is one of a project's sources (a take or a download under assets/ or media/ that no render wrote): +/// it isn't a delivery, so a delivery's rules (its loudness target) don't hold it +pub fn is_source(video: &Path) -> bool { + let Some(project) = record::up(video, ".raea/project") else { return false }; + let me = record::rel(&project, video); + (me.starts_with("assets/") || me.starts_with("media/")) && !record::read(&project).iter().any(|e| rendered(e, &me)) +} + +/// Luma as a decoder of raea's export gives it to the check (BT.709, full range: FFmpeg's format=gray of the +/// encoder's limited-range picture), from 8-bit RGBA `width` wide, its top-left `w` x `h` +pub fn luma(rgba: &[u8], width: usize, w: usize, h: usize) -> Vec { + let mut y = Vec::with_capacity(w * h); + for r in 0..h { + for p in rgba[r * width * 4..][..w * 4].chunks_exact(4) { + y.push((0.2126 * p[0] as f32 + 0.7152 * p[1] as f32 + 0.0722 * p[2] as f32).round().clamp(0.0, 255.0) as u8); + } + } + y +} + +/// `frames` as `comp` draws them, never encoded, as luma `w` x `h` (top-left): `raea render COMP --at F,... -o DIR/` +/// (PNG) run by `raea` from the project, kept out of its record as the quick check's stills are +pub fn unencoded(raea: &Path, project: &Path, comp: &Path, frames: &[i64], w: usize, h: usize) -> R>> { + let mut wanted: Vec = frames.to_vec(); + wanted.sort_unstable(); + wanted.dedup(); + let tmp = std::env::temp_dir().join(format!("raea-unencoded-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&tmp); + std::fs::create_dir_all(&tmp).map_err(|e| format!("{}: {e}", tmp.display()))?; + let at = wanted.iter().map(|f| f.to_string()).collect::>().join(","); + let read = (|| { + let run = Command::new(raea).arg("render").arg(comp).args(["--at", &at, "-o"]).arg(format!("{}/", tmp.display())).current_dir(project).env("RAEA_RECORD", "off").output().map_err(|e| e.to_string())?; + let mut pngs: Vec = std::fs::read_dir(&tmp).map_err(|e| e.to_string())?.filter_map(|e| e.ok().map(|e| e.path())).filter(|p| p.extension().is_some_and(|x| x == "png")).collect(); + pngs.sort(); + if pngs.len() != wanted.len() { + let said = String::from_utf8_lossy(&run.stderr).lines().chain(String::from_utf8_lossy(&run.stdout).lines()).find(|l| l.contains("REFUSED") || l.contains("rror")).unwrap_or("").to_string(); + return Err(format!("its render gave {} of {} frames {said}", pngs.len(), wanted.len()).trim_end().to_string()); + } + let mut by_frame = std::collections::HashMap::new(); + for (f, p) in wanted.iter().zip(&pngs) { + let img = image::open(p).map_err(|e| format!("{}: {e}", p.display()))?.to_rgba8(); + let (iw, ih) = (img.width() as usize, img.height() as usize); + if iw < w || ih < h { + return Err(format!("it draws {iw}x{ih}, smaller than the export's {w}x{h}")); + } + by_frame.insert(*f, luma(img.as_raw(), iw, w, h)); + } + Ok(frames.iter().map(|f| by_frame[f].clone()).collect()) + })(); + let _ = std::fs::remove_dir_all(&tmp); + read +} + +/// The frame a check reads for each sample taken every `every` seconds from 0: the last frame before the sample's +/// middle, as ffmpeg's fps filter picks it (and raea-app-check-read's readers), held to the picture's `last` frame +pub fn sampled(samples: usize, every: f64, fps: f64, last: i64) -> Vec { + (0..samples).map(|k| (((k as f64 + 0.5) * every * fps).ceil() as i64 - 1).clamp(0, last.max(0))).collect() +} + +/// A sampled measure of the export against the same measure of the unencoded frames. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct Against { + /// samples over the rule's line that the export made worse by more than its noise + pub worse: usize, + /// samples over the line before encoding: the picture's own + pub before: usize, + /// samples compared + pub of: usize, + /// the mean of the export's measure less the unencoded one's + pub added: f64, +} + +impl Against { + /// Whether the export added to it, beyond what a clean encode moves: at least three samples (and a tenth) worse, + /// and more than half the noise on average + pub fn by_encode(&self, noise: f64) -> bool { + self.worse >= 3usize.max(self.of / 10) && self.added > noise / 2.0 + } + + /// The first sample the export made worse + pub fn first(out: &[Option], before: &[Option], line: f64, noise: f64) -> Option { + out.iter().zip(before).position(|(o, b)| o.unwrap_or(0.0) > line && o.unwrap_or(0.0) - b.unwrap_or(0.0) > noise) + } +} + +/// Per sample, the export's measure against the unencoded frames' (None: nothing to measure, read as 0), over a +/// rule's `line`, where a clean encode moves it at most `noise` +pub fn against(out: &[Option], before: &[Option], line: f64, noise: f64) -> Against { + let n = out.len().min(before.len()); + let at = |s: &[Option], i: usize| s[i].unwrap_or(0.0); + let worse = (0..n).filter(|&i| at(out, i) > line && at(out, i) - at(before, i) > noise).count(); + let was = (0..n).filter(|&i| at(before, i) > line).count(); + let added = (0..n).map(|i| at(out, i) - at(before, i)).sum::() / n.max(1) as f64; + Against { worse, before: was, of: n, added } +} + +/// The newest check, in the project's record, of an earlier export of the composition `video` was rendered from +pub fn earlier(video: &Path) -> Option { + let project = record::up(video, ".raea/project")?; + let me = record::rel(&project, video); + let entries = record::read(&project); + let made = entries.iter().rposition(|e| rendered(e, &me))?; + // the composition's exports rendered before this one; a versioned copy (NAME-v2.tsx) is the same piece + let stem = |of: &str| piece(&Path::new(of).file_stem().map(|s| s.to_string_lossy().to_string()).unwrap_or_default()); + let comp = stem(&entries[made].of); + let exports: Vec = entries[..made].iter().filter(|e| e.by == "raea render" && e.kind == "video" && stem(&e.of) == comp).filter_map(|e| e.out.clone()).filter(|o| *o != me).collect(); + entries.into_iter().rev().find(|e| e.by == "raea check" && exports.contains(&e.of)) +} + +/// A composition's name without its version: walkaround-v2 is walkaround +pub fn piece(stem: &str) -> String { + match stem.rfind("-v") { + Some(i) if i + 2 < stem.len() && stem[i + 2..].bytes().all(|b| b.is_ascii_digit()) => stem[..i].to_string(), + _ => stem.to_string(), + } +} + +/// The same finding in another export: the same rule, about the same thing, at the same moment (within a second; a +/// finding about the whole piece matches one about the whole piece). Its words may differ ("worst 13%", "worst 15%"). +pub fn alike(a: &Finding, b: &Finding, fps: f64) -> bool { + let about = |f: &Finding| f.about.iter().map(|a| a.key.clone().unwrap_or_else(|| a.name.clone())).collect::>(); + a.check == b.check && about(a) == about(b) && match (a.frames, b.frames) { + (Some([x, _]), Some([y, _])) => ((x - y).abs() as f64) <= fps.max(1.0), + (None, None) => true, + _ => false, + } +} + +/// Each finding an earlier check had at the same moment says so, and that its fix hasn't moved it: rank and words +/// otherwise as measured +pub fn carried(now: &mut [Finding], before: &record::Entry, fps: f64) { + for f in now.iter_mut() { + if before.findings.iter().any(|b| alike(b, f, fps)) { + f.says.push_str(&format!(" [also in {} ({}): what changed between them didn't move it]", before.of, before.id)); + } + } +} + +#[cfg(test)] +mod tests; diff --git a/apps/check/cause/src/tests.rs b/apps/check/cause/src/tests.rs new file mode 100644 index 000000000..3ffc78f56 --- /dev/null +++ b/apps/check/cause/src/tests.rs @@ -0,0 +1,107 @@ +use super::*; +use raea_core_facts::Level; + +// The 28 samples `raea check` takes of the 11-16 Telluride walkaround (55.8 s, 24 fps, phone footage), as its +// blotching measures them: rendered to PNG with no encoding (BEFORE), those PNGs through x264 crf 12, and raea's own +// exports at --quality high and master. Blotching's line is 0.009; x264 moved a sample at most 0.005. +const BEFORE: [Option; 28] = [Some(0.0), None, Some(0.052), Some(0.062), None, Some(0.062), None, None, Some(0.076), Some(0.009), Some(0.03), Some(0.039), Some(0.068), None, None, None, None, Some(0.077), Some(0.062), None, None, Some(0.0), Some(0.0), Some(0.0), Some(0.0), Some(0.0), Some(0.0), Some(0.0)]; +const X264: [Option; 28] = [Some(0.0), None, Some(0.052), Some(0.062), None, Some(0.059), None, None, Some(0.078), Some(0.008), Some(0.028), Some(0.034), Some(0.071), None, None, None, None, Some(0.078), Some(0.062), None, Some(0.039), Some(0.0), Some(0.0), Some(0.0), Some(0.0), Some(0.0), Some(0.0), Some(0.0)]; +const HIGH: [Option; 28] = [Some(0.0), None, Some(0.042), Some(0.024), Some(0.052), Some(0.064), Some(0.05), None, Some(0.126), Some(0.063), Some(0.061), Some(0.122), Some(0.088), Some(0.04), Some(0.029), Some(0.047), Some(0.044), Some(0.096), Some(0.09), None, Some(0.029), Some(0.0), Some(0.0), Some(0.0), Some(0.0), Some(0.0), Some(0.0), Some(0.0)]; +const MASTER: [Option; 28] = [Some(0.0), None, Some(0.026), Some(0.002), None, Some(0.03), None, None, Some(0.022), None, None, None, None, None, None, None, None, Some(0.071), Some(0.053), None, None, Some(0.0), Some(0.0), Some(0.0), Some(0.0), Some(0.0), Some(0.0), Some(0.0)]; +const LINE: f64 = 0.009; +const NOISE: f64 = 0.005; + +/// What the picture already held is told from what the encode added: a clean encode and raea's master add nothing, +/// its high adds to 14 of 28 samples, and 9 samples hold the footage's own either way +#[test] +fn the_picture_s_own_is_told_from_the_encode_s() { + let clean = against(&X264, &BEFORE, LINE, NOISE); + assert!(!clean.by_encode(NOISE), "{clean:?}"); + assert!(clean.added.abs() < 0.002, "{clean:?}"); + let high = against(&HIGH, &BEFORE, LINE, NOISE); + assert!(high.by_encode(NOISE), "{high:?}"); + assert_eq!((high.worse, high.before, high.of), (14, 9, 28)); + assert_eq!(Against::first(&HIGH, &BEFORE, LINE, NOISE), Some(4)); + let master = against(&MASTER, &BEFORE, LINE, NOISE); + assert!(!master.by_encode(NOISE), "{master:?}"); + assert_eq!(master.worse, 0); + // a small, even addition over a clean picture still counts (VideoToolbox over a dark render, 10-25) + assert!(against(&[Some(0.013); 16], &[Some(0.003); 16], LINE, NOISE).by_encode(NOISE)); +} + +/// The frame read for each sample is the last before its middle, as the fps filter picks it +#[test] +fn the_sampled_frames_are_the_check_s() { + assert_eq!(sampled(3, 2.0, 24.0, 1338), vec![23, 71, 119]); + assert_eq!(sampled(2, 2.0, 30.0, 1000), vec![29, 89]); + assert_eq!(sampled(2, 2.0, 30000.0 / 1001.0, 1000), vec![29, 89]); + assert_eq!(sampled(2, 2.0, 24.0, 50), vec![23, 50]); +} + +/// Luma as the check reads raea's export: BT.709 weights, full range, cropped to the top-left +#[test] +fn luma_is_bt709_cropped() { + let px = |r: u8, g: u8, b: u8| [r, g, b, 255]; + let rgba: Vec = [px(255, 255, 255), px(0, 0, 0), px(255, 0, 0), px(0, 255, 0)].concat(); + assert_eq!(luma(&rgba, 2, 2, 2), vec![255, 0, 54, 182]); + assert_eq!(luma(&rgba, 2, 1, 2), vec![255, 54]); +} + +fn finding(check: &str, frame: Option, says: &str) -> Finding { + let f = Finding::new(check, Level::Error, says, "raea check"); + match frame { + Some(n) => f.frames(n, n + 1), + None => f, + } +} + +/// A finding is carried when the same rule finds the same thing at the same moment, whatever its numbers say +#[test] +fn a_carried_finding_is_the_same_rule_at_the_same_moment() { + let fps = 24.0; + assert!(alike(&finding("blotching", Some(96), "worst 13%"), &finding("blotching", Some(96), "worst 15%"), fps)); + assert!(alike(&finding("clipping", Some(624), "9 runs"), &finding("clipping", Some(640), "6 runs"), fps)); + assert!(!alike(&finding("clipping", Some(624), "9 runs"), &finding("clipping", Some(1104), "11 runs"), fps)); + assert!(!alike(&finding("clipping", Some(624), "x"), &finding("dropout", Some(624), "x"), fps)); + assert!(alike(&finding("brand", None, "a"), &finding("brand", None, "b"), fps)); + assert_eq!(piece("walkaround-v2"), "walkaround"); + assert_eq!(piece("edit-video"), "edit-video"); + assert_eq!(piece("telluride-v12"), "telluride"); +} + +/// A project whose record holds two renders of one composition and a check of the first: the second's check finds +/// that one, its blotching carried and its clipping new; a take under assets/ is a source, an export isn't +#[test] +fn the_earlier_check_is_of_the_composition_s_other_export() { + let dir = std::env::temp_dir().join(format!("raea-cause-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + for d in [".raea/record", "out", "assets/footage", "compositions"] { + std::fs::create_dir_all(dir.join(d)).unwrap(); + } + std::fs::write(dir.join(".raea/project"), "[project]\n\tid = pj_test\n").unwrap(); + for f in ["out/a-v1.mp4", "out/a-v2.mp4", "assets/footage/take.mov", "compositions/a.tsx", "compositions/a-v2.tsx"] { + std::fs::write(dir.join(f), b"").unwrap(); + } + let entry = |id: &str, by: &str, of: &str, out: Option<&str>, kind: &str, findings: Vec| record::Entry { id: id.into(), at: 0, by: by.into(), of: of.into(), out: out.map(Into::into), kind: kind.into(), commit: None, findings, despite: vec![] }; + let entries = [ + entry("20261116-170223-a", "raea render", "compositions/a.tsx", Some("out/a-v1.mp4"), "video", vec![]), + entry("20261116-170252-a-v1", "raea check", "out/a-v1.mp4", None, "check", vec![finding("blotching", Some(96), "worst 13%")]), + entry("20261116-171601-a-v2", "raea render", "compositions/a-v2.tsx", Some("out/a-v2.mp4"), "video", vec![]), + ]; + for e in &entries { + std::fs::write(dir.join(format!(".raea/record/{}.json", e.id)), serde_json::to_vec(e).unwrap()).unwrap(); + } + let v2 = dir.join("out/a-v2.mp4"); + assert_eq!(composition_of(&v2).map(|(_, c)| c.ends_with("compositions/a-v2.tsx")), Some(true)); + let before = earlier(&v2).expect("the check of a-v1"); + assert_eq!(before.of, "out/a-v1.mp4"); + let mut now = vec![finding("blotching", Some(96), "worst 15%"), finding("clipping", Some(624), "9 runs")]; + carried(&mut now, &before, 24.0); + assert!(now[0].says.contains("also in out/a-v1.mp4 (20261116-170252-a-v1)"), "{}", now[0].says); + assert_eq!(now[1].says, "9 runs"); + // nothing was rendered before a-v1, so its check has no earlier one, even with a-v2's check in the record + assert!(earlier(&dir.join("out/a-v1.mp4")).is_none()); + assert!(is_source(&dir.join("assets/footage/take.mov"))); + assert!(!is_source(&v2)); + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/apps/check/read/src/lib.rs b/apps/check/read/src/lib.rs index 2cbcf4f30..ac56a8d80 100644 --- a/apps/check/read/src/lib.rs +++ b/apps/check/read/src/lib.rs @@ -41,6 +41,8 @@ pub struct Probe { /// past the picture, and the container's duration counts that pub picture: f64, pub audio: bool, + /// the picture's frames a second (0 when it doesn't say) + pub fps: f64, } /// The picture's size, the file's duration, where the picture ends and whether there's sound: ffprobe's stream and @@ -50,7 +52,7 @@ pub fn probe(video: &str) -> R { let p = raea_app_media::probe_of(Path::new(video))?; let v = p.video.as_ref().ok_or("no video stream")?; let picture = p.track.as_ref().map_or(v.duration, |t| t.duration); - return Ok(Probe { w: v.width as f64, h: v.height as f64, duration: p.duration.max(picture), picture, audio: p.audio.is_some() }); + return Ok(Probe { w: v.width as f64, h: v.height as f64, duration: p.duration.max(picture), picture, audio: p.audio.is_some(), fps: v.fps }); } let entries = "stream=codec_type,width,height,r_frame_rate,duration,start_time:format=duration,start_time"; let out = Command::new("ffprobe").args(["-v", "error", "-show_entries", entries, "-of", "json", video]).output().map_err(|e| format!("ffprobe: {e}"))?; @@ -65,7 +67,11 @@ pub fn probe(video: &str) -> R { None => last_packet_end(video).map_or(duration, |end| end - origin), }; let audio = streams.iter().any(|s| s["codec_type"] == "audio"); - Ok(Probe { w: vid["width"].as_f64().unwrap_or(1920.0), h: vid["height"].as_f64().unwrap_or(1080.0), duration, picture, audio }) + let fps = vid["r_frame_rate"].as_str().and_then(|r| { + let (n, d) = r.split_once('/').unwrap_or((r, "1")); + Some(n.parse::().ok()? / d.parse::().ok().filter(|d| *d > 0.0)?) + }); + Ok(Probe { w: vid["width"].as_f64().unwrap_or(1920.0), h: vid["height"].as_f64().unwrap_or(1080.0), duration, picture, audio, fps: fps.unwrap_or(0.0) }) } /// Where the picture's last packet ends, for a container that doesn't state the video stream's own length (Matroska) diff --git a/apps/check/src/lib.rs b/apps/check/src/lib.rs index c7f3577f5..4e632b4fe 100644 --- a/apps/check/src/lib.rs +++ b/apps/check/src/lib.rs @@ -22,8 +22,10 @@ mod flash; use flash::*; mod defects; use defects::*; -mod text; -use text::*; +/// what a render's text breaks, from its manifest (split out: a render holds its own text to it) +use raea_app_check_text::*; +/// who can fix what it finds: the encode's or the picture's own, carried from an earlier export, a source +use raea_app_check_cause as cause; /// a video read through FFmpeg, or raea's own decoders where it isn't installed use raea_app_check_read::{self as read, Probe}; pub mod quick; @@ -32,15 +34,6 @@ pub use raea_app_check_inbox as inbox; #[cfg(test)] mod tests; -type R = Result; - -#[derive(Clone)] -struct Issue { - rule: &'static str, - p: u8, - at: Option, - says: String, -} /// `raea check …` with the person's taste, none or more (taste.ts and the kinds beside it, which the raea CLI evaluates: it owns the engine): /// its rules' checks hold the export to the person's own lines. @@ -167,6 +160,12 @@ fn measured(args: &[String], taste: Vec) -> R<(String, Vec, Vec, more: Vec, fps: f64) -> Vec, more: Vec, passed: Vec, out: Option<&str>, fps: f64) -> R<(String, bool)> { - let found = findings_of(issues, more, fps); + let mut found = findings_of(issues, more, fps); + // what an earlier export of the same composition already had says so: the change between them didn't move it + if let Some(e) = cause::earlier(std::path::Path::new(video)) { + cause::carried(&mut found, &e, fps); + } let bad = found.iter().any(|f| f.level == Level::Error); if let Some(out) = out { let j = json!({"file": video, "fps": fps, "findings": found, "passed": passed}); diff --git a/apps/check/src/picture.rs b/apps/check/src/picture.rs index 074f4fc17..f0737d281 100644 --- a/apps/check/src/picture.rs +++ b/apps/check/src/picture.rs @@ -101,6 +101,24 @@ pub(crate) fn x264_crf(head: &[u8]) -> Option { /// in its darks is the picture's own (8-bit glows and faint panels), which blotch_score can't always tell apart pub(crate) const X264_KEEPS: f64 = 16.0; +/// How far a clean encode moves a sample's blotching (x264 crf 12 over the walkaround's unencoded frames, 11-19: at most 0.005) +pub(crate) const BLOTCH_NOISE: f64 = 0.005; + +/// Blotching's samples against the same frames of the composition drawn without encoding (raea-app-check-cause): None +/// when the video isn't a render in a raea project's record +fn blotching_cause(video: &str, probe: &Probe, scores: &[Option], w: usize, h: usize) -> Option>)>> { + let (project, comp) = cause::composition_of(std::path::Path::new(video))?; + Some((|| { + if probe.fps <= 0.0 { + return Err("the video doesn't say its frame rate".to_string()); + } + let frames = cause::sampled(scores.len(), 2.0, probe.fps, (probe.picture * probe.fps).round() as i64 - 1); + let me = std::env::current_exe().map_err(|e| e.to_string())?; + let before: Vec> = cause::unencoded(&me, &project, &comp, &frames, w, h)?.iter().map(|y| blotch_score(y, w, h)).collect(); + Ok((cause::against(scores, &before, BLOTCHY, BLOTCH_NOISE), before)) + })()) +} + pub(crate) fn blotching(video: &str, probe: &Probe, issues: &mut Vec, passed: &mut Vec) -> R<()> { let mut head = Vec::new(); if let Ok(f) = std::fs::File::open(video) { @@ -112,8 +130,11 @@ pub(crate) fn blotching(video: &str, probe: &Probe, issues: &mut Vec, pas } let (w, h) = (probe.w as usize / 8 * 8, probe.h as usize / 8 * 8); let (mut bad, mut seen, mut worst, mut t) = (Vec::new(), 0, 0f64, 0.0); + let mut scores: Vec> = Vec::new(); let mut frame = |buf: &[u8]| { - if let Some(s) = blotch_score(buf, w, h) { + let score = blotch_score(buf, w, h); + scores.push(score); + if let Some(s) = score { seen += 1; worst = worst.max(s); if std::env::var_os("RAEA_CHECK_DEBUG").is_some() { @@ -143,18 +164,49 @@ pub(crate) fn blotching(video: &str, probe: &Probe, issues: &mut Vec, pas if !bad.is_empty() { let at = bad[0].0; let share = bad.len() as f64 / seen.max(1) as f64; - issues.push(Issue { - rule: "blotching", - p: if share > 0.2 { 1 } else { 2 }, - at: Some(at), - says: format!( - "blocky patches in dark areas in {} of {seen} sampled frames (worst {:.0}% of dark blocks stepping); the encoder starved the near-blacks: export as PNG frames and encode with more bits for darks", - bad.len(), - worst * 100.0 - ), - }); + let found = format!("blocky patches in dark areas in {} of {seen} sampled frames (worst {:.0}% of dark blocks stepping)", bad.len(), worst * 100.0); + let (p, at, says) = match blotching_cause(video, probe, &scores, w, h) { + // what the composition draws before any encoding is the picture's own (a phone's compression): no render + // setting removes it, so it's said, not worked on + Some(Ok(a)) if !a.0.by_encode(BLOTCH_NOISE) => (3, Some(at), format!("{found}: the picture's own, there as the composition draws them before any encoding (the footage's compression); the encode added none, so no render setting removes them")), + Some(Ok(a)) => { + let first = cause::Against::first(&scores, &a.1, BLOTCHY, BLOTCH_NOISE).map(|i| i as f64 * 2.0); + let own = if a.0.before > 0 { format!("; {} of the samples show the footage's own too, which stays", a.0.before) } else { String::new() }; + (if a.0.worse as f64 / a.0.of.max(1) as f64 > 0.2 { 1 } else { 2 }, first.or(Some(at)), format!("the encode added blocky patches in dark areas: {} of {} samples are blotchier than the composition draws them unencoded; render with --quality master, which gives the darks the bits they need{own}", a.0.worse, a.0.of)) + } + Some(Err(why)) => (if share > 0.2 { 1 } else { 2 }, Some(at), format!("{found}; whether the encode made them or the picture holds them couldn't be told ({why}): render with --quality master if the encode starved the darks")), + None => (if share > 0.2 { 1 } else { 2 }, Some(at), format!("{found}: the encode starved the near-blacks, or the footage holds them; render with --quality master, and check a render made in a raea project to tell which")), + }; + issues.push(Issue { rule: "blotching", p, at, says }); } else if seen > 0 { passed.push(format!("blotching: dark areas clean in {seen} sampled frames")); } Ok(()) } + +/// Caption contrast: at each text's middle frame, the pixels behind it against its colour, at least 3:1. The pixels +/// behind are the half of the box least like the text. +pub(crate) fn contrast(video: &str, runs: &[Run], w: f64, h: f64, probe: &Probe, issues: &mut Vec, passed: &mut Vec) { + let (sx, sy) = (probe.w / w, probe.h / h); + let mut low = 0; + let mut measured = 0; + for r in runs.iter().filter(|r| r.size >= 20.0).take(80) { + let [x, y, bw, bh] = r.box_; + let crop = [(x * sx).max(0.0) as u32, (y * sy).max(0.0) as u32, (bw * sx) as u32, (bh * sy) as u32]; + if crop[0] + crop[2] > probe.w as u32 || crop[1] + crop[3] > probe.h as u32 { + continue; + } + let t = (r.from + r.to) / 2.0; + // the same measure `raea world` joins to a text from its render (raea-sys-measure, DR2) + let ratio = if crate::read::native() { crate::read::contrast(video, t, crop, &r.color) } else { raea_sys_measure::contrast(std::path::Path::new(video), t, crop, &r.color) }; + let Some(ratio) = ratio else { continue }; + measured += 1; + if ratio < 3.0 { + low += 1; + issues.push(Issue { rule: "contrast", p: 1, at: Some(t), says: format!("{} is {ratio:.1}:1 against what's behind it; make it at least 3:1 (a band, a shadow for separation, or another colour)", quote(&r.text)) }); + } + } + if measured > 0 && low == 0 { + passed.push(format!("contrast: {measured} texts at 3:1 or better against their pixels")); + } +} diff --git a/apps/check/text/Cargo.toml b/apps/check/text/Cargo.toml new file mode 100644 index 000000000..41ac94820 --- /dev/null +++ b/apps/check/text/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "raea-app-check-text" +version = "0.1.0" +edition = "2024" +license = "FSL-1.1-ALv2" +publish = false +description = "raea check's text rules on a render's manifest: clipped, UI zones, reading time, face, cues, brand kit" + +[dependencies] +dunce = "1" +serde_json = "1" +# a transcript's words, read as `raea world` reads them (docs/direction.feature DR1) +raea-core-facts = { path = "../../../core/facts" } diff --git a/apps/check/src/text.rs b/apps/check/text/src/lib.rs similarity index 82% rename from apps/check/src/text.rs rename to apps/check/text/src/lib.rs index 2cd4110dc..a653a68a3 100644 --- a/apps/check/src/text.rs +++ b/apps/check/text/src/lib.rs @@ -1,25 +1,39 @@ -use super::*; +//! What a render's text breaks, from its manifest alone: clipped, in the platform's UI zones, too brief to read, over +//! the face, off its cue, and the brand kit it was styled from. raea check's text rules, split out of raea-app-check so +//! `raea render --manifest` holds its own text to them as it finishes, a check's round sooner. +use serde_json::Value; + +pub type R = Result; + +/// One thing a rule found: its rule, its rank (P0 breaks the piece to P3 polish), when (seconds) and what to do. +#[derive(Clone, Debug)] +pub struct Issue { + pub rule: &'static str, + pub p: u8, + pub at: Option, + pub says: String, +} /// A text as it stays on screen: from the frame it comes on to the frame it leaves. Typing (a text growing into the /// next) and counting (only its digits changing) are one run, judged by its last text. #[derive(Clone)] -pub(crate) struct Run { - pub(crate) text: String, - pub(crate) from: f64, - pub(crate) to: f64, - pub(crate) box_: [f64; 4], +pub struct Run { + pub text: String, + pub from: f64, + pub to: f64, + pub box_: [f64; 4], /// where its letters are on every frame it shows its whole text (the manifest's `ink`, else its box: a centred line in /// a full-width box is nowhere near x = 0, #89): a text the camera carries (a sign on a panning street) moves /// through the frame's edges and zones, which only counts against it when no frame shows it clear - pub(crate) boxes: Vec<[f64; 4]>, - pub(crate) color: String, - pub(crate) size: f64, + pub boxes: Vec<[f64; 4]>, + pub color: String, + pub size: f64, } /// The brand kit the cut was styled from (brand/ at or above the manifest's or the video's folder, as render finds /// brand.css): a kit `raea brand` read from a thin fetch that nobody corrected by hand is a placeholder, P1; a brand.css /// with no brand.json beside it wasn't read from the brand at all, P2 -pub(crate) fn brand_kit(near: &[&str], issues: &mut Vec, passed: &mut Vec) { +pub fn brand_kit(near: &[&str], issues: &mut Vec, passed: &mut Vec) { let mut seen = std::collections::HashSet::new(); for start in near.iter().filter(|p| !p.is_empty()) { let abs = dunce::canonicalize(&std::path::Path::new(start)).unwrap_or_else(|_| std::path::PathBuf::from(start)); @@ -50,7 +64,7 @@ pub(crate) fn brand_kit(near: &[&str], issues: &mut Vec, passed: &mut Vec } } -pub(crate) fn text_runs(manifest: &Value) -> Vec { +pub fn text_runs(manifest: &Value) -> Vec { let fps = manifest["fps"].as_f64().unwrap_or(30.0); let frames = manifest["frames"].as_array().cloned().unwrap_or_default(); let mut open: Vec = Vec::new(); @@ -95,11 +109,11 @@ pub(crate) fn text_runs(manifest: &Value) -> Vec { } /// The share of the face's area a text may cover before it's a fault (@raea/video's proofBoxes keeps the same 8%) -pub(crate) const FACE_COVER: f64 = 0.08; +pub const FACE_COVER: f64 = 0.08; /// The face over time from a perception.json (`raea cut`: each clip's first frame, the face at its middle in the /// edit's px): [from, to) in seconds and the face's box, for the stretches that have one -pub(crate) fn faces_of(perception: &Value, fps: f64) -> Vec<(f64, f64, [f64; 4])> { +pub fn faces_of(perception: &Value, fps: f64) -> Vec<(f64, f64, [f64; 4])> { let mut keys: Vec<(f64, Option<[f64; 4]>)> = perception.as_object().into_iter().flatten().filter_map(|(k, v)| { let face = v["face"].as_object().map(|f| ["x", "y", "w", "h"].map(|n| f.get(n).and_then(Value::as_f64).unwrap_or(0.0))); Some((k.parse::().ok()? / fps, face)) @@ -110,7 +124,7 @@ pub(crate) fn faces_of(perception: &Value, fps: f64) -> Vec<(f64, f64, [f64; 4]) /// Text over the face: a run that covers more than FACE_COVER of the face's area while that face is up, on every frame /// that shows it whole (a text the camera carries counts only when no frame shows it clear) -pub(crate) fn face_cover(runs: &[Run], faces: &[(f64, f64, [f64; 4])], issues: &mut Vec, passed: &mut Vec) { +pub fn face_cover(runs: &[Run], faces: &[(f64, f64, [f64; 4])], issues: &mut Vec, passed: &mut Vec) { let share = |b: &[f64; 4], f: &[f64; 4]| { let ox = (b[0] + b[2]).min(f[0] + f[2]) - b[0].max(f[0]); let oy = (b[1] + b[3]).min(f[1] + f[3]) - b[1].max(f[1]); @@ -134,7 +148,7 @@ pub(crate) fn face_cover(runs: &[Run], faces: &[(f64, f64, [f64; 4])], issues: & } /// the zones where text can't go, as [top, right, bottom, left] at 1080 wide (sdk/guides/craft/floor.md) -pub(crate) fn zones(w: f64, h: f64, shape: Option<&str>, platform: Option<&str>) -> [f64; 4] { +pub fn zones(w: f64, h: f64, shape: Option<&str>, platform: Option<&str>) -> [f64; 4] { let r = w / h; let shape = shape.unwrap_or(if r < 0.7 { "9:16" } else if r < 0.9 { "4:5" } else if r < 1.2 { "1:1" } else { "16:9" }); let z = match (shape, platform) { @@ -153,12 +167,12 @@ pub(crate) fn zones(w: f64, h: f64, shape: Option<&str>, platform: Option<&str>) /// A caption the voice reads out: every word of the text is said while it's on screen (from a little before it comes /// on). The viewer hears it as it shows, so it needs no reading time of its own (karaoke groups are up for as long as /// they take to say). -pub(crate) fn spoken(r: &Run, words: &[Word]) -> bool { +pub fn spoken(r: &Run, words: &[Word]) -> bool { let tokens: Vec = r.text.split_whitespace().map(norm).filter(|t| !t.is_empty()).collect(); !tokens.is_empty() && tokens.iter().all(|t| words.iter().any(|w| norm(&w.text) == *t && w.start >= r.from - 0.3 && w.start <= r.to + 0.1)) } -pub(crate) fn text_rules(runs: &[Run], heard: Option<&[Word]>, w: f64, h: f64, shape: Option<&str>, platform: Option<&str>, issues: &mut Vec, passed: &mut Vec) { +pub fn text_rules(runs: &[Run], heard: Option<&[Word]>, w: f64, h: f64, shape: Option<&str>, platform: Option<&str>, issues: &mut Vec, passed: &mut Vec) { let [zt, zr, zb, zl] = zones(w, h, shape, platform); let (mut clipped, mut unsafe_, mut brief) = (0, 0, 0); for r in runs { @@ -194,57 +208,30 @@ pub(crate) fn text_rules(runs: &[Run], heard: Option<&[Word]>, w: f64, h: f64, s } } -pub(crate) fn quote(s: &str) -> String { +pub fn quote(s: &str) -> String { let s: String = s.chars().take(40).collect(); format!("\u{201c}{s}\u{201d}") } -/// Caption contrast: at each text's middle frame, the pixels behind it against its colour, at least 3:1. The pixels -/// behind are the half of the box least like the text. -pub(crate) fn contrast(video: &str, runs: &[Run], w: f64, h: f64, probe: &Probe, issues: &mut Vec, passed: &mut Vec) { - let (sx, sy) = (probe.w / w, probe.h / h); - let mut low = 0; - let mut measured = 0; - for r in runs.iter().filter(|r| r.size >= 20.0).take(80) { - let [x, y, bw, bh] = r.box_; - let crop = [(x * sx).max(0.0) as u32, (y * sy).max(0.0) as u32, (bw * sx) as u32, (bh * sy) as u32]; - if crop[0] + crop[2] > probe.w as u32 || crop[1] + crop[3] > probe.h as u32 { - continue; - } - let t = (r.from + r.to) / 2.0; - // the same measure `raea world` joins to a text from its render (raea-sys-measure, DR2) - let ratio = if crate::read::native() { crate::read::contrast(video, t, crop, &r.color) } else { raea_sys_measure::contrast(std::path::Path::new(video), t, crop, &r.color) }; - let Some(ratio) = ratio else { continue }; - measured += 1; - if ratio < 3.0 { - low += 1; - issues.push(Issue { rule: "contrast", p: 1, at: Some(t), says: format!("{} is {ratio:.1}:1 against what's behind it; make it at least 3:1 (a band, a shadow for separation, or another colour)", quote(&r.text)) }); - } - } - if measured > 0 && low == 0 { - passed.push(format!("contrast: {measured} texts at 3:1 or better against their pixels")); - } -} - /// A transcript's word: its text, start and end in seconds (its end is its start when the transcript doesn't say). -pub(crate) use raea_core_facts::Word; +pub use raea_core_facts::Word; /// A transcript's words (raea_core_facts::transcript_words: {text, start} in seconds, or t, start_ms, or {text, from} /// in frames at `fps`), as the app's words.json and `raea render --words` have them. -pub(crate) fn transcript(path: &str, fps: f64) -> R> { +pub fn transcript(path: &str, fps: f64) -> R> { let v: Value = serde_json::from_str(&std::fs::read_to_string(path).map_err(|e| format!("{path}: {e}"))?).map_err(|e| format!("{path}: {e}"))?; v.as_array().ok_or_else(|| format!("{path}: expected an array of words ({{text, start}} in seconds or {{text, from}} in frames)"))?; Ok(raea_core_facts::transcript_words(&v, fps)) } -pub(crate) use raea_core_facts::word_key as norm; +pub use raea_core_facts::word_key as norm; -pub(crate) const SMALL: [&str; 24] = ["the", "and", "you", "your", "for", "with", "that", "this", "its", "it's", "are", "was", "our", "out", "from", "into", "then", "than", "but", "not", "all", "can", "has", "have"]; +pub const SMALL: [&str; 24] = ["the", "and", "you", "your", "for", "with", "that", "this", "its", "it's", "are", "was", "our", "out", "from", "into", "then", "than", "but", "not", "all", "can", "has", "have"]; /// Cues land on their words: a text whose words the voice says nearby comes on with the first of them, not after it. /// The text is found by its first word of weight, and then measured from the first word it shows: a caption "of the /// spend" comes on with "of", when the voice says "of the" just before "spend". -pub(crate) fn cues(runs: &[Run], words: &[Word], issues: &mut Vec, passed: &mut Vec) { +pub fn cues(runs: &[Run], words: &[Word], issues: &mut Vec, passed: &mut Vec) { let (mut checked, mut off) = (0, 0); for r in runs { let shown: Vec = r.text.split_whitespace().map(norm).filter(|t| !t.is_empty()).collect(); diff --git a/docs/manual/concepts/rendering.md b/docs/manual/concepts/rendering.md index 77f6aa011..91d57d371 100644 --- a/docs/manual/concepts/rendering.md +++ b/docs/manual/concepts/rendering.md @@ -54,8 +54,11 @@ By default the bitrate follows the size and frame rate: about 20 Mbit/s at 1080p `--quality` asks for constant quality: each frame gets the bits it needs, so a flat motion graphic stays small and a busy one stays clean. -Blotchy dark areas come from the encoder, not the composition. Export dark material as PNG frames and encode them -with more bits for the darks: +Blotchy dark areas come from the encoder, or from the footage itself: a phone's own compression shows in its darks. +When `raea check` finds them on a render made in a raea project, it draws the composition again at the frames it +sampled, without encoding, and tells the two apart. The picture's own is a note, since no render setting removes it. +What the encode added is the error, and `--quality master` gives the darks the bits they need. To hand dark material on +as frames instead, export PNG frames: ```sh RAEA_READBACK=rgba raea render piece.tsx -o frames/ diff --git a/docs/manual/reference/cli/check.md b/docs/manual/reference/cli/check.md index 33bd904e9..7e2d5a9bc 100644 --- a/docs/manual/reference/cli/check.md +++ b/docs/manual/reference/cli/check.md @@ -27,6 +27,13 @@ the platform's buttons, captions held too briefly and caption contrast. The tran word and music fighting the voice; the perception adds text over a face. A taste file adds the person's own rules, each an error when broken, since their rule beats the floor's default. +On a render made in a raea project, the check also uses the project's record. A finding an earlier export of the +same composition had at the same moment ends `[also in …]`: the change between them didn't move it, so don't do that +again; try something else, or say why it stands. A rule an encode can cause (blotching) is measured again on the +composition drawn without encoding, so what the footage already holds is a note and only what the encode added is an +error. A take or a download under the project's `assets/` or `media/` is a source, not a delivery: its loudness is +reported as a note, since the mix sets the delivery's level. + ## Every rule a project holds On a project, or a composition's file, with no `--quick`, `raea check` runs every rule that applies to each diff --git a/sdk/guides/craft/motion/rendering.md b/sdk/guides/craft/motion/rendering.md index cbae3d7d2..67d03113f 100644 --- a/sdk/guides/craft/motion/rendering.md +++ b/sdk/guides/craft/motion/rendering.md @@ -8,7 +8,10 @@ It measures clipped text, text under a platform's buttons, text over the face, captions on too briefly to read, caption contrast against the pixels behind them, frozen stretches, flashing past the limit, cues that miss their word, loudness, a sound that stops dead or (with `--words`) fights the voice, and encoder blotching in the darks. Each finding is ranked P0 to P3 - with its fix; it exits 1 while a P0 or P1 stands, and `-o check.json` writes them for the critique. An + with its fix; it exits 1 while a P0 or P1 stands, and `-o check.json` writes them for the critique. Fix every error + in one pass before rendering again. A finding that ends `[also in …]` was in the earlier export too: what you changed + didn't move it, so don't repeat that change; try another, or say why it stands. A note is for the person, not a + reason to render again. An exception in the composition (an unclamped value going NaN, keyframes out of order, a phrase `word()` can't find, a missing prop) fails the render and names the line. - **Watch it as a viewer would once the piece is close.** `raea check` measures; it can't judge a frame cropped too @@ -45,9 +48,10 @@ looking again, finds more than a still per change and costs less. A video's bitrate follows its size and frame rate (about 20 Mbit/s at 1080p30, 80 at 4K30; VideoToolbox spends less on flat pictures), and `--kbps N` sets it. `--quality draft|high|master` (or 0..1: 0.6, 0.85, 0.95) asks for constant quality instead, each frame getting the bits it needs: `master` keeps thin type and hard vector edges clean at 4K -(about 31 Mbit/s on flat motion graphics). Blotching in dark areas is the encoder, not the composition: export dark -material as PNG frames (`RAEA_READBACK=rgba raea render piece.tsx -o piece.frames/`) and encode them with more bits for -the darks. A `NAME.frames/` folder is never committed, and raea's storage dry run lists it a day after +(about 31 Mbit/s on flat motion graphics). Blotching in dark areas is the encoder's or the footage's own, and on a +render made in a project `raea check` says which: the footage's is a note to pass on, never a reason to render again; +for the encode's, render with `--quality master`. Dark material can also go out as PNG frames +(`RAEA_READBACK=rgba raea render piece.tsx -o piece.frames/`). A `NAME.frames/` folder is never committed, and raea's storage dry run lists it a day after the render unless something names it. Which to ask for. A version made for notes (a cut to review, a shot that will be joined, a preview) takes no diff --git a/tools/layers.toml b/tools/layers.toml index d6cb16eb8..bc4ca2616 100644 --- a/tools/layers.toml +++ b/tools/layers.toml @@ -178,9 +178,11 @@ raea-runtime-scene2d = "runtime" raea-runtime-shared = "runtime" raea-page = "runtime" raea-app-check = "apps" +raea-app-check-cause = "apps" raea-app-check-flash = "apps" raea-app-check-inbox = "apps" raea-app-check-read = "apps" +raea-app-check-text = "apps" raea-app-direction = "apps" raea-app-notes = "apps" # `raea fingerprint`: its verbs and the resident service that keeps the encoders loaded From 119f4580182b8fec627e0d7f448b6a623f9c7423 Mon Sep 17 00:00:00 2001 From: Zachary Guglin Date: Thu, 19 Nov 2026 14:15:56 -0800 Subject: [PATCH 2/6] raea render holds a video's text to the check's rules as it ends, and keeps a 2D frame's fill out of the lighting advice A round sooner: what the render's manifest already knows is said by the render, not left for the check after it (the 11-14 walkaround's price figures in the UI zone waited a render and a check). Text off the frame is an error, as the check has it. Text in a UI zone is a warn by the default zones, since the platform's are the check's to decide; reading time waits for the check, which has the transcript. Video renders only: a stills render samples too few frames to time a text. raea check's issue-to-finding conversion is now Issue::finding, which both use. Exposure's lighting advice ("a fill, an ambient or the sky's light reaches them nowhere") is a 3D scene's. A frame with no 3D in the render and no footage on it is the page's own fill, dark or bright by design (the walkaround's brand-dark logo card warned on every stills render), so it's measured and kept in the manifest, not said. A frame with footage keeps its warning: a clip that drew black still shows. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 1 + apps/check/src/lib.rs | 8 +----- apps/check/text/src/lib.rs | 17 +++++++++++++ apps/direction/Cargo.toml | 2 ++ apps/direction/src/rendered.rs | 46 +++++++++++++++++++++++++++++++++- runtime/export/src/render.rs | 36 ++++++++++++++++++++------ 6 files changed, 95 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index be3d26e68..df4730789 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13724,6 +13724,7 @@ version = "0.1.0" dependencies = [ "dunce", "raea-app-check", + "raea-app-check-text", "raea-app-notes", "raea-app-registry", "raea-core-brief", diff --git a/apps/check/src/lib.rs b/apps/check/src/lib.rs index 4e632b4fe..3a799ad20 100644 --- a/apps/check/src/lib.rs +++ b/apps/check/src/lib.rs @@ -203,13 +203,7 @@ fn things(p: &str, issues: &mut Vec, passed: &mut Vec) -> R<()> { /// `fps`), with `more` (a render's own), worst first then by time. fn findings_of(mut issues: Vec, more: Vec, fps: f64) -> Vec { issues.sort_by(|a, b| a.p.cmp(&b.p).then(a.at.unwrap_or(0.0).total_cmp(&b.at.unwrap_or(0.0)))); - let mut found = Vec::new(); - for i in &issues { - // an issue at a second is about the frame it falls on - let (f, fr) = (Finding::new(i.rule, Level::of_rank(i.p), &i.says, "raea check"), (i.at.unwrap_or(0.0) * fps).round() as i64); - found.push(if i.at.is_some() { f.frames(fr, fr + 1) } else { f }); - } - finding::ranked(found.into_iter().chain(more).collect()) + finding::ranked(issues.iter().map(|i| i.finding("raea check", fps)).chain(more).collect()) } /// The findings (`findings_of`), written to -o as JSON when asked, and the report's text with the count last. True diff --git a/apps/check/text/src/lib.rs b/apps/check/text/src/lib.rs index a653a68a3..5ec319c72 100644 --- a/apps/check/text/src/lib.rs +++ b/apps/check/text/src/lib.rs @@ -5,6 +5,8 @@ use serde_json::Value; pub type R = Result; +use raea_core_facts::{Finding, Level}; + /// One thing a rule found: its rule, its rank (P0 breaks the piece to P3 polish), when (seconds) and what to do. #[derive(Clone, Debug)] pub struct Issue { @@ -14,6 +16,21 @@ pub struct Issue { pub says: String, } +impl Issue { + /// As a Finding `by` a command (docs/direction.feature DR3: P0 and P1 errors, P2 a warn, P3 a note), about the + /// frame its second falls on at `fps` + pub fn finding(&self, by: &str, fps: f64) -> Finding { + let f = Finding::new(self.rule, Level::of_rank(self.p), &self.says, by); + match self.at { + Some(t) => { + let n = (t * fps).round() as i64; + f.frames(n, n + 1) + } + None => f, + } + } +} + /// A text as it stays on screen: from the frame it comes on to the frame it leaves. Typing (a text growing into the /// next) and counting (only its digits changing) are one run, judged by its last text. #[derive(Clone)] diff --git a/apps/direction/Cargo.toml b/apps/direction/Cargo.toml index e038d3c12..0ce62144f 100644 --- a/apps/direction/Cargo.toml +++ b/apps/direction/Cargo.toml @@ -19,6 +19,8 @@ serde_json = "1" # project's compositions raea-core-brief = { path = "../../core/brief" } raea-app-check = { path = "../check" } +# the check's text rules, held to a video render's manifest as it ends +raea-app-check-text = { path = "../check/text" } raea-app-registry = { path = "../workspace/registry" } # the project's notes, whose asks' checks raea check runs beside its rules (DR6) raea-app-notes = { path = "../notes" } diff --git a/apps/direction/src/rendered.rs b/apps/direction/src/rendered.rs index e7622eedd..14126a05c 100644 --- a/apps/direction/src/rendered.rs +++ b/apps/direction/src/rendered.rs @@ -38,7 +38,7 @@ pub fn rendered(args: &[String]) -> i32 { if args.first().map(String::as_str) != Some("render") { return 0; } - let found = finding::taken(); + let mut found = finding::taken(); let through = despite(args, std::env::var("RAEA_DESPITE").ok()); let excused: Vec = through.iter().map(|(c, w)| Despite { check: c.clone(), why: w.clone() }).collect(); let out = raea_runtime_bundle::flag(args, "-o").or_else(|| raea_runtime_bundle::flag(args, "--review")).map(PathBuf::from); @@ -46,6 +46,13 @@ pub fn rendered(args: &[String]) -> i32 { let manifest = raea_runtime_bundle::flag(args, "--manifest").map(PathBuf::from).or_else(|| Some(raea_runtime_world::manifest_for(out.as_deref()?, std::path::Path::new(args.get(1)?)))); if let Some(m) = manifest { if let Some(mut v) = std::fs::read(&m).ok().and_then(|b| serde_json::from_slice::(&b).ok()) { + // a video's text, held to the check's text rules as it finishes, a check's round sooner + if out.as_deref().and_then(|o| o.extension()).is_some_and(|x| x == "mp4" || x == "mov") { + for f in text_found(&v) { + println!("{}", f.line()); + found.push(f); + } + } v["findings"] = json!(found); if !excused.is_empty() { v["despite"] = json!(excused); @@ -63,6 +70,26 @@ pub fn rendered(args: &[String]) -> i32 { code } +/// What a video render's text breaks, from its manifest (raea-app-check-text, the rules `raea check` holds it to): text +/// off the frame is an error, as the check has it; text in a UI zone a warn, by the default zones, since the platform's +/// are the check's to decide (`--platform`); reading time waits for the check, which has the transcript to tell a +/// caption the voice reads out from one that has to be read. +fn text_found(manifest: &Value) -> Vec { + let runs = raea_app_check_text::text_runs(manifest); + let (w, h, fps) = (manifest["width"].as_f64().unwrap_or(1080.0), manifest["height"].as_f64().unwrap_or(1920.0), manifest["fps"].as_f64().unwrap_or(30.0)); + let (mut issues, mut passed) = (Vec::new(), Vec::new()); + raea_app_check_text::text_rules(&runs, None, w, h, None, None, &mut issues, &mut passed); + let mut found = Vec::new(); + for mut i in issues.into_iter().filter(|i| i.rule != "reading") { + if i.rule == "safe-area" { + i.p = 2; + i.says.push_str(" (by the default zones: `raea check --platform P` holds it to the platform's)"); + } + found.push(i.finding("raea render", fps)); + } + found +} + #[cfg(test)] mod tests { use super::*; @@ -71,6 +98,23 @@ mod tests { v.iter().map(|a| a.to_string()).collect() } + /// A video's text off the frame fails its render, as the check fails it; text in a UI zone is a warn by the + /// default zones; a word on screen too briefly is left to the check, which has the transcript + #[test] + fn a_video_render_holds_its_text_to_the_check_s_rules() { + let text = |t: &str, b: [f64; 4]| json!({"text": t, "box": b, "opacity": 1.0, "size": 60.0, "color": "rgb(255, 255, 255)"}); + let frames: Vec = (0..48).map(|f| json!({"frame": f, "videos": [], "text": [ + text("OFF THE EDGE", [900.0, 800.0, 400.0, 80.0]), + text("Per month", [700.0, 1233.0, 318.0, 44.0]), + if f < 3 { text("Two words", [200.0, 900.0, 300.0, 80.0]) } else { json!({"text": "", "box": [0, 0, 0, 0]}) }, + ]})).collect(); + let found = text_found(&json!({"width": 1080, "height": 1920, "fps": 24, "frames": frames})); + let got: Vec<(&str, Level)> = found.iter().map(|f| (f.check.as_str(), f.level)).collect(); + assert_eq!(got, vec![("clipped", Level::Error), ("safe-area", Level::Warn)], "{found:?}"); + assert!(found[1].says.contains("--platform")); + assert_eq!(found[0].frames, Some([0, 1])); + } + #[test] fn an_error_fails_the_render_unless_it_is_let_through_and_says_by_what() { let found = vec![Finding::new("clips", Level::Error, "rider in seat", "raea render"), Finding::new("tilted", Level::Warn, "vase leans", "raea render")]; diff --git a/runtime/export/src/render.rs b/runtime/export/src/render.rs index ed1976040..06db60428 100644 --- a/runtime/export/src/render.rs +++ b/runtime/export/src/render.rs @@ -150,15 +150,37 @@ fn with_exposure(said: String, out: &Path, manifest: Option<&Path>) -> String { return said; } match raea_sys_light_meter::exposure::of_pngs(&pngs) { - Ok((e, v)) => { - let fs = raea_sys_light_meter::exposure::said(&v); // counted with the render's other findings (DR3) - if let Some(m) = manifest { - if let Ok(mut doc) = std::fs::read_to_string(m).map_err(|e| e.to_string()).and_then(|t| serde_json::from_str::(&t).map_err(|e| e.to_string())) { - if let Some(o) = doc.as_object_mut() { - o.insert("exposure".into(), v.clone()); - let _ = std::fs::write(m, doc.to_string()); + Ok((e, mut v)) => { + // a frame of the page's own fill (no 3D in the render, no footage on it: a brand card's dark ground) is dark + // or bright by design, and the advice ("a fill, an ambient or the sky's light reaches them nowhere") is a 3D + // scene's: it's measured, not said. A frame with footage keeps it (a clip that drew black). 11-19: the + // walkaround's logo card warned on every stills render. + let doc: Option = manifest.and_then(|m| std::fs::read_to_string(m).ok()).and_then(|t| serde_json::from_str(&t).ok()); + if let Some(d) = doc.as_ref().filter(|d| d["has3d"] == false) { + let frames = d["frames"].as_array().cloned().unwrap_or_default(); + fn fill(f: &serde_json::Value) -> bool { + f["videos"].as_array().is_none_or(Vec::is_empty) + } + let mut names: Vec = Vec::new(); + if pngs.len() == 1 && frames.len() == 1 && fill(&frames[0]) { + names.push("the frame".into()); + } else if pngs.len() == frames.len() { + for (p, f) in pngs.iter().zip(&frames) { + if let (true, Some(n)) = (fill(f), p.file_name()) { + names.push(n.to_string_lossy().into_owned()); + } } } + if let Some(found) = v["found"].as_array_mut() { + found.retain(|f| !names.iter().any(|n| f["says"].as_str().is_some_and(|s| s.starts_with(&format!("{n}:"))))); + } + } + let fs = raea_sys_light_meter::exposure::said(&v); // counted with the render's other findings (DR3) + if let (Some(m), Some(mut doc)) = (manifest, doc) { + if let Some(o) = doc.as_object_mut() { + o.insert("exposure".into(), v.clone()); + let _ = std::fs::write(m, doc.to_string()); + } } let mut msg = format!("{said}\nEXPOSURE {}", e.line()); for f in &fs { From 6d076868f0a8cdd974a1c1cec7d2620a1552bbab Mon Sep 17 00:00:00 2001 From: Zachary Guglin Date: Thu, 19 Nov 2026 15:03:07 -0800 Subject: [PATCH 3/6] raea check measures clipping on each channel: the mono fold read a clean master as clipped The clipping rule read the sound folded to mono, which ffmpeg's -ac 1 (and raea's fold, after it) make at (L + R) * 0.707: a centred voice comes out 3 dB hotter than either side. The 11-14 walkaround's v1 and v2 masters peak at -1.5 and -1.6 dBTP, no channel near full scale, yet their folds passed 1.0 and the check found 9 to 36 flat-topped runs over full scale. The agent spent two renders on it and then mastered the voice outside raea; raea check on those same files, each channel at its own level, finds no clipping. Clipping is a channel's: each is now read as it is (read::channels, FFmpeg's -ac 2 or raea's own decoders), and a second is the most runs any channel had. A clipped channel is still found, one that the fold would have hidden among the other too. The dropout and dead-stop rules keep reading the fold. Co-Authored-By: Claude Opus 5.5 --- apps/check/read/src/lib.rs | 25 ++++++++++++++++++++++++- apps/check/src/defects.rs | 20 ++++++++++++++------ apps/check/src/sound.rs | 12 ++---------- apps/check/src/tests.rs | 16 ++++++++++++---- 4 files changed, 52 insertions(+), 21 deletions(-) diff --git a/apps/check/read/src/lib.rs b/apps/check/read/src/lib.rs index ac56a8d80..200f594ea 100644 --- a/apps/check/read/src/lib.rs +++ b/apps/check/read/src/lib.rs @@ -152,7 +152,30 @@ pub fn flash_frames(video: &str, w: usize, h: usize, mut each: impl FnMut(f64, & /// `ffmpeg -i V -vn -ac 1 -ar RATE -f f32le -` (sound.rs sound): the sound folded to mono at `rate` pub fn mono(video: &str, rate: u32) -> R> { - Ok(raea_app_media::audio_folded(Path::new(video), rate, true, None, None, false)?.channels.swap_remove(0)) + if native() { + return Ok(raea_app_media::audio_folded(Path::new(video), rate, true, None, None, false)?.channels.swap_remove(0)); + } + ffmpeg_f32(video, 1, rate) +} + +/// `ffmpeg -i V -vn -ac N -ar R -f f32le -`: the sound as N interleaved channels of floats +fn ffmpeg_f32(video: &str, channels: u32, rate: u32) -> R> { + let out = Command::new("ffmpeg").args(["-v", "error", "-i", video, "-vn", "-ac", &channels.to_string(), "-ar", &rate.to_string(), "-f", "f32le", "-"]).output().map_err(|e| format!("ffmpeg: {e}"))?; + if !out.status.success() { + return Err(format!("ffmpeg couldn't read the sound of {video}: {}", String::from_utf8_lossy(&out.stderr).trim())); + } + Ok(out.stdout.chunks_exact(4).map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]])).collect()) +} + +/// `ffmpeg -i V -vn -ac 2 -ar R -f f32le -` (defects.rs clipping): each channel at its own level. Clipping is a +/// channel's: on the mono fold (L + R) * 0.707 a centred voice reads 3 dB hotter than either side, and a master at +/// -1.6 dBTP read as full scale (11-19, the walkaround's flat tops). +pub fn channels(video: &str, rate: u32) -> R>> { + if native() { + return Ok(raea_app_media::audio_folded(Path::new(video), rate, false, None, None, false)?.channels); + } + let x = ffmpeg_f32(video, 2, rate)?; + Ok(vec![x.iter().step_by(2).copied().collect(), x.iter().skip(1).step_by(2).copied().collect()]) } /// `ffmpeg -i V -vn -af ebur128=metadata=1,ametadata=print:key=lavfi.r128.M:file=- -f null -` (sound.rs sound): the diff --git a/apps/check/src/defects.rs b/apps/check/src/defects.rs index 1e699f9db..44f51b64e 100644 --- a/apps/check/src/defects.rs +++ b/apps/check/src/defects.rs @@ -11,8 +11,12 @@ pub(crate) const FULL: f32 = 0.999; pub(crate) const FLAT_LEVEL: f32 = 0.05; pub(crate) const CLIP_RUNS: usize = 3; -pub(crate) fn clipped(x: &[f32]) -> Vec<(f64, String)> { - let mut per: Vec = vec![0; x.len() / SR + 1]; +pub(crate) fn clipped(channels: &[Vec]) -> Vec<(f64, String)> { + let len = channels.iter().map(Vec::len).max().unwrap_or(0); + let mut per: Vec = vec![0; len / SR + 1]; + // each channel's runs, a second's count the most any channel had: clipping is a channel's + for x in channels { + let mut own: Vec = vec![0; len / SR + 1]; let mut i = 0; while i < x.len() { let v = x[i]; @@ -22,18 +26,22 @@ pub(crate) fn clipped(x: &[f32]) -> Vec<(f64, String)> { j += 1; } if j - i >= 3 { - per[i / SR] += 1; + own[i / SR] += 1; } } else if v.abs() >= FLAT_LEVEL { while j < x.len() && x[j].to_bits() == v.to_bits() { j += 1; } if j - i >= 4 { - per[i / SR] += 1; + own[i / SR] += 1; } } i = j; } + for (p, o) in per.iter_mut().zip(own) { + *p = (*p).max(o); + } + } let mut out = Vec::new(); let mut s = 0; while s < per.len() { @@ -145,8 +153,8 @@ pub(crate) fn moments(rule: &'static str, p: u8, found: Vec<(f64, String)>) -> i } /// The sound's defects on the decoded mix (sound.rs decodes it once) -pub(crate) fn sound_defects(x: &[f32], issues: &mut Vec, passed: &mut Vec) { - for (rule, found, ok) in [("clipping", clipped(x), "clipping: no flat-topped runs"), ("dropout", dropouts(x), "dropout: the sound never drops to silence and back")] { +pub(crate) fn sound_defects(x: &[f32], channels: &[Vec], issues: &mut Vec, passed: &mut Vec) { + for (rule, found, ok) in [("clipping", clipped(channels), "clipping: no flat-topped runs"), ("dropout", dropouts(x), "dropout: the sound never drops to silence and back")] { if found.is_empty() { passed.push(ok.into()); } diff --git a/apps/check/src/sound.rs b/apps/check/src/sound.rs index 83d4ecf27..6c02d71cd 100644 --- a/apps/check/src/sound.rs +++ b/apps/check/src/sound.rs @@ -162,17 +162,9 @@ pub(crate) fn bed_up(rms: &[f64], speak: &[bool]) -> Vec<(f64, String)> { /// The sound's rules on an export: dead stops always; with the voice's words, a stretch with no voice over it and a bed /// too close under it pub(crate) fn sound(video: &str, words: Option<&[Word]>, issues: &mut Vec, passed: &mut Vec) -> R<()> { - let x = if crate::read::native() { - crate::read::mono(video, SR as u32)? - } else { - let out = Command::new("ffmpeg").args(["-v", "error", "-i", video, "-vn", "-ac", "1", "-ar", &SR.to_string(), "-f", "f32le", "-"]).output().map_err(|e| format!("ffmpeg: {e}"))?; - if !out.status.success() { - return Err(format!("ffmpeg couldn't read the sound of {video}: {}", String::from_utf8_lossy(&out.stderr).trim())); - } - out.stdout.chunks_exact(4).map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]])).collect() - }; + let x = crate::read::mono(video, SR as u32)?; let (rms, bands) = levels(&x); - sound_defects(&x, issues, passed); + sound_defects(&x, &crate::read::channels(video, SR as u32)?, issues, passed); let found = |list: Vec<(f64, String)>, p: u8| crate::defects::moments("sound", p, list); let stops = dead_stops(&rms, &bands); if stops.is_empty() { diff --git a/apps/check/src/tests.rs b/apps/check/src/tests.rs index 9782530c5..144fb1f98 100644 --- a/apps/check/src/tests.rs +++ b/apps/check/src/tests.rs @@ -356,15 +356,23 @@ fn a_card_sharing_one_word_with_another_line_isnt_held_to_it() { fn clipping_is_found_and_a_loud_clean_tone_isnt() { // a clean tone near full scale, then the same tone clamped (a limiter's flat tops), then pushed past full scale let loud = tone(3.0, &[(0.0, 3.0, -1.0)]); - assert!(clipped(&loud).is_empty(), "{:?}", clipped(&loud)); + let one = |x: &[f32]| clipped(&[x.to_vec()]); + assert!(one(&loud).is_empty(), "{:?}", one(&loud)); let clamped: Vec = loud.iter().map(|v| (v * 10.0).clamp(-0.3, 0.3)).collect(); - assert_eq!(clipped(&clamped).len(), 1); + assert_eq!(one(&clamped).len(), 1); let over: Vec = loud.iter().map(|v| (v * 2.0).clamp(-1.0, 1.0)).collect(); - assert_eq!(clipped(&over).len(), 1); + assert_eq!(one(&over).len(), 1); // one transient hitting full scale once isn't clipping let mut once = tone(3.0, &[(0.0, 3.0, -20.0)]); once[SR] = 1.0; - assert!(clipped(&once).is_empty()); + assert!(one(&once).is_empty()); + // clipping is a channel's: one side clipped is found; a centred voice at -1.6 dBFS a side is clean, though the mono + // fold (L + R) * 0.707 takes it past full scale (11-19: the walkaround's master read as clipping) + assert_eq!(clipped(&[loud.clone(), over.clone()]).len(), 1); + let centred: Vec = loud.iter().map(|v| v * 0.92).collect(); + assert!(clipped(&[centred.clone(), centred.clone()]).is_empty()); + let folded: Vec = centred.iter().map(|v| (v + v) * 0.7071).collect(); + assert_eq!(one(&folded).len(), 1); } #[test] From 4c3a3ba799c0ccfd2f5fe48f40dc078b4251d541 Mon Sep 17 00:00:00 2001 From: Zachary Guglin Date: Thu, 19 Nov 2026 15:03:08 -0800 Subject: [PATCH 4/6] raea mixes and masters every render's sound: dialogue at one level, a leveller before the limiter, room tone at the joins The person's rule, 11-19: raea does its own mix on everything, then masters it. Until now a render was mastered only with a , and then by a gain and a limiter alone; levels were the agent's to set. The 11-14 walkaround's lines ran from -13.8 to -26.9 LUFS, so the agent measured each with ffmpeg and set its gainDb, the limiter still took up to 8 dB off phone speech's peaks, and the joins between lines fell to dead silence (a dropout, the check said). It then built and mastered the voice outside raea. - The mix (raea-sys-mix-level, level.rs): each line of dialogue, a clip on the voice bus or footage's sound on no bus, is measured over the span of its take it plays (BS.1770, gated) and brought to -23 LUFS, by at most 12 dB. Room tone and silent takes (under -45) keep their level; so do music, effects and every other bus, which sit under dialogue as their buses set them. gainDb is an offset on top. Live play and the render level the same span the same way (the clip's span comes from the SDK's data-clip-from and data-clip-to). - The master: a leveller in the limiter (from 9 dB under the ceiling, 3:1 on the mix's RMS, 10 ms in, 150 ms out) rounds the loud passages so the limiter catches only what's left, in live play, the render and seedling's limiter alike; the gain is made up for what it took. Without a , every render masters to -14 LUFS at -1 dBTP. - The joins (joins.rs): a gap the mix drops into dead silence and comes straight back from gets the room's own tone, the quietest stretch of sound near it, looped and faded, found at the level the master plays it. A pause that fades is the edit's and stays. The render fills them; live play hears the gap. The walkaround's v1 sound, each line straight from its take: before, its lines spread 6.7 LU and it had a dropout (and the mono fold's false clipping); now 2.1 LU, no dropout, -14.5 LUFS, -1.6 dBTP, with nothing set by hand. The Kia cut, whose voice was mastered outside raea, comes out at -14.1 LUFS, -1.2 dBTP. MASTER_VERSION 4. The guides stop telling agents to level lines or add a to be mastered. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 8 ++ Cargo.toml | 1 + docs/manual/concepts/rendering.md | 10 +- runtime/export/src/lib.rs | 2 + runtime/sound/src/names.rs | 4 +- sdk/guides/craft/motion/delivery.md | 4 +- sdk/guides/craft/motion/rendering.md | 5 +- sdk/guides/craft/motion/sound.md | 10 +- sdk/video/src/media.tsx | 3 + systems/mix/Cargo.toml | 2 + systems/mix/level/Cargo.toml | 10 ++ systems/mix/level/src/joins.rs | 157 +++++++++++++++++++++++++++ systems/mix/level/src/level.rs | 113 +++++++++++++++++++ systems/mix/level/src/lib.rs | 5 + systems/mix/src/lib.rs | 3 +- systems/mix/src/nodes/limiter.rs | 23 +++- systems/mix/src/offline.rs | 24 +++- tools/layers.toml | 3 + 18 files changed, 370 insertions(+), 17 deletions(-) create mode 100644 systems/mix/level/Cargo.toml create mode 100644 systems/mix/level/src/joins.rs create mode 100644 systems/mix/level/src/level.rs create mode 100644 systems/mix/level/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index df4730789..7cdaf9287 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -16101,10 +16101,18 @@ dependencies = [ "cpal 0.15.3", "firewheel", "raea-io-media", + "raea-sys-mix-level", "raea-sys-tone", "serde_json", ] +[[package]] +name = "raea-sys-mix-level" +version = "0.1.0" +dependencies = [ + "raea-io-media", +] + [[package]] name = "raea-sys-ml" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index fb321c6ef..9207068cf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -80,6 +80,7 @@ members = [ "systems/k", "systems/tone", "systems/mix", + "systems/mix/level", "systems/ml", "systems/mlx", "systems/kimodo", diff --git a/docs/manual/concepts/rendering.md b/docs/manual/concepts/rendering.md index 91d57d371..0de285ab9 100644 --- a/docs/manual/concepts/rendering.md +++ b/docs/manual/concepts/rendering.md @@ -109,8 +109,14 @@ out of order, a phrase `word()` can't find, or a missing prop. ## Loudness -When the composition has a ``, the export comes out at -14 LUFS integrated (within 1) with true peak at or below --1 dBTP, which is what the major platforms play back at. +raea mixes and masters every render's sound. First the mix: each line of dialogue (a clip on the `voice` bus, or +footage's own sound on no bus) is measured over the part of its take it plays and brought to the same level, so no line +is louder or quieter than the next. Room tone and silent takes keep their level, and so do music, effects and anything +on another bus, which sit under the dialogue as their buses set them. A clip's `gainDb` is an offset on top of the +level. Then the master: a leveller rounds the loud passages, the whole piece comes up to its loudness, and a true-peak +limiter holds it at the ceiling. A join that drops into dead silence gets the room's own tone. The export comes out at +-14 LUFS integrated (within 1) with true peak at or below -1 dBTP, which is what the major platforms play back at; +`` sets another target. ## Mistakes to avoid diff --git a/runtime/export/src/lib.rs b/runtime/export/src/lib.rs index 1ab310d0f..beae40336 100644 --- a/runtime/export/src/lib.rs +++ b/runtime/export/src/lib.rs @@ -188,6 +188,8 @@ fn push_sound(cx: &mut Ctx, props: &Map, skey: &str) { // a sound track with samples: a short window says so (decoding the whole take for it held a long take whole) let audible = *cx.audible.entry(file.clone()).or_insert_with(|| raea_io_media::decode_audio_window(&file, 0.0, 0.05).is_ok_and(|a| a.len() > 0)); if !muted && volume > 0.0 && audible { + // raea's mix: dialogue at one level, gainDb an offset on it (raea-sys-mix level.rs), as live play has it + let volume = volume * raea_sys_mix::level::gain(&file, num(props, "data-clip-from").unwrap_or(0.0), num(props, "data-clip-to"), props.get("data-bus").and_then(|v| v.as_str())) as f64; cx.sounds.push(Sound { src: file.display().to_string(), at, volume, rate: num(props, "data-playback-rate").unwrap_or(1.0), key: skey.to_string(), fade_in: num(props, "data-fade-in"), fade_out: num(props, "data-fade-out"), bus: props.get("data-bus").and_then(|v| v.as_str()).map(str::to_string) }); } } diff --git a/runtime/sound/src/names.rs b/runtime/sound/src/names.rs index 4dce90bf7..7de7aa989 100644 --- a/runtime/sound/src/names.rs +++ b/runtime/sound/src/names.rs @@ -84,11 +84,13 @@ pub fn sounds_into<'a>(items: impl IntoIterator` per - composition does it (`motion/sound`). A spec that names another number wins. +- Sound to -14 LUFS integrated with true peak at or below -1 dBTP: raea's mix and master do it on every render + (`motion/sound`). A spec that names another number wins: `` sets it. - Render the export at `--quality master` (motion graphics and type) or `--quality high` (a film), and versions made for notes with no `--quality` flag; name it in lowercase letters, digits and hyphens, with its shape and length: `spring-launch-9x16-15s.mp4`. Each version is a new file beside the last, never written over it. diff --git a/sdk/guides/craft/motion/rendering.md b/sdk/guides/craft/motion/rendering.md index 67d03113f..99fda4b89 100644 --- a/sdk/guides/craft/motion/rendering.md +++ b/sdk/guides/craft/motion/rendering.md @@ -24,7 +24,10 @@ - **Without `--manifest`**, `raea render` writes the export alone, without the measurements `raea check` reads. Render another deliverable from its own file the same way (`raea render NAME.tsx -o NAME.mp4 --manifest NAME.json`) and check it. -- **Loudness** comes out at -14 LUFS integrated (±1) with true peak at or below -1 dBTP when `` is in the tree. +- **Loudness** comes out at -14 LUFS integrated (±1) with true peak at or below -1 dBTP: raea levels each line of + dialogue (the `voice` bus, or footage on no bus) to the same loudness and masters the piece, so don't measure lines + or set `gainDb` to level them, and don't master the voice outside raea. `gainDb` is for a deliberate change, on top + of raea's level. `` sets another target. ## Why diff --git a/sdk/guides/craft/motion/sound.md b/sdk/guides/craft/motion/sound.md index 2a6fd6fb5..c5a9efffb 100644 --- a/sdk/guides/craft/motion/sound.md +++ b/sdk/guides/craft/motion/sound.md @@ -36,11 +36,14 @@ - Narration over the picture, when the footage has no voice for it, is `` from `raea/speech` (Pocket TTS, on this computer; `voice` picks a Pocket voice or a WAV of one). -- Master the sum with one `` per composition. It brings the whole mix to -14 LUFS - integrated and limits true peaks under the ceiling, which is what the floor's loudness defect asks for. +- raea mixes and masters every render: each line of dialogue (the `voice` bus, or footage on no bus) comes to the same + loudness, a join into dead silence gets the room's tone, and the whole mix comes to -14 LUFS integrated with true + peaks under -1 dBTP, which is what the floor's loudness defect asks for. Don't level lines with `gainDb` or master + the voice outside raea. One `` per composition sets another target. - Put the sounds in the tree in time order. The engine keys a sound's stretch by its place in the tree, so adding a sound in the middle of the list can split the ones after it. -- Balance relative to the voice. The approved pieces, mastered by `` to -14 LUFS, sit like this: +- Balance relative to the voice. The approved pieces, mastered to -14 LUFS, sit like this (each a `gainDb` on top of + raea's level for the voice, or on the sound's own level for the rest): | sound | with a voice | without one (promo, deck, game) | |---|---|---| @@ -110,7 +113,6 @@ ends up shouting over the narrator while another is inaudible. - `loop` on an `