From d8945089edcda7aba8821c09754015bc154ba13d Mon Sep 17 00:00:00 2001 From: shinyobjectz Date: Wed, 18 Nov 2026 15:33:57 -0800 Subject: [PATCH] raea check reads a video without FFmpeg: raea's own decoders stand in where it isn't installed 0.1.7's raea check refused any video without a system ffprobe and ffmpeg, though raea media probe and loudness never needed them; Windows never has FFmpeg. Every read check makes now has a native reader in a new crate, apps/check/read (raea-app-check-read): the probe, each frame's light, the dark crops blotching scores, the flashing screen's area- and point-sampled frames, the sound folded to mono, momentary loudness, freezes, integrated loudness and true peak, and a caption's contrast. FFmpeg stays the reader where it's on the PATH (the reference the rules were tuned on); RAEA_CHECK_NATIVE=1 forces the native readers. tests/parity.rs holds each native read to FFmpeg's on a clip FFmpeg makes: crops, light, sound, meter, freeze and loudness agree to within a hundredth; flash frames to 4.4 levels. raea check gives identical output with and without FFmpeg on PATH for a moving clip with a freeze and loud sound, a render, and a strobe with stray frames. --- Cargo.lock | 12 ++ Cargo.toml | 1 + apps/check/Cargo.toml | 2 + apps/check/read/Cargo.toml | 15 +++ apps/check/read/src/lib.rs | 188 ++++++++++++++++++++++++++++++++ apps/check/read/tests/parity.rs | 126 +++++++++++++++++++++ apps/check/src/defects.rs | 3 + apps/check/src/flash.rs | 15 ++- apps/check/src/lib.rs | 49 +-------- apps/check/src/picture.rs | 30 +++-- apps/check/src/sound.rs | 55 ++++++---- apps/check/src/text.rs | 3 +- apps/media/src/change.rs | 8 +- apps/media/src/lib.rs | 4 +- apps/media/src/probe.rs | 2 +- docs/direction.feature | 3 + tools/layers.toml | 1 + 17 files changed, 433 insertions(+), 84 deletions(-) create mode 100644 apps/check/read/Cargo.toml create mode 100644 apps/check/read/src/lib.rs create mode 100644 apps/check/read/tests/parity.rs diff --git a/Cargo.lock b/Cargo.lock index 5be179e8..1d469dad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13763,6 +13763,7 @@ dependencies = [ "dunce", "raea-app-check-flash", "raea-app-check-inbox", + "raea-app-check-read", "raea-core-brief", "raea-core-facts", "raea-io-ffmpeg", @@ -13784,6 +13785,17 @@ dependencies = [ "serde_json", ] +[[package]] +name = "raea-app-check-read" +version = "0.1.0" +dependencies = [ + "raea-app-media", + "raea-io-home", + "raea-io-media", + "raea-sys-measure", + "serde_json", +] + [[package]] name = "raea-app-ci" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 24cdbaff..44288ffa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -189,6 +189,7 @@ members = [ "apps/check", "apps/check/flash", "apps/check/inbox", + "apps/check/read", "apps/direction", "apps/notes", "apps/fingerprint", diff --git a/apps/check/Cargo.toml b/apps/check/Cargo.toml index ec8ca01e..4b914bd8 100644 --- a/apps/check/Cargo.toml +++ b/apps/check/Cargo.toml @@ -19,6 +19,8 @@ raea-io-trace = { path = "../../io/trace" } raea-io-ffmpeg = { path = "../../io/ffmpeg" } # caption contrast and loudness, measured as `raea world` measures a render (DR2) raea-sys-measure = { path = "../../systems/measure" } +# what it reads of a video, through FFmpeg or raea's own decoders where it isn't installed +raea-app-check-read = { path = "read" } # the flashing screen (WCAG 2.3.1), split out to keep this crate small raea-app-check-flash = { path = "flash" } # what the person sent from the raea app, handed to the agent (split out to keep this crate small) diff --git a/apps/check/read/Cargo.toml b/apps/check/read/Cargo.toml new file mode 100644 index 00000000..2e3a3695 --- /dev/null +++ b/apps/check/read/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "raea-app-check-read" +version = "0.1.0" +edition = "2024" +license = "FSL-1.1-ALv2" +publish = false +description = "What `raea check` reads of a video: FFmpeg's probe, and raea's own decoders standing in for FFmpeg where it isn't installed" + +[dependencies] +serde_json = "1" +raea-io-home = { path = "../../../io/home" } +raea-io-media = { path = "../../../io/media", default-features = false } +raea-app-media = { path = "../../media" } +# caption contrast, as `raea world` measures it (DR2) +raea-sys-measure = { path = "../../../systems/measure" } diff --git a/apps/check/read/src/lib.rs b/apps/check/read/src/lib.rs new file mode 100644 index 00000000..c5ac82ca --- /dev/null +++ b/apps/check/read/src/lib.rs @@ -0,0 +1,188 @@ +//! What `raea check` reads of a video: through FFmpeg where it's on the PATH (the reference its rules were tuned on), +//! else through raea's own decoders (raea-app-media), so `raea check` works wherever `raea media probe` and `raea media +//! loudness` do, with nothing to install. 0.1.7 refused any video without FFmpeg, which Windows never has (Shane 11-18: +//! "can we make it so ffmpeg does work the way we need it?"). RAEA_CHECK_NATIVE=1 reads natively with FFmpeg there too; +//! tests/parity.rs holds the two to the same answers. Each reader names the FFmpeg command it stands in for. +use raea_io_media::VideoDecoder; +use serde_json::Value; +use std::path::Path; +use std::process::Command; + +type R = Result; + +/// Whether to read natively: FFmpeg's ffmpeg or ffprobe isn't on the PATH, or RAEA_CHECK_NATIVE is on +pub fn native() -> bool { + raea_io_home::env_on("RAEA_CHECK_NATIVE") || !(on_path("ffmpeg") && on_path("ffprobe")) +} + +fn on_path(tool: &str) -> bool { + let exe = if cfg!(windows) { format!("{tool}.exe") } else { tool.to_string() }; + std::env::var_os("PATH").is_some_and(|p| std::env::split_paths(&p).any(|d| d.join(&exe).is_file())) +} + +fn open(video: &str) -> R<(Box, raea_io_media::Info)> { + let d = raea_app_media::backend()?.open_video(Path::new(video))?; + let info = d.info().clone(); + Ok((d, info)) +} + +/// A luma byte as FFmpeg's format=gray gives it from a limited-range picture: full range, 16-235 spread to 0-255 and +/// rounded (FFmpeg 9: 16 is 0, 126 is 128, 235 is 255) +fn gray(y: u8) -> u8 { + (((y.clamp(16, 235) as u32 - 16) * 255 + 109) / 219) as u8 +} + +/// What `raea check` needs to know of a video before it reads it +pub struct Probe { + pub w: f64, + pub h: f64, + pub duration: f64, + /// when the picture ends, on the clock FFmpeg's filters read: it starts at the file's first stream, sound can run on + /// past the picture, and the container's duration counts that + pub picture: f64, + pub audio: bool, +} + +/// The picture's size, the file's duration, where the picture ends and whether there's sound: ffprobe's stream and +/// format entries, or raea's own probe without FFmpeg +pub fn probe(video: &str) -> R { + if native() { + let p = raea_app_media::probe_of(Path::new(video))?; + let v = p.video.as_ref().ok_or("no video stream")?; + let picture = p.track.as_ref().map_or(v.duration, |t| t.duration); + return Ok(Probe { w: v.width as f64, h: v.height as f64, duration: p.duration.max(picture), picture, audio: p.audio.is_some() }); + } + let entries = "stream=codec_type,width,height,r_frame_rate,duration,start_time:format=duration,start_time"; + let out = Command::new("ffprobe").args(["-v", "error", "-show_entries", entries, "-of", "json", video]).output().map_err(|e| format!("ffprobe: {e}"))?; + let v: Value = serde_json::from_slice(&out.stdout).map_err(|e| format!("ffprobe: {e}"))?; + let streams = v["streams"].as_array().cloned().unwrap_or_default(); + let vid = streams.iter().find(|s| s["codec_type"] == "video").ok_or("no video stream")?; + let seconds = |field: &Value| -> Option { field.as_str()?.parse().ok() }; + let duration = seconds(&v["format"]["duration"]).unwrap_or(0.0); + let origin = seconds(&v["format"]["start_time"]).unwrap_or(0.0); + let picture = match seconds(&vid["duration"]) { + Some(length) => seconds(&vid["start_time"]).unwrap_or(origin) - origin + length, + None => last_packet_end(video).map_or(duration, |end| end - origin), + }; + let audio = streams.iter().any(|s| s["codec_type"] == "audio"); + Ok(Probe { w: vid["width"].as_f64().unwrap_or(1920.0), h: vid["height"].as_f64().unwrap_or(1080.0), duration, picture, audio }) +} + +/// Where the picture's last packet ends, for a container that doesn't state the video stream's own length (Matroska) +fn last_packet_end(video: &str) -> Option { + let out = Command::new("ffprobe").args(["-v", "error", "-select_streams", "v:0", "-show_entries", "packet=pts_time,duration_time", "-of", "csv=p=0", video]).output().ok()?; + String::from_utf8_lossy(&out.stdout) + .lines() + .filter_map(|l| { + let mut parts = l.split(','); + let t: f64 = parts.next()?.trim().parse().ok()?; + Some(t + parts.next().and_then(|d| d.trim().parse::().ok()).unwrap_or(0.0)) + }) + .max_by(f64::total_cmp) +} + +/// `ffmpeg -i V -vf scale=32:18:flags=area,format=gray -f rawvideo -` averaged a frame, and the rate `ffmpeg -i` says +/// (defects.rs lumas): each frame's mean light, 0 to 1, and the frames a second +pub fn lumas(video: &str) -> R<(Vec, f64)> { + let (mut d, info) = open(video)?; + let mut out = Vec::new(); + for (_, at) in raea_app_media::frames_of(Path::new(video), &info, None, None) { + let (y, ..) = raea_app_media::yuv(d.frame_nv12_at(at)?); + out.push(y.iter().map(|&p| gray(p) as f32).sum::() / (y.len().max(1) as f32 * 255.0)); + } + Ok((out, raea_app_media::standard_fps(info.fps))) +} + +/// `ffmpeg -i V -vf fps=0.5,crop=W:H:0:0,format=gray -f rawvideo -` (picture.rs blotching): every 2 s, the picture's +/// top-left W x H in grey +pub fn crops(video: &str, w: usize, h: usize, mut each: impl FnMut(&[u8])) -> R<()> { + let (mut d, info) = open(video)?; + let frames = raea_app_media::frames_of(Path::new(video), &info, None, None); + for (tick, _) in raea_app_media::times(raea_app_media::video_span(Path::new(video), &info), None, None, 0.5) { + // the last frame before the tick's middle, as the fps filter picks it (by the frames' own times: a read at the + // middle less a microsecond came back a frame late) + let Some(&(_, at)) = frames.iter().take_while(|(pts, _)| *pts < tick + 1.0).last() else { continue }; + let f = d.frame_nv12_at(at)?; + let fw = f.width as usize; + let (y, ..) = raea_app_media::yuv(f); + let crop: Vec = (0..h.min(y.len() / fw.max(1))).flat_map(|r| y[r * fw..r * fw + w.min(fw)].iter().map(|&p| gray(p))).collect(); + each(&crop); + } + Ok(()) +} + +/// flash.rs's graph (showinfo, the picture at W x H area-sampled beside it point-sampled, rgb24): every frame with +/// its time, until `each` says stop. How many frames were read. +pub fn flash_frames(video: &str, w: usize, h: usize, mut each: impl FnMut(f64, &[u8])) -> R { + let (mut d, info) = open(video)?; + let mut buf = vec![0u8; w * 2 * h * 3]; + let mut n = 0; + for (pts, at) in raea_app_media::frames_of(Path::new(video), &info, None, None) { + let f = d.frame_at(at)?; + let (sw, sh) = (f.width as usize, f.height as usize); + for y in 0..h { + let (y0, y1) = (y * sh / h, ((y + 1) * sh / h).max(y * sh / h + 1)); + for x in 0..w { + let (x0, x1) = (x * sw / w, ((x + 1) * sw / w).max(x * sw / w + 1)); + let mut sum = [0u32; 3]; + for sy in y0..y1 { + for p in f.rgba[(sy * sw + x0) * 4..(sy * sw + x1) * 4].chunks_exact(4) { + (0..3).for_each(|c| sum[c] += p[c] as u32); + } + } + let count = ((y1 - y0) * (x1 - x0)) as u32; + let near = ((y * sh + sh / 2) / h * sw + (x * sw + sw / 2) / w) * 4; + for c in 0..3 { + buf[(y * w * 2 + x) * 3 + c] = ((sum[c] + count / 2) / count) as u8; + buf[(y * w * 2 + w + x) * 3 + c] = f.rgba[near + c]; + } + } + } + each(pts, &buf); + n += 1; + } + Ok(n) +} + +/// `ffmpeg -i V -vn -ac 1 -ar RATE -f f32le -` (sound.rs sound): the sound folded to mono at `rate` +pub fn mono(video: &str, rate: u32) -> R> { + Ok(raea_app_media::audio_folded(Path::new(video), rate, true, None, None, false)?.channels.swap_remove(0)) +} + +/// `ffmpeg -i V -vn -af ebur128=metadata=1,ametadata=print:key=lavfi.r128.M:file=- -f null -` (sound.rs sound): the +/// momentary loudness (400 ms) every 100 ms, each time and its LUFS +pub fn meter(video: &str) -> R<(Vec, Vec)> { + let a = raea_app_media::audio_folded(Path::new(video), raea_io_media::audio::RATE, false, None, None, false)?; + let n = (a.duration() * 10.0).floor() as usize; + Ok(((0..n).map(|i| i as f64 / 10.0).collect(), raea_io_media::audio::momentary(&a, 10.0, n))) +} + +/// `ffmpeg -i V -an -vf freezedetect=n=NOISE_DB:d=MIN -f null -`'s log (sound.rs freezes reads it either way) +pub fn freezes(video: &str, noise_db: f64, min: f64) -> R { + let found = raea_app_media::freezes(Path::new(video), 10f64.powf(noise_db / 20.0), min)?; + Ok(found.into_iter().map(|(s, end)| match end { + Some((d, e)) => format!("freeze_start: {s}\nfreeze_duration: {d}\nfreeze_end: {e}\n"), + None => format!("freeze_start: {s}\n"), + }).collect()) +} + +/// raea_sys_measure::contrast's read (`ffmpeg -ss AT -i V -frames:v 1 -vf crop=... -pix_fmt rgb24`): a text's contrast +/// against what's behind it, None when the crop falls outside the frame +pub fn contrast(video: &str, at: f64, crop: [u32; 4], color: &str) -> Option { + let [x, y, w, h] = [crop[0] & !1, crop[1] & !1, crop[2].max(2) & !1, crop[3].max(2) & !1].map(|v| v as usize); + let (mut d, _) = open(video).ok()?; + let f = d.frame_at(at).ok()?; + let fw = f.width as usize; + if x + w > fw || y + h > f.height as usize { + return None; + } + let rgb: Vec = (y..y + h).flat_map(|r| f.rgba[(r * fw + x) * 4..(r * fw + x + w) * 4].chunks_exact(4).flat_map(|p| [p[0], p[1], p[2]]).collect::>()).collect(); + raea_sys_measure::ratio_behind(&rgb, raea_sys_measure::lum(raea_sys_measure::rgb_of(color))) +} + +/// raea_sys_measure::loudness's read (`ffmpeg -i V -vn -af ebur128=peak=true -f null -`): the integrated loudness and +/// true peak, None when there's no sound +pub fn loudness(video: &str) -> Option<(f64, Option)> { + let a = raea_app_media::audio_folded(Path::new(video), raea_io_media::audio::RATE, false, None, None, false).ok().filter(|a| a.len() > 0)?; + Some((raea_io_media::audio::lufs(&a), Some(raea_io_media::audio::true_peak_db(&a)))) +} diff --git a/apps/check/read/tests/parity.rs b/apps/check/read/tests/parity.rs new file mode 100644 index 00000000..eb522971 --- /dev/null +++ b/apps/check/read/tests/parity.rs @@ -0,0 +1,126 @@ +//! Each native reader against the FFmpeg read it stands in for (lib.rs names the command), on a clip FFmpeg makes: a +//! moving test picture that freezes for its last 4 s, over stereo sound whose level swells. `raea check` without FFmpeg +//! must reach the verdicts it reaches with it, so each answer is held to FFmpeg's within what a rule could notice. +//! Skipped where FFmpeg isn't installed (there is nothing to compare against). + +use std::path::Path; +use std::process::Command; + +fn ffmpeg(args: &[&str]) -> (Vec, String) { + let out = Command::new("ffmpeg").args(["-hide_banner", "-nostats", "-nostdin"]).args(args).output().expect("ffmpeg runs"); + assert!(out.status.success(), "ffmpeg {args:?}: {}", String::from_utf8_lossy(&out.stderr)); + (out.stdout, String::from_utf8_lossy(&out.stderr).into_owned()) +} + +fn clip(dir: &Path) -> String { + let file = dir.join("clip.mp4").to_string_lossy().into_owned(); + let picture = "testsrc2=size=320x180:rate=30:duration=5,tpad=stop_mode=clone:stop_duration=4"; + let sound = "aevalsrc=0.4*sin(2*PI*440*t)*(0.55+0.45*sin(2*PI*0.25*t))|0.25*sin(2*PI*660*t):s=48000:d=9"; + ffmpeg(&["-y", "-f", "lavfi", "-i", picture, "-f", "lavfi", "-i", sound, "-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "aac", "-b:a", "192k", "-shortest", &file]); + file +} + +/// The mean absolute difference of two byte runs, in levels +fn apart(a: &[u8], b: &[u8]) -> f64 { + a.iter().zip(b).map(|(x, y)| x.abs_diff(*y) as u64).sum::() as f64 / a.len().max(1) as f64 +} + +/// Root-mean-square level in dB of each 100 ms of `x` at `rate` +fn levels(x: &[f32], rate: usize) -> Vec { + x.chunks_exact(rate / 10).map(|c| 10.0 * (c.iter().fold(0.0, |s, v| s + v * v) as f64 / c.len() as f64 + 1e-12).log10()).collect() +} + +/// Each measure's name, how far the native read is from FFmpeg's, and how far it may be: all printed, then held +struct Off(Vec<(String, f64, f64)>); + +impl Off { + fn by(&mut self, what: impl Into, off: f64, most: f64) { + self.0.push((what.into(), off, most)); + } +} + +#[test] +fn the_native_reads_match_ffmpeg_s() { + if Command::new("ffmpeg").arg("-version").output().is_err() { + eprintln!("skipped: no FFmpeg to compare against"); + return; + } + let dir = std::env::temp_dir().join(format!("raea-check-read-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let v = &clip(&dir); + let mut off = Off(vec![]); + let by_ffmpeg = raea_app_check_read::probe(v).unwrap(); + // SAFETY: set before any reader runs, on this test's only thread + unsafe { std::env::set_var("RAEA_CHECK_NATIVE", "1") }; + assert!(raea_app_check_read::native()); + let p = raea_app_check_read::probe(v).unwrap(); + assert_eq!((p.w, p.h, p.audio), (by_ffmpeg.w, by_ffmpeg.h, by_ffmpeg.audio)); + assert!((p.duration - by_ffmpeg.duration).abs() < 0.05 && (p.picture - by_ffmpeg.picture).abs() < 0.05, "{} {} vs {} {}", p.duration, p.picture, by_ffmpeg.duration, by_ffmpeg.picture); + + // flash-frame's light per frame + let (gray, _) = ffmpeg(&["-v", "error", "-i", v, "-map", "0:v:0", "-vf", "scale=32:18:flags=area,format=gray", "-f", "rawvideo", "-"]); + let theirs: Vec = gray.chunks_exact(32 * 18).map(|f| f.iter().map(|&b| b as f64).sum::() / (32.0 * 18.0 * 255.0)).collect(); + let (ours, fps) = raea_app_check_read::lumas(v).unwrap(); + assert_eq!((ours.len(), fps), (theirs.len(), 30.0)); + off.by("a frame's light, of 1", ours.iter().zip(&theirs).map(|(a, b)| (*a as f64 - b).abs()).fold(0.0, f64::max), 0.01); + + // blotching's dark crops every 2 s + let (w, h) = (320, 176); + let (gray, _) = ffmpeg(&["-v", "error", "-i", v, "-vf", &format!("fps=0.5,crop={w}:{h}:0:0,format=gray"), "-f", "rawvideo", "-"]); + let mut crops = vec![]; + raea_app_check_read::crops(v, w, h, |c| crops.push(c.to_vec())).unwrap(); + assert_eq!(crops.len(), gray.len() / (w * h)); + for (i, (a, b)) in crops.iter().zip(gray.chunks_exact(w * h)).enumerate() { + let bias = a.iter().zip(b).map(|(x, y)| *x as f64 - *y as f64).sum::() / a.len() as f64; + off.by(format!("blotching crop {i}, levels (bias {bias:+.2})"), apart(a, b), 3.0); + } + + // the flashing screen's frames: area-sampled beside point-sampled RGB + let (fw, fh) = (192, 108); + let graph = format!("split=2[area][point];[area]scale={fw}:{fh}:flags=area[a];[point]scale={fw}:{fh}:flags=neighbor[b];[a][b]hstack=inputs=2,format=rgb24"); + let (rgb, _) = ffmpeg(&["-v", "error", "-i", v, "-map", "0:v:0", "-vf", &graph, "-f", "rawvideo", "-"]); + let (mut frames, mut times) = (vec![], vec![]); + let n = raea_app_check_read::flash_frames(v, fw, fh, |t, buf| { + times.push(t); + frames.push(buf.to_vec()); + }) + .unwrap(); + assert_eq!(n, rgb.len() / (fw * 2 * fh * 3)); + assert!(times.windows(2).all(|t| (t[1] - t[0] - 1.0 / 30.0).abs() < 1e-3), "frame times step by a frame"); + off.by("a flash frame, levels", frames.iter().zip(rgb.chunks_exact(fw * 2 * fh * 3)).map(|(a, b)| apart(a, b)).fold(0.0, f64::max), 6.0); + + // the sound folded to mono, 100 ms at a time + let (pcm, _) = ffmpeg(&["-v", "error", "-i", v, "-vn", "-ac", "1", "-ar", "24000", "-f", "f32le", "-"]); + let theirs = levels(&pcm.chunks_exact(4).map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]])).collect::>(), 24000); + let ours = levels(&raea_app_check_read::mono(v, 24000).unwrap(), 24000); + assert!(ours.len().abs_diff(theirs.len()) <= 1, "{} windows vs {}", ours.len(), theirs.len()); + off.by("100 ms of mono sound, dB", ours.iter().zip(&theirs).skip(1).map(|(a, b)| (a - b).abs()).fold(0.0, f64::max), 0.5); + + // momentary loudness every 100 ms + let (log, _) = ffmpeg(&["-i", v, "-vn", "-af", "ebur128=metadata=1,ametadata=print:key=lavfi.r128.M:file=-", "-f", "null", "-"]); + let theirs: Vec = String::from_utf8_lossy(&log).lines().filter_map(|l| l.strip_prefix("lavfi.r128.M=")?.trim().parse().ok()).collect(); + let (at, ours) = raea_app_check_read::meter(v).unwrap(); + assert!(ours.len().abs_diff(theirs.len()) <= 2 && at.len() == ours.len(), "{} readings vs {}", ours.len(), theirs.len()); + off.by("a momentary loudness, LU", ours.iter().zip(&theirs).skip(4).map(|(a, b)| (a - b).abs()).fold(0.0, f64::max), 0.5); + + // the freeze: 5 s in, to the end + let (_, log) = ffmpeg(&["-i", v, "-an", "-vf", "freezedetect=n=-60dB:d=3", "-f", "null", "-"]); + let starts = |log: &str| log.lines().filter_map(|l| l.split("freeze_start: ").nth(1)?.trim().parse::().ok()).collect::>(); + let (theirs, ours) = (starts(&log), starts(&raea_app_check_read::freezes(v, -60.0, 3.0).unwrap())); + assert!(ours.len() == 1 && theirs.len() == 1, "freezes from {ours:?} vs {theirs:?}"); + off.by("the freeze's start, s", (ours[0] - theirs[0]).abs(), 0.1); + + // the mix's loudness and a caption's contrast, against raea-sys-measure's FFmpeg reads + let (i, tp) = raea_app_check_read::loudness(v).unwrap(); + let (fi, ftp) = raea_sys_measure::loudness(Path::new(v)).unwrap(); + off.by("integrated loudness, LU", (i - fi).abs(), 0.3); + off.by("true peak, dB", (tp.unwrap() - ftp.unwrap()).abs(), 0.5); + let ours = raea_app_check_read::contrast(v, 1.0, [40, 60, 120, 60], "rgb(255, 255, 255)").unwrap(); + let theirs = raea_sys_measure::contrast(Path::new(v), 1.0, [40, 60, 120, 60], "rgb(255, 255, 255)").unwrap(); + off.by("a caption's contrast, share", (ours / theirs - 1.0).abs(), 0.05); + let _ = std::fs::remove_dir_all(&dir); + for (what, by, most) in &off.0 { + eprintln!("{} {what}: {by:.4} (at most {most})", if by < most { "ok " } else { "OVER" }); + } + assert!(off.0.iter().all(|(_, by, most)| by < most), "a native read is further from FFmpeg's than a rule allows"); +} diff --git a/apps/check/src/defects.rs b/apps/check/src/defects.rs index d3ffdb77..1e699f9d 100644 --- a/apps/check/src/defects.rs +++ b/apps/check/src/defects.rs @@ -113,6 +113,9 @@ pub(crate) fn flash_frames(luma: &[f32], fps: f64) -> Vec<(f64, String)> { /// Each frame's mean brightness (0-1), decoded small and grey fn lumas(video: &str) -> R<(Vec, f64)> { + if crate::read::native() { + return crate::read::lumas(video); + } let out = Command::new("ffmpeg") .args(["-v", "error", "-nostdin", "-i", video, "-map", "0:v:0", "-vf", "scale=32:18:flags=area,format=gray", "-f", "rawvideo", "-"]) .output() diff --git a/apps/check/src/flash.rs b/apps/check/src/flash.rs index 8a2477c7..a6981007 100644 --- a/apps/check/src/flash.rs +++ b/apps/check/src/flash.rs @@ -34,6 +34,14 @@ pub(crate) fn log_level(line: &str) -> Option<&str> { pub(crate) fn flashing(video: &str, probe: &Probe, issues: &mut Vec, passed: &mut Vec) -> R<()> { let w = FLASH_W; let h = (((w as f64 * probe.h / probe.w.max(1.0)) / 2.0).round() as usize * 2).max(6); + if crate::read::native() { + let mut screen = FlashScreen::new(w, h); + if crate::read::flash_frames(video, w, h, |t, buf| screen.push(buf, t))? == 0 { + return Err(format!("{video}: flashing can't be screened, no frame decoded")); + } + screened(screen, 0, "", issues, passed); + return Ok(()); + } let graph = format!("showinfo,split=2[area][point];[area]scale={w}:{h}:flags=area[a];[point]scale={w}:{h}:flags=neighbor[b];[a][b]hstack=inputs=2,format=rgb24"); let sync = raea_io_ffmpeg::passthrough(); let mut child = Command::new("ffmpeg") @@ -92,6 +100,12 @@ pub(crate) fn flashing(video: &str, probe: &Probe, issues: &mut Vec, pass if !status.success() || frames == 0 { return Err(format!("{video}: flashing can't be screened, FFmpeg decoded {frames} frames ({first})")); } + screened(screen, errors, &first, issues, passed); + Ok(()) +} + +/// What the screen found, after `errors` errors decoding the picture (`first` the first of them) +fn screened(screen: FlashScreen, errors: usize, first: &str, issues: &mut Vec, passed: &mut Vec) { if errors > 0 { issues.push(Issue { rule: "decode", @@ -110,5 +124,4 @@ pub(crate) fn flashing(video: &str, probe: &Probe, issues: &mut Vec, pass None if errors == 0 => passed.push("flashing: no more than 3 flashes a second found (a screen, not a certification)".to_string()), None => {} } - Ok(()) } diff --git a/apps/check/src/lib.rs b/apps/check/src/lib.rs index 2e6acb2e..c7f3577f 100644 --- a/apps/check/src/lib.rs +++ b/apps/check/src/lib.rs @@ -24,6 +24,8 @@ mod defects; use defects::*; mod text; use text::*; +/// a video read through FFmpeg, or raea's own decoders where it isn't installed +use raea_app_check_read::{self as read, Probe}; pub mod quick; pub mod hook; pub use raea_app_check_inbox as inbox; @@ -145,7 +147,7 @@ fn measured(args: &[String], taste: Vec) -> R<(String, Vec, Vec = match flag(args, "--manifest") { Some(m) => Some(serde_json::from_str(&std::fs::read_to_string(m).map_err(|e| format!("{m}: {e}"))?).map_err(|e| format!("{m}: {e}"))?), @@ -228,47 +230,6 @@ fn report(video: &str, issues: Vec, more: Vec, passed: Vec Option { - let out = Command::new("ffprobe").args(["-v", "error", "-select_streams", "v:0", "-show_entries", "packet=pts_time,duration_time", "-of", "csv=p=0", video]).output().ok()?; - String::from_utf8_lossy(&out.stdout) - .lines() - .filter_map(|l| { - let mut parts = l.split(','); - let t: f64 = parts.next()?.trim().parse().ok()?; - Some(t + parts.next().and_then(|d| d.trim().parse::().ok()).unwrap_or(0.0)) - }) - .max_by(f64::total_cmp) -} - -impl Probe { - fn of(video: &str) -> R { - let entries = "stream=codec_type,width,height,r_frame_rate,duration,start_time:format=duration,start_time"; - let out = Command::new("ffprobe").args(["-v", "error", "-show_entries", entries, "-of", "json", video]).output().map_err(|e| format!("ffprobe: {e}"))?; - let v: Value = serde_json::from_slice(&out.stdout).map_err(|e| format!("ffprobe: {e}"))?; - let streams = v["streams"].as_array().cloned().unwrap_or_default(); - let vid = streams.iter().find(|s| s["codec_type"] == "video").ok_or("no video stream")?; - let seconds = |field: &Value| field.as_str().and_then(|d| d.parse::().ok()); - let duration = seconds(&v["format"]["duration"]).unwrap_or(0.0); - let origin = seconds(&v["format"]["start_time"]).unwrap_or(0.0); - let picture = match seconds(&vid["duration"]) { - Some(length) => seconds(&vid["start_time"]).unwrap_or(origin) - origin + length, - None => last_packet_end(video).map_or(duration, |end| end - origin), - }; - let audio = streams.iter().any(|s| s["codec_type"] == "audio"); - Ok(Probe { w: vid["width"].as_f64().unwrap_or(1920.0), h: vid["height"].as_f64().unwrap_or(1080.0), duration, picture, audio }) - } + s.push_str(&format!("raea check: {}", finding::count(&found))); + Ok((s, bad)) } diff --git a/apps/check/src/picture.rs b/apps/check/src/picture.rs index b10f1ee9..074f4fc1 100644 --- a/apps/check/src/picture.rs +++ b/apps/check/src/picture.rs @@ -111,17 +111,9 @@ pub(crate) fn blotching(video: &str, probe: &Probe, issues: &mut Vec, pas return Ok(()); } let (w, h) = (probe.w as usize / 8 * 8, probe.h as usize / 8 * 8); - let mut child = Command::new("ffmpeg") - .args(["-v", "error", "-i", video, "-vf", &format!("fps=0.5,crop={w}:{h}:0:0,format=gray"), "-f", "rawvideo", "-"]) - .stdout(std::process::Stdio::piped()) - .spawn() - .map_err(|e| format!("ffmpeg: {e}"))?; - let mut out = child.stdout.take().unwrap(); - let mut buf = vec![0u8; w * h]; - let (mut bad, mut seen, mut worst) = (Vec::new(), 0, 0f64); - let mut t = 0.0; - while std::io::Read::read_exact(&mut out, &mut buf).is_ok() { - if let Some(s) = blotch_score(&buf, w, h) { + let (mut bad, mut seen, mut worst, mut t) = (Vec::new(), 0, 0f64, 0.0); + let mut frame = |buf: &[u8]| { + if let Some(s) = blotch_score(buf, w, h) { seen += 1; worst = worst.max(s); if std::env::var_os("RAEA_CHECK_DEBUG").is_some() { @@ -132,8 +124,22 @@ pub(crate) fn blotching(video: &str, probe: &Probe, issues: &mut Vec, pas } } t += 2.0; + }; + if crate::read::native() { + crate::read::crops(video, w, h, &mut frame)?; + } else { + let mut child = Command::new("ffmpeg") + .args(["-v", "error", "-i", video, "-vf", &format!("fps=0.5,crop={w}:{h}:0:0,format=gray"), "-f", "rawvideo", "-"]) + .stdout(std::process::Stdio::piped()) + .spawn() + .map_err(|e| format!("ffmpeg: {e}"))?; + let mut out = child.stdout.take().unwrap(); + let mut buf = vec![0u8; w * h]; + while std::io::Read::read_exact(&mut out, &mut buf).is_ok() { + frame(&buf); + } + let _ = child.wait(); } - let _ = child.wait(); if !bad.is_empty() { let at = bad[0].0; let share = bad.len() as f64 / seen.max(1) as f64; diff --git a/apps/check/src/sound.rs b/apps/check/src/sound.rs index e4ed1454..83d4ecf2 100644 --- a/apps/check/src/sound.rs +++ b/apps/check/src/sound.rs @@ -162,11 +162,15 @@ pub(crate) fn bed_up(rms: &[f64], speak: &[bool]) -> Vec<(f64, String)> { /// The sound's rules on an export: dead stops always; with the voice's words, a stretch with no voice over it and a bed /// too close under it pub(crate) fn sound(video: &str, words: Option<&[Word]>, issues: &mut Vec, passed: &mut Vec) -> R<()> { - let out = Command::new("ffmpeg").args(["-v", "error", "-i", video, "-vn", "-ac", "1", "-ar", &SR.to_string(), "-f", "f32le", "-"]).output().map_err(|e| format!("ffmpeg: {e}"))?; - if !out.status.success() { - return Err(format!("ffmpeg couldn't read the sound of {video}: {}", String::from_utf8_lossy(&out.stderr).trim())); - } - let x: Vec = out.stdout.chunks_exact(4).map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]])).collect(); + let x = if crate::read::native() { + crate::read::mono(video, SR as u32)? + } else { + let out = Command::new("ffmpeg").args(["-v", "error", "-i", video, "-vn", "-ac", "1", "-ar", &SR.to_string(), "-f", "f32le", "-"]).output().map_err(|e| format!("ffmpeg: {e}"))?; + if !out.status.success() { + return Err(format!("ffmpeg couldn't read the sound of {video}: {}", String::from_utf8_lossy(&out.stderr).trim())); + } + out.stdout.chunks_exact(4).map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]])).collect() + }; let (rms, bands) = levels(&x); sound_defects(&x, issues, passed); let found = |list: Vec<(f64, String)>, p: u8| crate::defects::moments("sound", p, list); @@ -178,6 +182,23 @@ pub(crate) fn sound(video: &str, words: Option<&[Word]>, issues: &mut Vec let Some(words) = words.filter(|w| !w.is_empty()) else { return Ok(()) }; let speak = speaking(words, rms.len()); // momentary loudness (EBU R128, 400 ms) every 100 ms, from ffmpeg's meter + let (mt, mv) = if crate::read::native() { crate::read::meter(video)? } else { ffmpeg_meter(video)? }; + if let Some((voice, over)) = over_voice(&mt, &mv, &speak) { + if over.is_empty() { + passed.push(format!("sound: nothing without the voice is louder than it ({voice:.1} LUFS momentary)")); + } + issues.extend(found(over, 1)); + } + let bed = bed_up(&rms, &speak); + if bed.is_empty() { + passed.push(format!("sound: the bed sits {BED:.0} dB or more under the voice")); + } + issues.extend(found(bed, 2)); + Ok(()) +} + +/// `ffmpeg ... -af ebur128=metadata=1,ametadata=print:key=lavfi.r128.M:file=-`: each meter reading's time and LUFS +fn ffmpeg_meter(video: &str) -> R<(Vec, Vec)> { let meter = Command::new("ffmpeg") .args(["-nostats", "-hide_banner", "-i", video, "-vn", "-af", "ebur128=metadata=1,ametadata=print:key=lavfi.r128.M:file=-", "-f", "null", "-"]) .output() @@ -191,18 +212,7 @@ pub(crate) fn sound(video: &str, words: Option<&[Word]>, issues: &mut Vec } } mt.truncate(mv.len()); - if let Some((voice, over)) = over_voice(&mt, &mv, &speak) { - if over.is_empty() { - passed.push(format!("sound: nothing without the voice is louder than it ({voice:.1} LUFS momentary)")); - } - issues.extend(found(over, 1)); - } - let bed = bed_up(&rms, &speak); - if bed.is_empty() { - passed.push(format!("sound: the bed sits {BED:.0} dB or more under the voice")); - } - issues.extend(found(bed, 2)); - Ok(()) + Ok((mt, mv)) } pub(crate) fn ffmpeg_stderr(args: &[&str]) -> R { @@ -212,8 +222,9 @@ pub(crate) fn ffmpeg_stderr(args: &[&str]) -> R { /// -14 LUFS integrated (±1) and a true peak at or below -1 dBTP (the floor's mix line) pub(crate) fn loudness(video: &str, taste: Option<&Taste>, issues: &mut Vec, passed: &mut Vec) -> R<()> { - // the same measure `raea world` joins to a render as its Mix (raea-sys-measure, DR2) - let Some((i, peak)) = raea_sys_measure::loudness(std::path::Path::new(video)) else { return Ok(()) }; + // the same measure `raea world` joins to a render as its Mix (raea-sys-measure, DR2), or raea's own without FFmpeg + let measured = if crate::read::native() { crate::read::loudness(video) } else { raea_sys_measure::loudness(std::path::Path::new(video)) }; + let Some((i, peak)) = measured else { return Ok(()) }; let mut ok = true; let target = taste.map_or(-14.0, |t| t.value); if (i - target).abs() > 1.0 { @@ -257,7 +268,11 @@ pub(crate) fn frozen_stretches(log: &str, duration: f64) -> Vec<(f64, f64)> { pub(crate) fn freezes(video: &str, probe: &Probe, taste: Option<&Taste>, issues: &mut Vec, passed: &mut Vec) -> R<()> { let most = taste.map_or(3.0, |t| t.value); - let err = ffmpeg_stderr(&["-hide_banner", "-nostats", "-i", video, "-an", "-vf", &format!("freezedetect=n=-60dB:d={most}"), "-f", "null", "-"])?; + let err = if crate::read::native() { + crate::read::freezes(video, -60.0, most)? + } else { + ffmpeg_stderr(&["-hide_banner", "-nostats", "-i", video, "-an", "-vf", &format!("freezedetect=n=-60dB:d={most}"), "-f", "null", "-"])? + }; let stretches = frozen_stretches(&err, probe.picture); for &(s, d) in &stretches { let says = format!("nothing changes for {d:.1} s; give it a reason to hold, or cut it shorter"); diff --git a/apps/check/src/text.rs b/apps/check/src/text.rs index 9b5f2b18..2cd4110d 100644 --- a/apps/check/src/text.rs +++ b/apps/check/src/text.rs @@ -213,7 +213,8 @@ pub(crate) fn contrast(video: &str, runs: &[Run], w: f64, h: f64, probe: &Probe, } let t = (r.from + r.to) / 2.0; // the same measure `raea world` joins to a text from its render (raea-sys-measure, DR2) - let Some(ratio) = raea_sys_measure::contrast(std::path::Path::new(video), t, crop, &r.color) else { continue }; + let ratio = if crate::read::native() { crate::read::contrast(video, t, crop, &r.color) } else { raea_sys_measure::contrast(std::path::Path::new(video), t, crop, &r.color) }; + let Some(ratio) = ratio else { continue }; measured += 1; if ratio < 3.0 { low += 1; diff --git a/apps/media/src/change.rs b/apps/media/src/change.rs index 8b71aa41..2c6116f6 100644 --- a/apps/media/src/change.rs +++ b/apps/media/src/change.rs @@ -3,7 +3,7 @@ use super::*; /// The decoder's own planes (NV12: luma, then Cb Cr interleaved at half size), as ffmpeg's filters see a 4:2:0 frame. -pub(crate) fn yuv(f: Frame) -> (Vec, Vec, Vec) { +pub fn yuv(f: Frame) -> (Vec, Vec, Vec) { let n = (f.width * f.height) as usize; let mut buf = f.rgba; let uv = buf.split_off(n.min(buf.len())); @@ -18,14 +18,14 @@ pub(crate) fn sad(a: &[u8], b: &[u8]) -> u64 { /// Samples from `from` to `to` at `fps` (the source's frames when `fps` is its rate): each tick's time, and the time /// to read, the last frame before the middle of the tick, as ffmpeg's fps filter picks it (at 6 fps from 30, tick 0 is /// source frame 2 and tick 1 frame 7). -pub(crate) fn times(duration: f64, from: Option, to: Option, fps: f64) -> Vec<(f64, f64)> { +pub fn times(duration: f64, from: Option, to: Option, fps: f64) -> Vec<(f64, f64)> { let (a, b) = (from.unwrap_or(0.0).max(0.0), to.unwrap_or(duration).min(duration)); let n = ((b - a) * fps).round().max(0.0) as usize; (0..n).map(|i| (a + i as f64 / fps, a + (i as f64 + 0.5) / fps - 1e-6)).collect() } /// The video's own duration: its track's (what ffmpeg's filters run to), else the decoder's. -pub(crate) fn video_span(path: &Path, info: &Info) -> f64 { +pub fn video_span(path: &Path, info: &Info) -> f64 { video_track(path).map(|t| t.duration).unwrap_or(info.duration) } @@ -75,7 +75,7 @@ pub fn cuts(path: &Path, threshold: f64, from: Option, to: Option) -> } /// Every frame the file holds as (tick, time to read): its own times when the file states them, else a fixed step. -pub(crate) fn frames_of(path: &Path, info: &Info, from: Option, to: Option) -> Vec<(f64, f64)> { +pub fn frames_of(path: &Path, info: &Info, from: Option, to: Option) -> Vec<(f64, f64)> { match video_track(path) { Some(t) => t.pts.iter().filter(|p| from.is_none_or(|a| **p >= a - 1e-6) && to.is_none_or(|b| **p < b)).map(|p| (*p, *p + 1e-4)).collect(), None => times(info.duration, from, to, info.fps), diff --git a/apps/media/src/lib.rs b/apps/media/src/lib.rs index d26a053a..92c12b1a 100644 --- a/apps/media/src/lib.rs +++ b/apps/media/src/lib.rs @@ -44,10 +44,12 @@ use encode::*; mod ff; use ff::*; mod frames; +/// what `raea check` reads natively where there's no FFmpeg (apps/check read.rs) +pub use {change::{frames_of, freezes, times, video_span, yuv}, probe::{probe_of, standard_fps}, sound::audio_folded}; type R = Result; -fn backend() -> R> { +pub fn backend() -> R> { raea_io_media::backend(&raea_sys_frame::media_backend()) } diff --git a/apps/media/src/probe.rs b/apps/media/src/probe.rs index e76ec64d..43caa79c 100644 --- a/apps/media/src/probe.rs +++ b/apps/media/src/probe.rs @@ -270,7 +270,7 @@ pub fn probe(path: &Path) -> R { /// A measured frame rate (frames over duration) snapped to the standard rate it is within 1% of: a phone's variable-rate /// take that averages 23.92 is 24 (ffprobe's avg_frame_rate says 24/1). -pub(crate) fn standard_fps(fps: f64) -> f64 { +pub fn standard_fps(fps: f64) -> f64 { let rates = [24000.0 / 1001.0, 24.0, 25.0, 30000.0 / 1001.0, 30.0, 48.0, 50.0, 60000.0 / 1001.0, 60.0, 120.0]; // the nearest, within 1% (30 is 30, not 29.97) rates.into_iter().filter(|r| (fps - r).abs() / r < 0.01).min_by(|a, b| (fps - a).abs().total_cmp(&(fps - b).abs())).unwrap_or(fps) diff --git a/docs/direction.feature b/docs/direction.feature index 884889ac..1ff93976 100644 --- a/docs/direction.feature +++ b/docs/direction.feature @@ -69,6 +69,9 @@ Feature: Direction that holds: one world of facts, findings and rules, so a pers | what each frame shows, as laid out | `raea render --manifest` (runtime/export/src/lib.rs write_manifest): each text's words and box, each video's source and time, the 3D's clipping, the sounds heard, the layers | written only when asked for; read by `raea check` alone; its clipping list is a second copy of what was printed | | parts that clip, float, sink | systems/contact (every render), `raea scene lint` | findings are printed text or loose JSON `{check, level, says}` | | an export's floor | apps/check (`raea check`: text, picture, flash, sound; `--quick`; `--taste`) | its own `Issue {rule, p, at, says}`; exports only | + # `raea check` reads a video through FFmpeg where it's installed and through raea's own decoders where it isn't + # (apps/check/read; RAEA_CHECK_NATIVE=1 forces them): the same findings either way, held there by tests/parity.rs. + # 0.1.7 refused any video without FFmpeg, though `raea media probe` and `loudness` never needed it. | a person's standing rules with checks | core/brief/src/taste.rs (`Rule { text, check }`), the taste loop (docs/craft-system.md J) | six closed forms, all about an export's text and sound; a rule is proposed only when a correction repeats; nothing per project or per note | | notes as tasks | review.feature R3 (to do, working, done), a pin's `part` | "done" is the agent's word; nothing proves it or keeps it | | notes mapped to a verb | skills/skills/raea/craft/verbs | one verb a note; a note that holds several asks loses the rest | diff --git a/tools/layers.toml b/tools/layers.toml index ad6b67d3..63892765 100644 --- a/tools/layers.toml +++ b/tools/layers.toml @@ -180,6 +180,7 @@ raea-page = "runtime" raea-app-check = "apps" raea-app-check-flash = "apps" raea-app-check-inbox = "apps" +raea-app-check-read = "apps" raea-app-direction = "apps" raea-app-notes = "apps" # `raea fingerprint`: its verbs and the resident service that keeps the encoders loaded