Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,7 @@ members = [
"apps/check",
"apps/check/flash",
"apps/check/inbox",
"apps/check/read",
"apps/direction",
"apps/notes",
"apps/fingerprint",
Expand Down
2 changes: 2 additions & 0 deletions apps/check/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ raea-io-trace = { path = "../../io/trace" }
raea-io-ffmpeg = { path = "../../io/ffmpeg" }
# caption contrast and loudness, measured as `raea world` measures a render (DR2)
raea-sys-measure = { path = "../../systems/measure" }
# what it reads of a video, through FFmpeg or raea's own decoders where it isn't installed
raea-app-check-read = { path = "read" }
# the flashing screen (WCAG 2.3.1), split out to keep this crate small
raea-app-check-flash = { path = "flash" }
# what the person sent from the raea app, handed to the agent (split out to keep this crate small)
Expand Down
15 changes: 15 additions & 0 deletions apps/check/read/Cargo.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
[package]
name = "raea-app-check-read"
version = "0.1.0"
edition = "2024"
license = "FSL-1.1-ALv2"
publish = false
description = "What `raea check` reads of a video: FFmpeg's probe, and raea's own decoders standing in for FFmpeg where it isn't installed"

[dependencies]
serde_json = "1"
raea-io-home = { path = "../../../io/home" }
raea-io-media = { path = "../../../io/media", default-features = false }
raea-app-media = { path = "../../media" }
# caption contrast, as `raea world` measures it (DR2)
raea-sys-measure = { path = "../../../systems/measure" }
188 changes: 188 additions & 0 deletions apps/check/read/src/lib.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,188 @@
//! What `raea check` reads of a video: through FFmpeg where it's on the PATH (the reference its rules were tuned on),
//! else through raea's own decoders (raea-app-media), so `raea check` works wherever `raea media probe` and `raea media
//! loudness` do, with nothing to install. 0.1.7 refused any video without FFmpeg, which Windows never has (Shane 11-18:
//! "can we make it so ffmpeg does work the way we need it?"). RAEA_CHECK_NATIVE=1 reads natively with FFmpeg there too;
//! tests/parity.rs holds the two to the same answers. Each reader names the FFmpeg command it stands in for.
use raea_io_media::VideoDecoder;
use serde_json::Value;
use std::path::Path;
use std::process::Command;

type R<T> = Result<T, String>;

/// Whether to read natively: FFmpeg's ffmpeg or ffprobe isn't on the PATH, or RAEA_CHECK_NATIVE is on
pub fn native() -> bool {
raea_io_home::env_on("RAEA_CHECK_NATIVE") || !(on_path("ffmpeg") && on_path("ffprobe"))
}

fn on_path(tool: &str) -> bool {
let exe = if cfg!(windows) { format!("{tool}.exe") } else { tool.to_string() };
std::env::var_os("PATH").is_some_and(|p| std::env::split_paths(&p).any(|d| d.join(&exe).is_file()))
}

fn open(video: &str) -> R<(Box<dyn VideoDecoder>, raea_io_media::Info)> {
let d = raea_app_media::backend()?.open_video(Path::new(video))?;
let info = d.info().clone();
Ok((d, info))
}

/// A luma byte as FFmpeg's format=gray gives it from a limited-range picture: full range, 16-235 spread to 0-255 and
/// rounded (FFmpeg 9: 16 is 0, 126 is 128, 235 is 255)
fn gray(y: u8) -> u8 {
(((y.clamp(16, 235) as u32 - 16) * 255 + 109) / 219) as u8
}

/// What `raea check` needs to know of a video before it reads it
pub struct Probe {
pub w: f64,
pub h: f64,
pub duration: f64,
/// when the picture ends, on the clock FFmpeg's filters read: it starts at the file's first stream, sound can run on
/// past the picture, and the container's duration counts that
pub picture: f64,
pub audio: bool,
}

/// The picture's size, the file's duration, where the picture ends and whether there's sound: ffprobe's stream and
/// format entries, or raea's own probe without FFmpeg
pub fn probe(video: &str) -> R<Probe> {
if native() {
let p = raea_app_media::probe_of(Path::new(video))?;
let v = p.video.as_ref().ok_or("no video stream")?;
let picture = p.track.as_ref().map_or(v.duration, |t| t.duration);
return Ok(Probe { w: v.width as f64, h: v.height as f64, duration: p.duration.max(picture), picture, audio: p.audio.is_some() });
}
let entries = "stream=codec_type,width,height,r_frame_rate,duration,start_time:format=duration,start_time";
let out = Command::new("ffprobe").args(["-v", "error", "-show_entries", entries, "-of", "json", video]).output().map_err(|e| format!("ffprobe: {e}"))?;
let v: Value = serde_json::from_slice(&out.stdout).map_err(|e| format!("ffprobe: {e}"))?;
let streams = v["streams"].as_array().cloned().unwrap_or_default();
let vid = streams.iter().find(|s| s["codec_type"] == "video").ok_or("no video stream")?;
let seconds = |field: &Value| -> Option<f64> { field.as_str()?.parse().ok() };
let duration = seconds(&v["format"]["duration"]).unwrap_or(0.0);
let origin = seconds(&v["format"]["start_time"]).unwrap_or(0.0);
let picture = match seconds(&vid["duration"]) {
Some(length) => seconds(&vid["start_time"]).unwrap_or(origin) - origin + length,
None => last_packet_end(video).map_or(duration, |end| end - origin),
};
let audio = streams.iter().any(|s| s["codec_type"] == "audio");
Ok(Probe { w: vid["width"].as_f64().unwrap_or(1920.0), h: vid["height"].as_f64().unwrap_or(1080.0), duration, picture, audio })
}

/// Where the picture's last packet ends, for a container that doesn't state the video stream's own length (Matroska)
fn last_packet_end(video: &str) -> Option<f64> {
let out = Command::new("ffprobe").args(["-v", "error", "-select_streams", "v:0", "-show_entries", "packet=pts_time,duration_time", "-of", "csv=p=0", video]).output().ok()?;
String::from_utf8_lossy(&out.stdout)
.lines()
.filter_map(|l| {
let mut parts = l.split(',');
let t: f64 = parts.next()?.trim().parse().ok()?;
Some(t + parts.next().and_then(|d| d.trim().parse::<f64>().ok()).unwrap_or(0.0))
})
.max_by(f64::total_cmp)
}

/// `ffmpeg -i V -vf scale=32:18:flags=area,format=gray -f rawvideo -` averaged a frame, and the rate `ffmpeg -i` says
/// (defects.rs lumas): each frame's mean light, 0 to 1, and the frames a second
pub fn lumas(video: &str) -> R<(Vec<f32>, f64)> {
let (mut d, info) = open(video)?;
let mut out = Vec::new();
for (_, at) in raea_app_media::frames_of(Path::new(video), &info, None, None) {
let (y, ..) = raea_app_media::yuv(d.frame_nv12_at(at)?);
out.push(y.iter().map(|&p| gray(p) as f32).sum::<f32>() / (y.len().max(1) as f32 * 255.0));
}
Ok((out, raea_app_media::standard_fps(info.fps)))
}

/// `ffmpeg -i V -vf fps=0.5,crop=W:H:0:0,format=gray -f rawvideo -` (picture.rs blotching): every 2 s, the picture's
/// top-left W x H in grey
pub fn crops(video: &str, w: usize, h: usize, mut each: impl FnMut(&[u8])) -> R<()> {
let (mut d, info) = open(video)?;
let frames = raea_app_media::frames_of(Path::new(video), &info, None, None);
for (tick, _) in raea_app_media::times(raea_app_media::video_span(Path::new(video), &info), None, None, 0.5) {
// the last frame before the tick's middle, as the fps filter picks it (by the frames' own times: a read at the
// middle less a microsecond came back a frame late)
let Some(&(_, at)) = frames.iter().take_while(|(pts, _)| *pts < tick + 1.0).last() else { continue };
let f = d.frame_nv12_at(at)?;
let fw = f.width as usize;
let (y, ..) = raea_app_media::yuv(f);
let crop: Vec<u8> = (0..h.min(y.len() / fw.max(1))).flat_map(|r| y[r * fw..r * fw + w.min(fw)].iter().map(|&p| gray(p))).collect();
each(&crop);
}
Ok(())
}

/// flash.rs's graph (showinfo, the picture at W x H area-sampled beside it point-sampled, rgb24): every frame with
/// its time, until `each` says stop. How many frames were read.
pub fn flash_frames(video: &str, w: usize, h: usize, mut each: impl FnMut(f64, &[u8])) -> R<usize> {
let (mut d, info) = open(video)?;
let mut buf = vec![0u8; w * 2 * h * 3];
let mut n = 0;
for (pts, at) in raea_app_media::frames_of(Path::new(video), &info, None, None) {
let f = d.frame_at(at)?;
let (sw, sh) = (f.width as usize, f.height as usize);
for y in 0..h {
let (y0, y1) = (y * sh / h, ((y + 1) * sh / h).max(y * sh / h + 1));
for x in 0..w {
let (x0, x1) = (x * sw / w, ((x + 1) * sw / w).max(x * sw / w + 1));
let mut sum = [0u32; 3];
for sy in y0..y1 {
for p in f.rgba[(sy * sw + x0) * 4..(sy * sw + x1) * 4].chunks_exact(4) {
(0..3).for_each(|c| sum[c] += p[c] as u32);
}
}
let count = ((y1 - y0) * (x1 - x0)) as u32;
let near = ((y * sh + sh / 2) / h * sw + (x * sw + sw / 2) / w) * 4;
for c in 0..3 {
buf[(y * w * 2 + x) * 3 + c] = ((sum[c] + count / 2) / count) as u8;
buf[(y * w * 2 + w + x) * 3 + c] = f.rgba[near + c];
}
}
}
each(pts, &buf);
n += 1;
}
Ok(n)
}

/// `ffmpeg -i V -vn -ac 1 -ar RATE -f f32le -` (sound.rs sound): the sound folded to mono at `rate`
pub fn mono(video: &str, rate: u32) -> R<Vec<f32>> {
Ok(raea_app_media::audio_folded(Path::new(video), rate, true, None, None, false)?.channels.swap_remove(0))
}

/// `ffmpeg -i V -vn -af ebur128=metadata=1,ametadata=print:key=lavfi.r128.M:file=- -f null -` (sound.rs sound): the
/// momentary loudness (400 ms) every 100 ms, each time and its LUFS
pub fn meter(video: &str) -> R<(Vec<f64>, Vec<f64>)> {
let a = raea_app_media::audio_folded(Path::new(video), raea_io_media::audio::RATE, false, None, None, false)?;
let n = (a.duration() * 10.0).floor() as usize;
Ok(((0..n).map(|i| i as f64 / 10.0).collect(), raea_io_media::audio::momentary(&a, 10.0, n)))
}

/// `ffmpeg -i V -an -vf freezedetect=n=NOISE_DB:d=MIN -f null -`'s log (sound.rs freezes reads it either way)
pub fn freezes(video: &str, noise_db: f64, min: f64) -> R<String> {
let found = raea_app_media::freezes(Path::new(video), 10f64.powf(noise_db / 20.0), min)?;
Ok(found.into_iter().map(|(s, end)| match end {
Some((d, e)) => format!("freeze_start: {s}\nfreeze_duration: {d}\nfreeze_end: {e}\n"),
None => format!("freeze_start: {s}\n"),
}).collect())
}

/// raea_sys_measure::contrast's read (`ffmpeg -ss AT -i V -frames:v 1 -vf crop=... -pix_fmt rgb24`): a text's contrast
/// against what's behind it, None when the crop falls outside the frame
pub fn contrast(video: &str, at: f64, crop: [u32; 4], color: &str) -> Option<f64> {
let [x, y, w, h] = [crop[0] & !1, crop[1] & !1, crop[2].max(2) & !1, crop[3].max(2) & !1].map(|v| v as usize);
let (mut d, _) = open(video).ok()?;
let f = d.frame_at(at).ok()?;
let fw = f.width as usize;
if x + w > fw || y + h > f.height as usize {
return None;
}
let rgb: Vec<u8> = (y..y + h).flat_map(|r| f.rgba[(r * fw + x) * 4..(r * fw + x + w) * 4].chunks_exact(4).flat_map(|p| [p[0], p[1], p[2]]).collect::<Vec<_>>()).collect();
raea_sys_measure::ratio_behind(&rgb, raea_sys_measure::lum(raea_sys_measure::rgb_of(color)))
}

/// raea_sys_measure::loudness's read (`ffmpeg -i V -vn -af ebur128=peak=true -f null -`): the integrated loudness and
/// true peak, None when there's no sound
pub fn loudness(video: &str) -> Option<(f64, Option<f64>)> {
let a = raea_app_media::audio_folded(Path::new(video), raea_io_media::audio::RATE, false, None, None, false).ok().filter(|a| a.len() > 0)?;
Some((raea_io_media::audio::lufs(&a), Some(raea_io_media::audio::true_peak_db(&a))))
}
126 changes: 126 additions & 0 deletions apps/check/read/tests/parity.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
//! Each native reader against the FFmpeg read it stands in for (lib.rs names the command), on a clip FFmpeg makes: a
//! moving test picture that freezes for its last 4 s, over stereo sound whose level swells. `raea check` without FFmpeg
//! must reach the verdicts it reaches with it, so each answer is held to FFmpeg's within what a rule could notice.
//! Skipped where FFmpeg isn't installed (there is nothing to compare against).

use std::path::Path;
use std::process::Command;

fn ffmpeg(args: &[&str]) -> (Vec<u8>, String) {
let out = Command::new("ffmpeg").args(["-hide_banner", "-nostats", "-nostdin"]).args(args).output().expect("ffmpeg runs");
assert!(out.status.success(), "ffmpeg {args:?}: {}", String::from_utf8_lossy(&out.stderr));
(out.stdout, String::from_utf8_lossy(&out.stderr).into_owned())
}

fn clip(dir: &Path) -> String {
let file = dir.join("clip.mp4").to_string_lossy().into_owned();
let picture = "testsrc2=size=320x180:rate=30:duration=5,tpad=stop_mode=clone:stop_duration=4";
let sound = "aevalsrc=0.4*sin(2*PI*440*t)*(0.55+0.45*sin(2*PI*0.25*t))|0.25*sin(2*PI*660*t):s=48000:d=9";
ffmpeg(&["-y", "-f", "lavfi", "-i", picture, "-f", "lavfi", "-i", sound, "-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "aac", "-b:a", "192k", "-shortest", &file]);
file
}

/// The mean absolute difference of two byte runs, in levels
fn apart(a: &[u8], b: &[u8]) -> f64 {
a.iter().zip(b).map(|(x, y)| x.abs_diff(*y) as u64).sum::<u64>() as f64 / a.len().max(1) as f64
}

/// Root-mean-square level in dB of each 100 ms of `x` at `rate`
fn levels(x: &[f32], rate: usize) -> Vec<f64> {
x.chunks_exact(rate / 10).map(|c| 10.0 * (c.iter().fold(0.0, |s, v| s + v * v) as f64 / c.len() as f64 + 1e-12).log10()).collect()
}

/// Each measure's name, how far the native read is from FFmpeg's, and how far it may be: all printed, then held
struct Off(Vec<(String, f64, f64)>);

impl Off {
fn by(&mut self, what: impl Into<String>, off: f64, most: f64) {
self.0.push((what.into(), off, most));
}
}

#[test]
fn the_native_reads_match_ffmpeg_s() {
if Command::new("ffmpeg").arg("-version").output().is_err() {
eprintln!("skipped: no FFmpeg to compare against");
return;
}
let dir = std::env::temp_dir().join(format!("raea-check-read-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let v = &clip(&dir);
let mut off = Off(vec![]);
let by_ffmpeg = raea_app_check_read::probe(v).unwrap();
// SAFETY: set before any reader runs, on this test's only thread
unsafe { std::env::set_var("RAEA_CHECK_NATIVE", "1") };
assert!(raea_app_check_read::native());
let p = raea_app_check_read::probe(v).unwrap();
assert_eq!((p.w, p.h, p.audio), (by_ffmpeg.w, by_ffmpeg.h, by_ffmpeg.audio));
assert!((p.duration - by_ffmpeg.duration).abs() < 0.05 && (p.picture - by_ffmpeg.picture).abs() < 0.05, "{} {} vs {} {}", p.duration, p.picture, by_ffmpeg.duration, by_ffmpeg.picture);

// flash-frame's light per frame
let (gray, _) = ffmpeg(&["-v", "error", "-i", v, "-map", "0:v:0", "-vf", "scale=32:18:flags=area,format=gray", "-f", "rawvideo", "-"]);
let theirs: Vec<f64> = gray.chunks_exact(32 * 18).map(|f| f.iter().map(|&b| b as f64).sum::<f64>() / (32.0 * 18.0 * 255.0)).collect();
let (ours, fps) = raea_app_check_read::lumas(v).unwrap();
assert_eq!((ours.len(), fps), (theirs.len(), 30.0));
off.by("a frame's light, of 1", ours.iter().zip(&theirs).map(|(a, b)| (*a as f64 - b).abs()).fold(0.0, f64::max), 0.01);

// blotching's dark crops every 2 s
let (w, h) = (320, 176);
let (gray, _) = ffmpeg(&["-v", "error", "-i", v, "-vf", &format!("fps=0.5,crop={w}:{h}:0:0,format=gray"), "-f", "rawvideo", "-"]);
let mut crops = vec![];
raea_app_check_read::crops(v, w, h, |c| crops.push(c.to_vec())).unwrap();
assert_eq!(crops.len(), gray.len() / (w * h));
for (i, (a, b)) in crops.iter().zip(gray.chunks_exact(w * h)).enumerate() {
let bias = a.iter().zip(b).map(|(x, y)| *x as f64 - *y as f64).sum::<f64>() / a.len() as f64;
off.by(format!("blotching crop {i}, levels (bias {bias:+.2})"), apart(a, b), 3.0);
}

// the flashing screen's frames: area-sampled beside point-sampled RGB
let (fw, fh) = (192, 108);
let graph = format!("split=2[area][point];[area]scale={fw}:{fh}:flags=area[a];[point]scale={fw}:{fh}:flags=neighbor[b];[a][b]hstack=inputs=2,format=rgb24");
let (rgb, _) = ffmpeg(&["-v", "error", "-i", v, "-map", "0:v:0", "-vf", &graph, "-f", "rawvideo", "-"]);
let (mut frames, mut times) = (vec![], vec![]);
let n = raea_app_check_read::flash_frames(v, fw, fh, |t, buf| {
times.push(t);
frames.push(buf.to_vec());
})
.unwrap();
assert_eq!(n, rgb.len() / (fw * 2 * fh * 3));
assert!(times.windows(2).all(|t| (t[1] - t[0] - 1.0 / 30.0).abs() < 1e-3), "frame times step by a frame");
off.by("a flash frame, levels", frames.iter().zip(rgb.chunks_exact(fw * 2 * fh * 3)).map(|(a, b)| apart(a, b)).fold(0.0, f64::max), 6.0);

// the sound folded to mono, 100 ms at a time
let (pcm, _) = ffmpeg(&["-v", "error", "-i", v, "-vn", "-ac", "1", "-ar", "24000", "-f", "f32le", "-"]);
let theirs = levels(&pcm.chunks_exact(4).map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]])).collect::<Vec<_>>(), 24000);
let ours = levels(&raea_app_check_read::mono(v, 24000).unwrap(), 24000);
assert!(ours.len().abs_diff(theirs.len()) <= 1, "{} windows vs {}", ours.len(), theirs.len());
off.by("100 ms of mono sound, dB", ours.iter().zip(&theirs).skip(1).map(|(a, b)| (a - b).abs()).fold(0.0, f64::max), 0.5);

// momentary loudness every 100 ms
let (log, _) = ffmpeg(&["-i", v, "-vn", "-af", "ebur128=metadata=1,ametadata=print:key=lavfi.r128.M:file=-", "-f", "null", "-"]);
let theirs: Vec<f64> = String::from_utf8_lossy(&log).lines().filter_map(|l| l.strip_prefix("lavfi.r128.M=")?.trim().parse().ok()).collect();
let (at, ours) = raea_app_check_read::meter(v).unwrap();
assert!(ours.len().abs_diff(theirs.len()) <= 2 && at.len() == ours.len(), "{} readings vs {}", ours.len(), theirs.len());
off.by("a momentary loudness, LU", ours.iter().zip(&theirs).skip(4).map(|(a, b)| (a - b).abs()).fold(0.0, f64::max), 0.5);

// the freeze: 5 s in, to the end
let (_, log) = ffmpeg(&["-i", v, "-an", "-vf", "freezedetect=n=-60dB:d=3", "-f", "null", "-"]);
let starts = |log: &str| log.lines().filter_map(|l| l.split("freeze_start: ").nth(1)?.trim().parse::<f64>().ok()).collect::<Vec<_>>();
let (theirs, ours) = (starts(&log), starts(&raea_app_check_read::freezes(v, -60.0, 3.0).unwrap()));
assert!(ours.len() == 1 && theirs.len() == 1, "freezes from {ours:?} vs {theirs:?}");
off.by("the freeze's start, s", (ours[0] - theirs[0]).abs(), 0.1);

// the mix's loudness and a caption's contrast, against raea-sys-measure's FFmpeg reads
let (i, tp) = raea_app_check_read::loudness(v).unwrap();
let (fi, ftp) = raea_sys_measure::loudness(Path::new(v)).unwrap();
off.by("integrated loudness, LU", (i - fi).abs(), 0.3);
off.by("true peak, dB", (tp.unwrap() - ftp.unwrap()).abs(), 0.5);
let ours = raea_app_check_read::contrast(v, 1.0, [40, 60, 120, 60], "rgb(255, 255, 255)").unwrap();
let theirs = raea_sys_measure::contrast(Path::new(v), 1.0, [40, 60, 120, 60], "rgb(255, 255, 255)").unwrap();
off.by("a caption's contrast, share", (ours / theirs - 1.0).abs(), 0.05);
let _ = std::fs::remove_dir_all(&dir);
for (what, by, most) in &off.0 {
eprintln!("{} {what}: {by:.4} (at most {most})", if by < most { "ok " } else { "OVER" });
}
assert!(off.0.iter().all(|(_, by, most)| by < most), "a native read is further from FFmpeg's than a rule allows");
}
Loading
Loading